# librerias
library(dplyr)
library(tidyr)
library(ggplot2)
library(tm)
library(readr)
library(SnowballC)
library(wordcloud)
library(R.utils)
library(stringi)
library(textmineR)
library(tokenizers)

Introduction

The goal of this project is just to display that you’ve gotten used to working with the data and that you are on track to create your prediction algorithm. That explains your exploratory analysis and your goals for the eventual app and algorithm. This document should be concise and explain only the major features of the data you have identified and briefly summarize your plans for creating the prediction algorithm and Shiny app in a way that would be understandable to a non-data scientist manager. You should make use of tables and plots to illustrate important summaries of the data set.

Summary of text files used

We have three text files from different sources: Twitter, blogs and news. We are going to use the documents in English.

# Obtener archivos
con <- file("en_US_twitter.txt", "rb")
twitter <- readLines(con, encoding = "UTF-8", skipNul=TRUE)
close(con)
con <- file("en_US_blogs.txt", "rb")
blogs <- readLines(con, encoding = "UTF-8", skipNul=TRUE)
close(con)
con <- file("en_US_news.txt", "rb")
news <- readLines(con, encoding = "UTF-8", skipNul=TRUE)
close(con)

We generate a table with the most general information of these files.
We answer to Has the data scientist done basic summaries of the three files? Word counts, line counts and basic data tables?

# Tamaño
S_twitter <- round(file.info("en_US_twitter.txt")$size/1024^2,2)
S_blogs <- round(file.info("en_US_blogs.txt")$size/1024^2,2)
S_news <- round(file.info("en_US_news.txt")$size/1024^2,2)

# Lineas
L_twitter <- length(twitter)
L_blogs <- length(blogs)
L_news <- length(news)

# Palabras
W_twitter <- sum(stri_count_words(twitter))
W_blogs <- sum(stri_count_words(blogs))
W_news <- sum(stri_count_words(news))

# Caracteres
Ch_twitter <- sum(nchar(twitter))
Ch_blogs <- sum(nchar(blogs))
Ch_news <- sum(nchar(news)) 

# Tabla resumen
tb_summary <- data.frame(
  "File_Name" = c("twitter", "blogs", "news"),
  "Size(MB)" = c(S_twitter,S_blogs,S_news),
  "Number_Lines" = c(L_twitter,L_blogs,L_news),
  "Number_Words" = c(W_twitter,W_blogs,W_news),
  "Number_Characters" = c(Ch_twitter,Ch_blogs,Ch_news))
tb_summary
##   File_Name Size.MB. Number_Lines Number_Words Number_Characters
## 1   twitter   159.36      2360148     30096690         162096241
## 2     blogs   200.42       899288     37546806         206824505
## 3      news   196.28      1010242     34762658         203223159

View information

As the files are excessively large, we only work with a part of them.
We answer to Has the data scientist made basic plots, such as histograms to illustrate features of the data?

set.seed(1234)
twitter_small <- sample(twitter, size = 1000)
blogs_small <- sample(blogs, size = 1000)
news_small <- sample(news, size = 1000)

dt_small <- c(twitter_small,blogs_small,news_small)

# Crear CORPUS
corpus <- VCorpus(VectorSource(dt_small))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeWords, stopwords())
corpus <- tm_map(corpus, stemDocument)
corpus <- tm_map(corpus, stripWhitespace)

# Nube de palabras
nov_ptd <- tm_map(corpus, PlainTextDocument)
wordcloud(nov_ptd, max.words = 95, random.order = F, colors = brewer.pal(name = "Dark2", n = 8))

nov_tdm <- TermDocumentMatrix(corpus)
nov_tdm
## <<TermDocumentMatrix (terms: 10138, documents: 3000)>>
## Non-/sparse entries: 43320/30370680
## Sparsity           : 100%
## Maximal term length: 44
## Weighting          : term frequency (tf)
nov_mat <- as.matrix(nov_tdm)
dim(nov_mat)
## [1] 10138  3000
nov_mat <- nov_mat %>% rowSums() %>% sort(decreasing = TRUE)
nov_mat <- data.frame(palabra = names(nov_mat), frec = nov_mat)

wordcloud(
  words = nov_mat$palabra,
  freq = nov_mat$frec,
  max.words = 80,
  random.order = F,
  colors=brewer.pal(name = "Dark2", n = 8))

nov_mat[1:20, ] %>% 
  arrange()
##       palabra frec
## said     said  327
## will     will  287
## one       one  280
## like     like  261
## just     just  248
## time     time  245
## get       get  240
## can       can  217
## year     year  204
## see       see  168
## make     make  162
## want     want  160
## day       day  159
## know     know  149
## love     love  147
## new       new  145
## work     work  145
## first   first  142
## good     good  134
## think   think  133
ggplot((nov_mat[1:10, ] %>% arrange()),aes(palabra, frec)) +
  geom_bar(stat = "identity", position = "dodge", color = "black", fill = "#87CEFA") +
  geom_text(aes(hjust = 1.3, label = frec)) +
  coord_flip() +
  labs(title = "Ten most frequent words in the files under study", x = "WORDS", y = "FREQUENCY")

Conclusion and next steps

We should continue with the study to see what set of words usually appear, being able to predict the words to be used by users.