Introduction

The goal of this project is just to display that you’ve gotten used to working with the data and that you are on track to create your prediction algorithm. Please submit a report on R Pubs (http://rpubs.com/) that explains your exploratory analysis and your goals for the eventual app and algorithm. This document should be concise and explain only the major features of the data you have identified and briefly summarize your plans for creating the prediction algorithm and Shiny app in a way that would be understandable to a non-data scientist manager. You should make use of tables and plots to illustrate important summaries of the data set. The motivation for this project is to: 1. Demonstrate that you’ve downloaded the data and have successfully loaded it in. 2. Create a basic report of summary statistics about the data sets. 3. Report any interesting findings that you amassed so far. 4. Get feedback on your plans for creating a prediction algorithm and Shiny app.

Loading and reading data

library(stringi)

blogs_file <- "data/en_US/en_US.blogs.txt"
blogs <- readLines(blogs_file, warn = FALSE, encoding = "UTF-8", skipNul = TRUE)

twitter_file <- "data/en_US/en_US.twitter.txt"
twitter <- readLines(twitter_file, warn = FALSE, encoding = "UTF-8", skipNul = TRUE)

news_file <- "data/en_US/en_US.news.txt"
news <- readLines(news_file, warn = FALSE, encoding = "UTF-8", skipNul = TRUE)

Preparing the dataset

twitter_list <- stri_count_words(twitter)
blogs_list <- stri_count_words(blogs)
news_list <- stri_count_words(news)

twitter_words <- sum(twitter_list)
twitter_lines <- length(twitter)

blogs_words <- sum(blogs_list)
blogs_lines <- length(blogs)

news_words <- sum(news_list)
news_lines <- length(news)

cat("Dataset", "\t", "Words", "\t", "\t", "Lines",  "\t", "Average words per line", "\n",
    "Twitter", "\t", twitter_words, "\t", twitter_lines, "\t", twitter_words/twitter_lines, "\n",
    "Blogs", "\t", blogs_words, "\t", blogs_lines, "\t", blogs_words/blogs_lines, "\n",
    "News", "\t", "\t", news_words, "\t", "\t", news_lines, "\t", news_words/news_lines, "\n", sep = "")
## Dataset  Words       Lines   Average words per line
## Twitter  30096690    2360148 12.75204
## Blogs    37546806    899288  41.7517
## News     2674561     77259   34.61812

Visualize corrent data

par(mfrow = c(3, 1), mar = c(4, 4, 2, 1)) 
hist(twitter_list, xlab = "Distribution (words per line)", main = "TWITTER", 
     xlim = c(0, 40), breaks = 50)
hist(blogs_list, xlab = "Distribution (words per line)", main = "BLOGS", 
     xlim = c(0, 200), breaks = 500)
hist(news_list, xlab = "Distribution (words per line)", main = "NEWS", 
     xlim = c(0, 200), breaks = 500)

blogs_file <- "data/en_US/en_US.blogs.txt"
blogs <- readLines(blogs_file, warn = FALSE, encoding = "UTF-8", skipNul = TRUE)

twitter_file <- "data/en_US/en_US.twitter.txt"
twitter <- readLines(twitter_file, warn = FALSE, encoding = "UTF-8", skipNul = TRUE)

news_file <- "data/en_US/en_US.news.txt"
news <- readLines(news_file, warn = FALSE, encoding = "UTF-8", skipNul = TRUE)

Pre-processing the training datasets

set.seed(123)
twitter_s <- sample(twitter, length(twitter) * 0.01)
blogs_s <- sample(blogs, length(blogs) * 0.01)
news_s <- sample(news, length(news) * 0.01)
tbn <- c(twitter_s, blogs_s, news_s); rm(twitter_s, blogs_s, news_s)
#summary
tbn_corpus <- as.list(strsplit(tbn, " "))
tbn_corpus <- unlist(tbn_corpus)
#Delete non-alphabetic symbols (puncutation, numbers, etc.).

tbn_corpus <- strsplit(gsub("[^[:alnum:] ]", "", tbn_corpus), " +")
tbn_corpus <- unlist(tbn_corpus)

output

top10 <- as.data.frame(table(tbn_corpus[nchar(tbn_corpus) > 6]))
top10 <- top10[order(top10$Freq, decreasing = T),]
top10 <- top10[c(1:10),]
names(top10)[1] <- paste("word"); names(top10)[2] <- paste("freq")
#plot 
barplot(top10$freq, names = top10$word, las = 2)