Setup Load initial packages and clear the global workspace.

library(knitr) rm(list = ls(all.names = TRUE)) setwd(“~/Coursera/Data Science Capstone/Project”)

Load the Data Download, unzip and load the training data.

The files are downloaded and stored locally: * Blog: en_US.blogs.txt * News: en_US.news.txt * Twitter: en_US.twitter.txt

trainURL <- “https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip” trainDataFile <- “data/Coursera-SwiftKey.zip” if (!file.exists(‘data’)) { dir.create(‘data’) } if (!file.exists(“data/final/en_US”)) { tempFile <- tempfile() download.file(trainURL, tempFile) unzip(tempFile, exdir = “data”) unlink(tempFile) }

blogs

blogsFileName <- “data/final/en_US/en_US.blogs.txt” con <- file(blogsFileName, open = “r”) blogs <- readLines(con, encoding = “UTF-8”, skipNul = TRUE) close(con)

news

newsFileName <- “data/final/en_US/en_US.news.txt” con <- file(newsFileName, open = “r”) news <- readLines(con, encoding = “UTF-8”, skipNul = TRUE)

close(con)

twitter

twitterFileName <- “data/final/en_US/en_US.twitter.txt” con <- file(twitterFileName, open = “r”) twitter <- readLines(con, encoding = “UTF-8”, skipNul = TRUE) close(con) rm(con)

Basic Data Summary

Prior to building the unified document corpus and cleaning the data, a basic summary of the three text corpora is being provided which includes file sizes, number of lines, number of characters, and number of words for each source file. Also included are basic statistics on the number of words per line (min, mean, and max).

install.packages(“kableExtra)

library(stringi) library(kableExtra) # assign sample size #sampleSize = 0.01 sampleSize = 0.002 # file size fileSizeMB <- round(file.info(c(blogsFileName, newsFileName, twitterFileName))$size / 1024 ^ 2) # num lines per file numLines <- sapply(list(blogs, news, twitter), length) # num characters per file numChars <- sapply(list(nchar(blogs), nchar(news), nchar(twitter)), sum) # num words per file numWords <- sapply(list(blogs, news, twitter), stri_stats_latex)[4,] # words per line wpl <- lapply(list(blogs, news, twitter), function(x) stri_count_words(x)) # words per line summary wplSummary = sapply(list(blogs, news, twitter), function(x) summary(stri_count_words(x))[c(‘Min.’, ‘Mean’, ‘Max.’)]) rownames(wplSummary) = c(‘WPL.Min’, ‘WPL.Mean’, ‘WPL.Max’) summary <- data.frame( File = c(“en_US.blogs.txt”, “en_US.news.txt”, “en_US.twitter.txt”), FileSize = paste(fileSizeMB, ” MB”), Lines = numLines, Characters = numChars, Words = numWords, t(rbind(round(wplSummary))) ) kable(summary, row.names = FALSE, align = c(“l”, rep(“r”, 7)), caption = ““) %>% kable_styling(position =”left”)

install.packages(“ggplot2”) install.packages(“gridExtra”) install.packages(“crayon”)

library(ggplot2) library(gridExtra) plot1 <- qplot(wpl[[1]], geom = “histogram”, main = “US Blogs”, xlab = “Words per Line”, ylab = “Frequency”, binwidth = 5) plot2 <- qplot(wpl[[2]], geom = “histogram”, main = “US News”, xlab = “Words per Line”, ylab = “Frequency”, binwidth = 5) plot3 <- qplot(wpl[[3]], geom = “histogram”, main = “US Twitter”, xlab = “Words per Line”, ylab = “Frequency”, binwidth = 1) plotList = list(plot1, plot2, plot3) do.call(grid.arrange, c(plotList, list(ncol = 1))) # free up some memory rm(plot1, plot2, plot3)

###Sample and Clean the Data # set seed for reproducability set.seed(666) # sample all three data sets sampleBlogs <- sample(blogs, length(blogs) * sampleSize, replace = FALSE) sampleNews <- sample(news, length(news) * sampleSize, replace = FALSE) sampleTwitter <- sample(twitter, length(twitter) * sampleSize, replace = FALSE) # remove all non-English characters from the sampled data sampleBlogs <- iconv(sampleBlogs, “latin1”, “ASCII”, sub = ““) sampleNews <- iconv(sampleNews,”latin1”, “ASCII”, sub = ““) sampleTwitter <- iconv(sampleTwitter,”latin1”, “ASCII”, sub = ““) # combine all three data sets into a single data set and write to disk sampleData <- c(sampleBlogs, sampleNews, sampleTwitter) sampleDataFileName <-”data/final/en_US/en_US.sample.txt” con <- file(sampleDataFileName, open = “w”) writeLines(sampleData, con) close(con) # get number of lines and words from the sample data set sampleDataLines <- length(sampleData); sampleDataWords <- sum(stri_count_words(sampleData)) # remove variables no longer needed to free up memory rm(blogs, news, twitter, sampleBlogs, sampleNews, sampleTwitter)

R Markdown

This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see http://rmarkdown.rstudio.com.

When you click the Knit button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this:

summary(cars)
##      speed           dist       
##  Min.   : 4.0   Min.   :  2.00  
##  1st Qu.:12.0   1st Qu.: 26.00  
##  Median :15.0   Median : 36.00  
##  Mean   :15.4   Mean   : 42.98  
##  3rd Qu.:19.0   3rd Qu.: 56.00  
##  Max.   :25.0   Max.   :120.00

Including Plots

You can also embed plots, for example:

Note that the echo = FALSE parameter was added to the code chunk to prevent printing of the R code that generated the plot.