The goal of the capstone project is to create a predictive text model using a large text corpus of documents as training data. Natural language processing techniques will be used to perform the analysis.
if (!dir.exists("data")) {
url <- "https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip"
if (!file.exists("Coursera-SwiftKey.zip")) {
download.file(url, destfile = "Coursera-SwiftKey.zip", method = "curl")
unzip("Coursera-SwiftKey.zip")
}
}
news <- readLines("final/en_US/en_US.news.txt", warn = FALSE, skipNul = TRUE)
twitter <- readLines("final/en_US/en_US.twitter.txt", skipNul = TRUE)
blogs <- readLines("final/en_US/en_US.blogs.txt", skipNul = TRUE)
library(stringi)
data.table <- matrix(0, nrow = 3, ncol = 2, dimnames = list(c("twitter", "blogs", "news"),c("lines", "words")))
twitter_num_lines <- length(twitter)
blogs_num_lines <- length(blogs)
news_num_lines <- length(news)
twitter_num_words <- sum(stri_count_words(twitter))
blogs_num_words <- sum(stri_count_words(blogs))
news_num_words <- sum(stri_count_words(news))
data.table[1,1] <- twitter_num_lines
data.table[1,2] <- twitter_num_words
data.table[2,1] <- blogs_num_lines
data.table[2,2] <- blogs_num_words
data.table[3,1] <- news_num_lines
data.table[3,2] <- news_num_words
library(knitr)
kable(data.table)
| lines | words | |
|---|---|---|
| 2360148 | 30096690 | |
| blogs | 899288 | 37546806 |
| news | 77259 | 2674561 |
set.seed(1)
sample_data <- c(sample(blogs, length(blogs)*0.01),sample(news, length(news)*0.01),sample(twitter, length(twitter)*0.01))
rm(blogs, news, twitter)
library(tm)
## Warning: package 'tm' was built under R version 4.3.3
## Loading required package: NLP
text_corpus <- VCorpus(VectorSource(sample_data))
text_corpus <- tm_map(text_corpus, content_transformer(tolower)) #trasform the text in lowercase characters
text_corpus <- tm_map(text_corpus, removePunctuation) #remove the punctuation
text_corpus <- tm_map(text_corpus, removeNumbers) #remove all the numbers
text_corpus <- tm_map(text_corpus, removeWords, stopwords("english")) #remove the stopwords
text_corpus <- tm_map(text_corpus, stripWhitespace) #remove the whitespaces
library(ggplot2)
## Warning: package 'ggplot2' was built under R version 4.3.3
##
## Attaching package: 'ggplot2'
## The following object is masked from 'package:NLP':
##
## annotate
onegram_tdm <- TermDocumentMatrix(text_corpus)
freq <- sort(rowSums(as.matrix(removeSparseTerms(onegram_tdm, 0.99))), decreasing = TRUE)
onegram_dt = data.frame(word = names(freq), freq = freq)
dfsub <- subset(onegram_dt[1:20,])
ggplot(dfsub, aes(x = reorder(word, -freq), y = freq)) +
geom_bar(stat = "identity") +
labs(x = "Words", y = "Count", title = "Top 20 Words") +
theme(axis.text.x = element_text(angle = 45, hjust = 1), plot.title = element_text(hjust = 0.5))
TwoGramTokenizer <- function(x) NGramTokenizer(x, Weka_control(min = 2, max = 2))
twogram_dt = data.frame(word = names(freq), freq = freq)
dfsub <- subset(twogram_dt[1:20,])
ggplot(dfsub, aes(x = reorder(word, -freq), y = freq)) +
geom_bar(stat = "identity") +
labs(x = "Words", y = "Count", title = "Top 2-grams") +
theme(axis.text.x = element_text(angle = 45, hjust = 1), plot.title = element_text(hjust = 0.5))
ThreeGramTokenizer <- function(x) NGramTokenizer(x, Weka_control(min = 3, max = 3))
threegram_df = data.frame(word = names(freq), freq = freq)
dfsub <- subset(threegram_df[1:20,])
ggplot(dfsub, aes(x = reorder(word, -freq), y = freq)) +
geom_bar(stat = "identity") +
labs(x = "Words", y = "Count", title = "Top 3-grams") +
theme(axis.text.x = element_text(angle = 45, hjust = 1), plot.title = element_text(hjust = 0.5))