This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see http://rmarkdown.rstudio.com.
When you click the Knit button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this:
You can also embed plots, for example:
#I will share the work steps #firstly, loading the data , and summary statistics of data #When I uploaded data, it can be seen that there are 3 entries; blog , news , twitter, from total 4 different languages
#for this task , we only use English data and I assume that we will combine/match it with different languages ( german, finnish , russian)
#I already unzipped data and now read the data and make summary statistics
blogs <- readLines("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.blogs.txt", skipNul = TRUE, warn = FALSE)
news <- readLines("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.news.txt", skipNul = TRUE, warn = FALSE)
twitter <- readLines("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.twitter.txt", skipNul = TRUE, warn=FALSE)
#datas are so big so it is enough just to readLines function
#Now , data information
blog_info <- file.info("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.blogs.txt")$size/ 1024 ^ 2
new_info <- file.info("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.news.txt")$size/ 1024 ^ 2
twitter_info <- file.info("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.twitter.txt")$size/ 1024 ^ 2
eng_data_set_size <- c(blog_info,new_info,twitter_info)
data_frame_size<-data.frame(eng_data_set_size)
names(data_frame_size)[1] <-"MBs"
row.names(data_frame_size) <- c("Blog", "News", "Twitter")
data_frame_size
## MBs
## Blog 200.4242
## News 196.2775
## Twitter 159.3641
#As we can see here, data frame size for blog: 200 megabyte, for news: 196 megabyte, for twitter: 159 megabyte
#how many lines there are in English dataset, calculate
blog_entries_line_count<-length(blogs)
news_entries_line_count<-length(news)
twitter_feeds_line_count<-length(twitter)
data_set_length <- c(blog_entries_line_count,news_entries_line_count, twitter_feeds_line_count)
eng_data_frame_line_count <- data.frame(data_set_length)
names(eng_data_frame_line_count)[1] <-"Line Count"
row.names(eng_data_frame_line_count) <- c("Blog entries", "News entries", "Twitter Feeds")
eng_data_frame_line_count
## Line Count
## Blog entries 899288
## News entries 1010242
## Twitter Feeds 2360148
#As you can see in the table…
#How many words there are in this…!$!..dataset :))
library(ngram)
blog_entries_word_count <- wordcount(blogs)
news_entries_word_count <- wordcount(news)
twitter_feeds_word_count <- wordcount(twitter)
data_set_word_count <- c(blog_entries_word_count, news_entries_word_count, twitter_feeds_word_count)
eng_data_frame_word_count <- data.frame(data_set_word_count)
names(eng_data_frame_word_count)[1] <-"Word Count"
row.names(eng_data_frame_word_count) <- c("Blog entries", "News entries", "Twitter Feeds")
eng_data_frame_word_count
## Word Count
## Blog entries 37334131
## News entries 34372530
## Twitter Feeds 30373583
#As we can see in the table…
summary(blogs)
## Length Class Mode
## 899288 character character
#Length Class Mode
#899288 character character
head(blogs, 2)
## [1] "In the years thereafter, most of the Oil fields and platforms were named after pagan “gods”."
## [2] "We love you Mr. Brown."
summary(news)
## Length Class Mode
## 1010242 character character
head(news, 2)
## [1] "He wasn't home alone, apparently."
## [2] "The St. Louis plant had to close. It would die of old age. Workers had been making cars there since the onset of mass automotive production in the 1920s."
summary(twitter)
## Length Class Mode
## 2360148 character character
head(twitter, 2)
## [1] "How are you? Btw thanks for the RT. You gonna be in DC anytime soon? Love to see you. Been way, way too long."
## [2] "When you meet someone special... you'll know. Your heart will beat more rapidly and you'll smile for no reason."
#for exploratory part and making plots #I need to reduce the data by processing #I got help from web/github, how I need to reduce the data. In the file, lines will be cut to the 1/100
sample_size <- 0.01
blogs_index <- sample(seq_len(blog_entries_line_count),blog_entries_line_count*sample_size)
news_index <- sample(seq_len(length(news)),length(news)*sample_size)
twitter_index <- sample(seq_len(length(twitter)),length(twitter)*sample_size)
blogs_sub <- blogs[blogs_index[]]
news_sub <- news[news_index[]]
twitter_sub <- twitter
#I could not make a plot properly. Data is too big. I am not able to process it still.
library(ggplot2)
mostFrequent30 <-as.data.frame(blogs_sub[1:30])
mostFrequent30 <-data.frame(Words = row.names(mostFrequent30),mostFrequent30)
names(mostFrequent30)[2] = "Frequency"
row.names(mostFrequent30) <-NULL
mf30Plot = ggplot(data=mostFrequent30, aes(x=Words, y=Frequency, fill=Frequency)) + geom_bar(stat="identity") + guides(fill=FALSE) + theme(axis.text.x=element_text(angle=90))
## Warning: The `<scale>` argument of `guides()` cannot be `FALSE`. Use "none" instead as
## of ggplot2 3.3.4.
# mf30Plot +labs(title="30 Most Frequently Used Words")
mf30Plot + ggtitle("30 Most Frequently Used Words") + theme(plot.title = element_text(hjust = 0.5))
mostFrequent30 <-as.data.frame(news_sub[1:30])
mostFrequent30 <-data.frame(Words = row.names(mostFrequent30),mostFrequent30)
names(mostFrequent30)[2] = "Frequency"
row.names(mostFrequent30) <-NULL
mf30Plot = ggplot(data=mostFrequent30, aes(x=Words, y=Frequency, fill=Frequency)) + geom_bar(stat="identity") + guides(fill=FALSE) + theme(axis.text.x=element_text(angle=90)) #mf30Plot +labs(title="30 Most Frequently Used Words")
mf30Plot + ggtitle("30 Most Frequently Used Words") + theme(plot.title = element_text(hjust = 0.5))
mostFrequent30 <-as.data.frame(twitter_sub[1:30])
mostFrequent30 <-data.frame(Words = row.names(mostFrequent30),mostFrequent30)
names(mostFrequent30)[2] = "Frequency"
row.names(mostFrequent30) <-NULL
mf30Plot = ggplot(data=mostFrequent30, aes(x=Words, y=Frequency, fill=Frequency)) + geom_bar(stat="identity") + guides(fill=FALSE) + theme(axis.text.x=element_text(angle=90))
# mf30Plot +labs(title="30 Most Frequently Used Words")
mf30Plot + ggtitle("30 Most Frequently Used Words") + theme(plot.title = element_text(hjust = 0.5))
# Plan is next-word prediction for Shiny application -> Further
plan
Note that the echo = FALSE parameter was added to the
code chunk to prevent printing of the R code that generated the
plot.