R Markdown

This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see http://rmarkdown.rstudio.com.

When you click the Knit button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this:

Including Plots

You can also embed plots, for example:

#I will share the work steps #firstly, loading the data , and summary statistics of data #When I uploaded data, it can be seen that there are 3 entries; blog , news , twitter, from total 4 different languages

#for this task , we only use English data and I assume that we will combine/match it with different languages ( german, finnish , russian)

#I already unzipped data and now read the data and make summary statistics

blogs <- readLines("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.blogs.txt", skipNul = TRUE, warn = FALSE)
news <- readLines("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.news.txt", skipNul = TRUE, warn = FALSE)
twitter <- readLines("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.twitter.txt", skipNul = TRUE, warn=FALSE)

#datas are so big so it is enough just to readLines function

#Now , data information

blog_info <- file.info("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.blogs.txt")$size/ 1024 ^ 2
new_info <- file.info("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.news.txt")$size/ 1024 ^ 2
twitter_info <- file.info("/Users/asudeberber/Documents/CourseraMycourses/DataScienceSpecializationJohnHopkins/Final Capstone Project/final/en_US/en_US.twitter.txt")$size/ 1024 ^ 2

eng_data_set_size <- c(blog_info,new_info,twitter_info)
data_frame_size<-data.frame(eng_data_set_size)
names(data_frame_size)[1] <-"MBs"
row.names(data_frame_size) <- c("Blog", "News", "Twitter")
data_frame_size
##              MBs
## Blog    200.4242
## News    196.2775
## Twitter 159.3641

#As we can see here, data frame size for blog: 200 megabyte, for news: 196 megabyte, for twitter: 159 megabyte

#how many lines there are in English dataset, calculate

blog_entries_line_count<-length(blogs)
news_entries_line_count<-length(news)
twitter_feeds_line_count<-length(twitter)

data_set_length <- c(blog_entries_line_count,news_entries_line_count, twitter_feeds_line_count)
eng_data_frame_line_count <- data.frame(data_set_length)
names(eng_data_frame_line_count)[1] <-"Line Count"
row.names(eng_data_frame_line_count) <- c("Blog entries", "News entries", "Twitter Feeds")
eng_data_frame_line_count
##               Line Count
## Blog entries      899288
## News entries     1010242
## Twitter Feeds    2360148

#As you can see in the table…

#How many words there are in this…!$!..dataset :))

library(ngram)
blog_entries_word_count <- wordcount(blogs)
news_entries_word_count <- wordcount(news)
twitter_feeds_word_count <- wordcount(twitter)

data_set_word_count <- c(blog_entries_word_count, news_entries_word_count, twitter_feeds_word_count)
eng_data_frame_word_count <- data.frame(data_set_word_count)
names(eng_data_frame_word_count)[1] <-"Word Count"
row.names(eng_data_frame_word_count) <- c("Blog entries", "News entries", "Twitter Feeds")
eng_data_frame_word_count
##               Word Count
## Blog entries    37334131
## News entries    34372530
## Twitter Feeds   30373583

#As we can see in the table…

summary(blogs)
##    Length     Class      Mode 
##    899288 character character
#Length     Class      Mode 
   #899288 character character 
head(blogs, 2)
## [1] "In the years thereafter, most of the Oil fields and platforms were named after pagan “gods”."
## [2] "We love you Mr. Brown."
summary(news)
##    Length     Class      Mode 
##   1010242 character character
head(news, 2)
## [1] "He wasn't home alone, apparently."                                                                                                                        
## [2] "The St. Louis plant had to close. It would die of old age. Workers had been making cars there since the onset of mass automotive production in the 1920s."
summary(twitter)
##    Length     Class      Mode 
##   2360148 character character
head(twitter, 2)
## [1] "How are you? Btw thanks for the RT. You gonna be in DC anytime soon? Love to see you. Been way, way too long."  
## [2] "When you meet someone special... you'll know. Your heart will beat more rapidly and you'll smile for no reason."

#for exploratory part and making plots #I need to reduce the data by processing #I got help from web/github, how I need to reduce the data. In the file, lines will be cut to the 1/100

sample_size <- 0.01
blogs_index <- sample(seq_len(blog_entries_line_count),blog_entries_line_count*sample_size)
news_index <- sample(seq_len(length(news)),length(news)*sample_size)
twitter_index <- sample(seq_len(length(twitter)),length(twitter)*sample_size)

blogs_sub <- blogs[blogs_index[]]
news_sub <- news[news_index[]]
twitter_sub <- twitter

#I could not make a plot properly. Data is too big. I am not able to process it still.

library(ggplot2)
mostFrequent30 <-as.data.frame(blogs_sub[1:30])
mostFrequent30 <-data.frame(Words = row.names(mostFrequent30),mostFrequent30)
names(mostFrequent30)[2] = "Frequency"
row.names(mostFrequent30) <-NULL
mf30Plot = ggplot(data=mostFrequent30, aes(x=Words, y=Frequency, fill=Frequency)) + geom_bar(stat="identity") +  guides(fill=FALSE) + theme(axis.text.x=element_text(angle=90))
## Warning: The `<scale>` argument of `guides()` cannot be `FALSE`. Use "none" instead as
## of ggplot2 3.3.4.
# mf30Plot +labs(title="30 Most Frequently Used Words")
mf30Plot + ggtitle("30 Most Frequently Used Words") + theme(plot.title = element_text(hjust = 0.5))

mostFrequent30 <-as.data.frame(news_sub[1:30])
mostFrequent30 <-data.frame(Words = row.names(mostFrequent30),mostFrequent30)
names(mostFrequent30)[2] = "Frequency"
row.names(mostFrequent30) <-NULL
mf30Plot = ggplot(data=mostFrequent30, aes(x=Words, y=Frequency, fill=Frequency)) + geom_bar(stat="identity") +  guides(fill=FALSE) + theme(axis.text.x=element_text(angle=90)) #mf30Plot +labs(title="30 Most Frequently Used Words")
mf30Plot + ggtitle("30 Most Frequently Used Words") + theme(plot.title = element_text(hjust = 0.5))

mostFrequent30 <-as.data.frame(twitter_sub[1:30])
mostFrequent30 <-data.frame(Words = row.names(mostFrequent30),mostFrequent30)
names(mostFrequent30)[2] = "Frequency"
row.names(mostFrequent30) <-NULL
mf30Plot = ggplot(data=mostFrequent30, aes(x=Words, y=Frequency, fill=Frequency)) + geom_bar(stat="identity") +  guides(fill=FALSE) + theme(axis.text.x=element_text(angle=90))
# mf30Plot +labs(title="30 Most Frequently Used Words")
mf30Plot + ggtitle("30 Most Frequently Used Words") + theme(plot.title = element_text(hjust = 0.5))

# Plan is next-word prediction for Shiny application -> Further plan

Note that the echo = FALSE parameter was added to the code chunk to prevent printing of the R code that generated the plot.