########################## The Capstone Project ################################
###############################################################################
library(tm)## Loading required package: NLPlibrary(lda)
library(qdap)## Loading required package: ggplot2
##
## Attaching package: 'ggplot2'
##
## The following object is masked from 'package:NLP':
##
## annotate
##
## Loading required package: qdapDictionaries
## Loading required package: qdapRegex
## Loading required package: qdapTools
## Loading required package: RColorBrewer
##
## Attaching package: 'qdap'
##
## The following objects are masked from 'package:tm':
##
## as.DocumentTermMatrix, as.TermDocumentMatrix
##
## The following object is masked from 'package:base':
##
## Filterlibrary(RWeka)
library(catdata)## Loading required package: MASSlibrary(stringr)
library(stringi)
library(SnowballC)
library(tm.plugin.dc)## Loading required package: DSLsource('score.R')
options(warn=-1)
#### READ THE DATA ##################
######################################
twr <- readLines("final/en_US/en_US.twitter.txt")
file.info("final/en_US/en_US.twitter.txt")## size isdir mode mtime
## final/en_US/en_US.twitter.txt 167105338 FALSE 644 2014-07-22 11:12:58
## ctime atime uid
## final/en_US/en_US.twitter.txt 2014-11-09 12:02:44 2014-11-16 16:13:06 501
## gid uname grname
## final/en_US/en_US.twitter.txt 20 ytouhami staffhead(twr)## [1] "How are you? Btw thanks for the RT. You gonna be in DC anytime soon? Love to see you. Been way, way too long."
## [2] "When you meet someone special... you'll know. Your heart will beat more rapidly and you'll smile for no reason."
## [3] "they've decided its more fun if I don't."
## [4] "So Tired D; Played Lazer Tag & Ran A LOT D; Ughh Going To Sleep Like In 5 Minutes ;)"
## [5] "Words from a complete stranger! Made my birthday even better :)"
## [6] "First Cubs game ever! Wrigley field is gorgeous. This is perfect. Go Cubs Go!"length(twr)## [1] 2360148news <- readLines("final/en_US/en_US.news.txt")
file.info("final/en_US/en_US.news.txt")## size isdir mode mtime
## final/en_US/en_US.news.txt 205811889 FALSE 644 2014-07-22 11:13:04
## ctime atime uid gid
## final/en_US/en_US.news.txt 2014-11-09 12:01:02 2014-11-16 16:13:35 501 20
## uname grname
## final/en_US/en_US.news.txt ytouhami stafflength(news)## [1] 1010242blogs <- readLines("final/en_US/en_US.blogs.txt")
file.info("final/en_US/en_US.blogs.txt")## size isdir mode mtime
## final/en_US/en_US.blogs.txt 210160014 FALSE 644 2014-07-22 11:13:06
## ctime atime uid
## final/en_US/en_US.blogs.txt 2014-11-09 12:01:02 2014-11-16 16:14:08 501
## gid uname grname
## final/en_US/en_US.blogs.txt 20 ytouhami stafflength(blogs)## [1] 899288#### Summary of Basic Data Tables and Counts ############
########################################################
# inspecting twitter file
twr[1:3]## [1] "How are you? Btw thanks for the RT. You gonna be in DC anytime soon? Love to see you. Been way, way too long."
## [2] "When you meet someone special... you'll know. Your heart will beat more rapidly and you'll smile for no reason."
## [3] "they've decided its more fun if I don't."stri_stats_latex(twr)## CharsWord CharsCmdEnvir CharsWhite Words Cmds
## 125570616 3032 35958481 30451128 963
## Envirs
## 0# inspecting news file
news[1:3]## [1] "He wasn't home alone, apparently."
## [2] "The St. Louis plant had to close. It would die of old age. Workers had been making cars there since the onset of mass automotive production in the 1920s."
## [3] "WSU's plans quickly became a hot topic on local online sites. Though most people applauded plans for the new biomedical center, many deplored the potential loss of the building."stri_stats_latex(news)## CharsWord CharsCmdEnvir CharsWhite Words Cmds
## 162227130 2 40263955 34494539 1
## Envirs
## 0# inspecting blogs file
blogs[1:3]## [1] "In the years thereafter, most of the Oil fields and platforms were named after pagan “gods”."
## [2] "We love you Mr. Brown."
## [3] "Chad has been awesome with the kids and holding down the fort while I work later than usual! The kids have been busy together playing Skylander on the XBox together, after Kyan cashed in his $$$ from his piggy bank. He wanted that game so bad and used his gift card from his birthday he has been saving and the money to get it (he never taps into that thing either, that is how we know he wanted it so bad). We made him count all of his money to make sure that he had enough! It was very cute to watch his reaction when he realized he did! He also does a very good job of letting Lola feel like she is playing too, by letting her switch out the characters! She loves it almost as much as him."stri_stats_latex(blogs)## CharsWord CharsCmdEnvir CharsWhite Words Cmds
## 162464653 9 42636700 37570839 3
## Envirs
## 0#### Some answers to Questions from Quizzes ###############
############################################################
# Ratio of love-to-hate words in the twiter file
a = scan(file='final/en_US/en_US.twitter.txt',what="list")
a1 = data.frame(lst=a)
length(which(a1 == "love"))/ length(which(a1 == "hate"))## [1] 4.793427# Locate the tweet with the word "biostat"
n <- grep(pattern ="biostat", twr)
twr[n]## [1] "i know how you feel.. i have biostats on tuesday and i have yet to study =/"############# list of profanites to remove #############
profs <- readLines("prof_google.txt")
length(profs)## [1] 458#profs = data.frame(lst=profs)
#clean_twr <- tm_map(twr, removeWords, profs)
stri_stats_latex(profs)## CharsWord CharsCmdEnvir CharsWhite Words Cmds
## 3127 617 125 493 28
## Envirs
## 0# Histograms and explanatory plot ##################
###################################################
require( ggplot2 )
qplot(twr[1:50])
pos.words <- c('love', 'like', 'nice', 'positive')
neg.words <- c( 'hate', 'dislike', 'mean', 'negative')
file.sco = score(twr[1:50000], pos.words, neg.words, .progress='text')## Loading required package: plyr
##
## Attaching package: 'plyr'
##
## The following object is masked from 'package:qdapTools':
##
## id##
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|== | 2%
|
|== | 3%
|
|== | 4%
|
|=== | 4%
|
|=== | 5%
|
|==== | 5%
|
|==== | 6%
|
|==== | 7%
|
|===== | 7%
|
|===== | 8%
|
|====== | 8%
|
|====== | 9%
|
|====== | 10%
|
|======= | 10%
|
|======= | 11%
|
|======= | 12%
|
|======== | 12%
|
|======== | 13%
|
|========= | 13%
|
|========= | 14%
|
|========= | 15%
|
|========== | 15%
|
|========== | 16%
|
|=========== | 16%
|
|=========== | 17%
|
|=========== | 18%
|
|============ | 18%
|
|============ | 19%
|
|============= | 19%
|
|============= | 20%
|
|============= | 21%
|
|============== | 21%
|
|============== | 22%
|
|=============== | 22%
|
|=============== | 23%
|
|=============== | 24%
|
|================ | 24%
|
|================ | 25%
|
|================= | 25%
|
|================= | 26%
|
|================= | 27%
|
|================== | 27%
|
|================== | 28%
|
|=================== | 28%
|
|=================== | 29%
|
|=================== | 30%
|
|==================== | 30%
|
|==================== | 31%
|
|==================== | 32%
|
|===================== | 32%
|
|===================== | 33%
|
|====================== | 33%
|
|====================== | 34%
|
|====================== | 35%
|
|======================= | 35%
|
|======================= | 36%
|
|======================== | 36%
|
|======================== | 37%
|
|======================== | 38%
|
|========================= | 38%
|
|========================= | 39%
|
|========================== | 39%
|
|========================== | 40%
|
|========================== | 41%
|
|=========================== | 41%
|
|=========================== | 42%
|
|============================ | 42%
|
|============================ | 43%
|
|============================ | 44%
|
|============================= | 44%
|
|============================= | 45%
|
|============================== | 45%
|
|============================== | 46%
|
|============================== | 47%
|
|=============================== | 47%
|
|=============================== | 48%
|
|================================ | 48%
|
|================================ | 49%
|
|================================ | 50%
|
|================================= | 50%
|
|================================= | 51%
|
|================================= | 52%
|
|================================== | 52%
|
|================================== | 53%
|
|=================================== | 53%
|
|=================================== | 54%
|
|=================================== | 55%
|
|==================================== | 55%
|
|==================================== | 56%
|
|===================================== | 56%
|
|===================================== | 57%
|
|===================================== | 58%
|
|====================================== | 58%
|
|====================================== | 59%
|
|======================================= | 59%
|
|======================================= | 60%
|
|======================================= | 61%
|
|======================================== | 61%
|
|======================================== | 62%
|
|========================================= | 62%
|
|========================================= | 63%
|
|========================================= | 64%
|
|========================================== | 64%
|
|========================================== | 65%
|
|=========================================== | 65%
|
|=========================================== | 66%
|
|=========================================== | 67%
|
|============================================ | 67%
|
|============================================ | 68%
|
|============================================= | 68%
|
|============================================= | 69%
|
|============================================= | 70%
|
|============================================== | 70%
|
|============================================== | 71%
|
|============================================== | 72%
|
|=============================================== | 72%
|
|=============================================== | 73%
|
|================================================ | 73%
|
|================================================ | 74%
|
|================================================ | 75%
|
|================================================= | 75%
|
|================================================= | 76%
|
|================================================== | 76%
|
|================================================== | 77%
|
|================================================== | 78%
|
|=================================================== | 78%
|
|=================================================== | 79%
|
|==================================================== | 79%
|
|==================================================== | 80%
|
|==================================================== | 81%
|
|===================================================== | 81%
|
|===================================================== | 82%
|
|====================================================== | 82%
|
|====================================================== | 83%
|
|====================================================== | 84%
|
|======================================================= | 84%
|
|======================================================= | 85%
|
|======================================================== | 85%
|
|======================================================== | 86%
|
|======================================================== | 87%
|
|========================================================= | 87%
|
|========================================================= | 88%
|
|========================================================== | 88%
|
|========================================================== | 89%
|
|========================================================== | 90%
|
|=========================================================== | 90%
|
|=========================================================== | 91%
|
|=========================================================== | 92%
|
|============================================================ | 92%
|
|============================================================ | 93%
|
|============================================================= | 93%
|
|============================================================= | 94%
|
|============================================================= | 95%
|
|============================================================== | 95%
|
|============================================================== | 96%
|
|=============================================================== | 96%
|
|=============================================================== | 97%
|
|=============================================================== | 98%
|
|================================================================ | 98%
|
|================================================================ | 99%
|
|=================================================================| 99%
|
|=================================================================| 100%hist(file.sco$score)
q <- qplot(file.sco$score)
q <- q + theme_bw()
q## stat_bin: binwidth defaulted to range/30. Use 'binwidth = x' to adjust this.