comments_work3.Rmd

library(reshape2)
library(ggplot2)
library(tm) 

Loading required package: NLP

Attaching package: 'NLP'

The following object is masked from 'package:ggplot2':

    annotate

library(topicmodels) 
library(RColorBrewer)
library(wordcloud) 

Reading the file that contains comments and converting it from wide to long format

c <- read.csv("/Users/analucic/Documents/comments_work3.csv", stringsAsFactors=FALSE)
d <- melt(c, variable.name = "key", value.names = "value", id.vars = c("id"), factorsAsStrings=F)
head(d)

Keeping the column that contains comments only and pre-processing comments:converting terms to lower case, removing punctuation, and whitespace

text <- d$value 
text<- text[!is.na(text)]
text <- gsub("'", '', text)
text <- gsub("’", '', text)
text <- gsub("“", '', text)
text <- gsub("”", '', text)
text <- gsub("’re", '', text)
text <- gsub("'", '', text)
m <- Corpus(VectorSource(text))  
m <- tm_map(m, tolower)

Warning in tm_map.SimpleCorpus(m, tolower): transformation drops documents

m <- tm_map(m, removePunctuation) 

Warning in tm_map.SimpleCorpus(m, removePunctuation): transformation drops
documents

m <- tm_map(m, stripWhitespace)

Warning in tm_map.SimpleCorpus(m, stripWhitespace): transformation drops
documents

Removing words that we don’t want in the data set, in addition to the default text mining library stopwords list

myStopWords <- c("thank", "thanks", "use", "see", "used", "via", "amp", "kylo", "ben", "solo", "rey", "leia", "luke", "skywalker", "han", "solo", "chewie", "dean", "winchester", "sam", "castiel", "love", "heart", "yay", "omg", "captain", "america", "steve", "rogers", "bucky", "barnes", "winter", "soldier", "sam", "wilson", "tony", "stark", "ironman", "falcon", "black", "widow", "natasha", "clint", "s", "m", "3", "cas", "cant", "like", "much", "really", "cas", "reylo", "balthazar", "im", "d")
m <- tm_map(m, removeWords, c(stopwords("english"), myStopWords)) 

Warning in tm_map.SimpleCorpus(m, removeWords, c(stopwords("english"),
myStopWords)): transformation drops documents

Creating a term document matrix: terms in rows and documents as columns

tdm <- TermDocumentMatrix(m, control = list(wordLengths = c(1, Inf)))
tdm = removeSparseTerms(tdm, 0.999)
tdm

<<TermDocumentMatrix (terms: 220, documents: 226320)>>
Non-/sparse entries: 118745/49671655
Sparsity           : 100%
Maximal term length: 12
Weighting          : term frequency (tf)

Finding associated terms with the terms of interest, level of correlation is specified as 0.3

findAssocs(tdm, c("twitter", "tumblr", "ao3", "reading", "writing", "learning", "looking", "art", "fanart", "dresses", "multimedia", "meta", "fanon", "canon", "discord", "pillowfort", "dreamwidth", "kofi", "imagine", "see", "recipe", "links", "website", "wookiepedia", "explain", "describe", "description", "visualize", "hear", "smell", "taste", "understand", "know"), corlimit=0.3)

$twitter
numeric(0)

$tumblr
numeric(0)

$ao3
numeric(0)

$reading
commenting      words       kind 
      0.60       0.31       0.30 

$writing
numeric(0)

$learning
numeric(0)

$looking
forward 
   0.67 

$art
numeric(0)

$fanart
numeric(0)

$dresses
numeric(0)

$multimedia
numeric(0)

$meta
numeric(0)

$fanon
numeric(0)

$canon
numeric(0)

$discord
numeric(0)

$pillowfort
numeric(0)

$dreamwidth
numeric(0)

$kofi
numeric(0)

$imagine
numeric(0)

$see
numeric(0)

$recipe
numeric(0)

$links
numeric(0)

$website
numeric(0)

$wookiepedia
numeric(0)

$explain
numeric(0)

$describe
numeric(0)

$description
numeric(0)

$visualize
numeric(0)

$hear
numeric(0)

$smell
numeric(0)

$taste
numeric(0)

$understand
numeric(0)

$know
numeric(0)

Establishing frequent terms. Taking a subset of those that appear more than and equal to 1,000 times (more comments available than for other two works)

(freq.terms <- findFreqTerms(tdm, lowfreq = 1000)) 

 [1] "one"     "also"    "oh"      "reading" "wait"    "always"  "chapter"
 [8] "can"     "dont"    "reys"    "even"    "know"    "next"    "will"   
[15] "just"    "well"    "think"   "going"   "now"     "time"    "good"   
[22] "way"     "read"    "story"   "update"  "get"     "please"  "fic"    

one

also

oh

reading

wait

always

chapter

can

dont

reys

even

know

next

will

just

well

think

going

now

time

good

way

read

story

update

get

please

fic

term.freq <- rowSums(as.matrix(tdm))  
term.freq <- subset(term.freq, term.freq >= 1000) 

Converting frequent terms into a data frame format for easier plotting

df <- data.frame(term = names(term.freq), freq = term.freq) 
ggplot(df, aes(x = reorder(term, -freq), y = freq)) + geom_bar(stat = "identity") +  theme(axis.text.x=element_text(angle=45, hjust=1))

Establishing word frequency. Specified minimum frequency for the word cloud plot is 1,000

wm <- as.matrix(tdm) 
word.freq <- sort(rowSums(wm), decreasing = T)
pal <- brewer.pal(9, "BuGn")[-(1:4)]
wordcloud(words = names(word.freq), freq = word.freq, min.freq = 1000, random.order = F, colors = pal) 

Transposing the term document matrix and converting it into a document term matrix (documents as rows and colums as terms)

dtm <- as.DocumentTermMatrix(tdm) 

summing frequencies for documents/individual comments and eliminating those that do not have any terms

#dtm = removeSparseTerms(dtm, 0.99)
#dtm
row_total = apply(dtm, 1, sum) 
dtm.new = dtm[row_total>0,] 
dtm.new

<<DocumentTermMatrix (documents: 33261, terms: 220)>>
Non-/sparse entries: 118745/7198675
Sparsity           : 98%
Maximal term length: 12
Weighting          : term frequency (tf)

Running topic modeling (Latent Dirichlet Allocation) on the document term matrix and requeting 8 topics. Specifying that each topic model should consist of 10 words.

lda <- LDA(dtm.new, k = 8) 
term <- terms(lda, 10) 
term

      Topic 1   Topic 2   Topic 3   Topic 4   Topic 5   Topic 6
 [1,] "will"    "just"    "chapter" "chapter" "will"    "one"  
 [2,] "just"    "fic"     "just"    "snoke"   "reading" "two"  
 [3,] "also"    "chapter" "also"    "reading" "chapter" "fic"  
 [4,] "oh"      "will"    "oh"      "will"    "next"    "way"  
 [5,] "going"   "back"    "think"   "going"   "one"     "just" 
 [6,] "next"    "writing" "youre"   "wait"    "need"    "will" 
 [7,] "writing" "also"    "wait"    "time"    "dont"    "read" 
 [8,] "things"  "now"     "well"    "well"    "oh"      "reys" 
 [9,] "reys"    "dont"    "story"   "youre"   "end"     "hope" 
[10,] "time"    "know"    "next"    "happy"   "get"     "still"
      Topic 7      Topic 8 
 [1,] "chapter"    "please"
 [2,] "reading"    "just"  
 [3,] "right"      "now"   
 [4,] "oh"         "story" 
 [5,] "think"      "time"  
 [6,] "well"       "hope"  
 [7,] "time"       "going" 
 [8,] "just"       "one"   
 [9,] "commenting" "update"
[10,] "soon"       "take"  

will

just

also

oh

going

next

writing

things

reys

time

just

fic

chapter

will

back

writing

also

now

dont

know

chapter

just

also

oh

think

youre

wait

well

story

next

chapter

snoke

reading

will

going

wait

time

well

youre

happy

will

reading

chapter

next

one

need

dont

oh

end

get

one

two

fic

way

just

will

read

reys

hope

still

chapter

reading

right

oh

think

well

time

just

commenting

soon

please

just

now

story

time

hope

going

one

update

take

50 most frequent words and their frequencies

findMostFreqTerms(dtm, n = 50, INDEX = rep(1, dtm$nrow))[[1]]

   chapter       just       will    reading       also         oh 
      3444       3101       2203       2026       1640       1590 
       one      going       wait        now      story        get 
      1564       1491       1438       1431       1426       1399 
      time       good       know     update        can       next 
      1361       1360       1321       1238       1231       1231 
      read      think       well       reys     please        fic 
      1221       1161       1136       1130       1089       1079 
       way       dont       even     always    amazing       hope 
      1057       1031       1018       1003        990        986 
     happy      loved      youre    writing      great       want 
       928        896        882        878        849        841 
      kind      snoke       feel      first        two       glad 
       832        829        815        806        804        782 
     words      still commenting       make      right       back 
       769        759        758        741        734        716 
     kylos  beautiful 
       700        695