comments_work2.Rmd

library(reshape2)
library(ggplot2)
library(tm) 

Loading required package: NLP

Attaching package: 'NLP'

The following object is masked from 'package:ggplot2':

    annotate

library(topicmodels) 
library(RColorBrewer)
library(wordcloud) 

Reading the file that contains scraped comments and converting it from wide to long format

c <- read.csv("/Users/analucic/Documents/comments_work2.csv", stringsAsFactors=FALSE)
d <- melt(c, variable.name = "key", value.names = "value", id.vars = c("id"), factorsAsStrings=F)
head(d)

Keeping the column that contains comments only and pre-processing comments:converting terms to lower case, removing punctuation, and whitespace

text <- d$value 
text<- text[!is.na(text)]
text <- gsub("'", '', text)
text <- gsub("’", '', text)
text <- gsub("“", '', text)
text <- gsub("”", '', text)
text <- gsub("’re", '', text)
text <- gsub("'", '', text)
m <- Corpus(VectorSource(text))  
m <- tm_map(m, tolower)

Warning in tm_map.SimpleCorpus(m, tolower): transformation drops documents

m <- tm_map(m, removePunctuation) 

Warning in tm_map.SimpleCorpus(m, removePunctuation): transformation drops
documents

m <- tm_map(m, stripWhitespace)

Warning in tm_map.SimpleCorpus(m, stripWhitespace): transformation drops
documents

Removing certain words that we don’t want in the data set, in addition to the default text mining library stopwords list

myStopWords <- c("thank", "thanks", "use", "see", "used", "via", "amp", "kylo", "ben", "solo", "rey", "leia", "luke", "skywalker", "han", "solo", "chewie", "dean", "winchester", "sam", "castiel", "love", "heart", "yay", "omg", "captain", "america", "steve", "rogers", "bucky", "barnes", "winter", "soldier", "sam", "wilson", "tony", "stark", "ironman", "falcon", "black", "widow", "natasha", "clint", "s", "m", "3", "cas", "cant", "like", "much", "really", "cas", "reylo", "balthazar", "im", "d")
m <- tm_map(m, removeWords, c(stopwords("english"), myStopWords)) 

Warning in tm_map.SimpleCorpus(m, removeWords, c(stopwords("english"),
myStopWords)): transformation drops documents

Creating a term document matrix: terms in rows and documents as columns

tdm <- TermDocumentMatrix(m, control = list(wordLengths = c(1, Inf)))
tdm

<<TermDocumentMatrix (terms: 3913, documents: 9128)>>
Non-/sparse entries: 16213/35701651
Sparsity           : 100%
Maximal term length: 32
Weighting          : term frequency (tf)

Finding associated terms with the terms of interest, level of correlation is specified as 0.3

findAssocs(tdm, c("twitter", "tumblr", "ao3", "reading", "writing", "learning", "looking", "art", "fanart", "dresses", "multimedia", "meta", "fanon", "canon", "discord", "pillowfort", "dreamwidth", "kofi", "imagine", "see", "recipe", "links", "website", "wookiepedia", "explain", "describe", "description", "visualize", "hear", "smell", "taste", "understand", "know"), corlimit=0.3)

$twitter
numeric(0)

$tumblr
  noise     pop pitched 
   0.50    0.50    0.35 

$ao3
dumb 
0.32 

$reading
numeric(0)

$writing
     alisa       mwah          ❤️ appreciate 
      0.44       0.43       0.35       0.35 

$learning
numeric(0)

$looking
   forward      alisa       mwah appreciate          ❤️ 
      0.83       0.64       0.63       0.54       0.52 

$art
mentioning    softest   mastered boyswonder   origamii 
      0.35       0.35       0.35       0.35       0.35 

$fanart
numeric(0)

$dresses
numeric(0)

$multimedia
numeric(0)

$meta
numeric(0)

$fanon
numeric(0)

$canon
clapping  besides  delight 
    0.71     0.50     0.32 

$discord
numeric(0)

$pillowfort
numeric(0)

$dreamwidth
numeric(0)

$kofi
numeric(0)

$imagine
      relief        drawn      sparkly particularly         pink 
        0.71         0.71         0.71         0.50         0.50 
     worried 
        0.35 

$see
numeric(0)

$recipe
chicken    soup 
   0.35    0.35 

$links
numeric(0)

$website
numeric(0)

$wookiepedia
numeric(0)

$explain
   caught     guess     check hopefully 
     0.45      0.35      0.33      0.32 

$describe
   fuzzies     filled incredible      words 
      0.50       0.35       0.32       0.31 

$description
exact 
 0.71 

$visualize
numeric(0)

$hear
numeric(0)

$smell
numeric(0)

$taste
numeric(0)

$understand
concern    hits 
   0.50    0.35 

$know
dont 
0.34 

Establishing frequent terms. Taking a subset of those that appear more than and equal to 50 times

(freq.terms <- findFreqTerms(tdm, lowfreq = 50)) 

 [1] "loved"     "now"       "oh"        "chapter"   "good"     
 [6] "read"      "man"       "sweet"     "just"      "well"     
[11] "aw"        "better"    "glad"      "youre"     "fic"      
[16] "best"      "one"       "happy"     "right"     "christmas"
[21] "hope"      "new"       "story"     "buckys"    "amazing"  
[26] "writing"   "way"       "forward"   "looking"   "will"     
[31] "little"    "make"      "get"       "life"      "reading"  
[36] "also"      "dont"      "even"      "enjoying"  "gonna"    
[41] "going"     "next"      "time"      "know"      "can"      
[46] "got"       "♥"         "think"     "made"      "ive"      
[51] "still"    

loved

now

oh

chapter

good

read

man

sweet

just

well

aw

better

glad

youre

fic

best

one

happy

right

christmas

hope

new

story

buckys

amazing

writing

way

forward

looking

will

little

make

get

life

reading

also

dont

even

enjoying

gonna

going

next

time

know

can

got

♥

think

made

ive

still

term.freq <- rowSums(as.matrix(tdm))  
term.freq <- subset(term.freq, term.freq >= 50) 

Converting frequent terms into a data frame format for easier plotting

df <- data.frame(term = names(term.freq), freq = term.freq) 
ggplot(df, aes(x = reorder(term, -freq), y = freq)) + geom_bar(stat = "identity") +  theme(axis.text.x=element_text(angle=45, hjust=1))

Establishing word frequency. Specified minimum frequency for the word cloud plot is 50

wm <- as.matrix(tdm) 
word.freq <- sort(rowSums(wm), decreasing = T)
pal <- brewer.pal(9, "BuGn")[-(1:4)]
wordcloud(words = names(word.freq), freq = word.freq, min.freq = 50, random.order = F, colors = pal) 

Transposing the term document matrix and converting it into a document term matrix (documents as rows and colums as terms)

dtm <- as.DocumentTermMatrix(tdm) 

summing frequencies for documents/individual comments and eliminating those that do not have any terms

#dtm = removeSparseTerms(dtm, 0.99)
#dtm
row_total = apply(dtm, 1, sum) 
dtm.new = dtm[row_total>0,] 
dtm.new

<<DocumentTermMatrix (documents: 3240, terms: 3913)>>
Non-/sparse entries: 16213/12661907
Sparsity           : 100%
Maximal term length: 32
Weighting          : term frequency (tf)

Running topic modeling (Latent Dirichlet Allocation) on the document term matrix and requeting 8 topics. Specifying that each topic model should consist of 10 words.

lda <- LDA(dtm.new, k = 8) 
term <- terms(lda, 10) 
term

      Topic 1  Topic 2    Topic 3  Topic 4   Topic 5     Topic 6     
 [1,] "buckys" "just"     "story"  "chapter" "oh"        "glad"      
 [2,] "make"   "loved"    "aw"     "know"    "happy"     "youre"     
 [3,] "ive"    "made"     "now"    "dont"    "christmas" "writing"   
 [4,] "yes"    "can"      "right"  "next"    "well"      "little"    
 [5,] "thing"  "hes"      "man"    "time"    "sweet"     "forward"   
 [6,] "back"   "say"      "going"  "even"    "hope"      "enjoying"  
 [7,] "go"     "two"      "♥"      "think"   "best"      "looking"   
 [8,] "god"    "chapters" "life"   "every"   "new"       "still"     
 [9,] "ever"   "getting"  "great"  "wait"    "hard"      "gonna"     
[10,] "kudos"  "makes"    "theyre" "stella"  "wonderful" "appreciate"
      Topic 7   Topic 8  
 [1,] "fic"     "reading"
 [2,] "read"    "one"    
 [3,] "will"    "good"   
 [4,] "better"  "get"    
 [5,] "also"    "enjoyed"
 [6,] "amazing" "got"    
 [7,] "perfect" "things" 
 [8,] "feel"    "boys"   
 [9,] "cute"    "write"  
[10,] "nice"    "advent" 

buckys

make

ive

yes

thing

back

go

god

ever

kudos

just

loved

made

can

hes

say

two

chapters

getting

makes

story

aw

now

right

man

going

♥

life

great

theyre

chapter

know

dont

next

time

even

think

every

wait

stella

oh

happy

christmas

well

sweet

hope

best

new

hard

wonderful

glad

youre

writing

little

forward

enjoying

looking

still

gonna

appreciate

fic

read

will

better

also

amazing

perfect

feel

cute

nice

reading

one

good

get

enjoyed

got

things

boys

write

advent

50 most frequent words and their frequencies

findMostFreqTerms(dtm, n = 50, INDEX = rep(1, dtm$nrow))[[1]]

  reading      glad      just       fic   chapter        oh     story 
      183       172       164       151       148       146       118 
    youre     happy      good        aw   writing       one christmas 
      110       110       106       102        98        94        93 
      now      know       get      well     loved      read      will 
       89        89        84        82        81        81        81 
     dont    little     right     sweet    better      hope      next 
       76        75        74        73        72        70        70 
   buckys       man     going      made      also      time       way 
       66        64        64        64        63        62        61 
     best     think       can      make      even       got   forward 
       59        59        56        55        54        54        53 
     life  enjoying       ive   looking     still       new   amazing 
       53        53        52        51        51        50        50 
    gonna 
       50