comments_work1.Rmd

library(reshape2)
library(ggplot2)
library(tm) 

Loading required package: NLP

Attaching package: 'NLP'

The following object is masked from 'package:ggplot2':

    annotate

library(topicmodels) 
library(RColorBrewer)
library(wordcloud) 

Reading the file that contains scraped comments from the web site and converting the file from wide to long format

c <- read.csv("/Users/analucic/Documents/comments_work1.csv", stringsAsFactors=FALSE)
d <- melt(c, variable.name = "key", value.names = "value", id.vars = c("id"), factorsAsStrings=F)
head(d)

Keeping the column that contains comments only and pre-processing comments:converting terms to lower case, removing punctuation, and whitespace

text <- d$value 
text<- text[!is.na(text)]
text <- gsub("’", '', text)
text <- gsub("'", '', text)
text <- gsub("“", '', text)
text <- gsub("”", '', text)
text <- gsub("’re", '', text)
text <- gsub("'", '', text)
m <- Corpus(VectorSource(text))  
m <- tm_map(m, tolower)

Warning in tm_map.SimpleCorpus(m, tolower): transformation drops documents

m <- tm_map(m, removePunctuation) 

Warning in tm_map.SimpleCorpus(m, removePunctuation): transformation drops
documents

m <- tm_map(m, stripWhitespace)

Warning in tm_map.SimpleCorpus(m, stripWhitespace): transformation drops
documents

Removing certain words that we don’t want in the data set, in addition to the default text mining library stopwords list

myStopWords <- c("thank", "thanks", "use", "see", "used", "via", "amp", "kylo", "ben", "solo", "rey", "leia", "luke", "skywalker", "han", "solo", "chewie", "dean", "winchester", "sam", "castiel", "love", "heart", "yay", "omg", "captain", "america", "steve", "rogers", "bucky", "barnes", "winter", "soldier", "sam", "wilson", "tony", "stark", "ironman", "falcon", "black", "widow", "natasha", "clint", "'", "s", "m", "3", "cas", "cant", "dont", "like", "much", "really", "cas", "reylo", "balthazar", "im", "d", "make", "makes", "get")
m <- tm_map(m, removeWords, c(stopwords("english"), myStopWords)) 

Warning in tm_map.SimpleCorpus(m, removeWords, c(stopwords("english"),
myStopWords)): transformation drops documents

Creating a term document matrix: terms in rows and documents as columns

tdm <- TermDocumentMatrix(m, control = list(wordLengths = c(1, Inf)))
tdm

<<TermDocumentMatrix (terms: 2108, documents: 1830)>>
Non-/sparse entries: 4701/3852939
Sparsity           : 100%
Maximal term length: 36
Weighting          : term frequency (tf)

Finding associated terms with the terms of interest, level of correlation is specified as 0.3

findAssocs(tdm, c("twitter", "tumblr", "ao3", "reading", "writing", "learning", "looking", "art", "fanart", "dresses", "multimedia", "meta", "fanon", "canon", "discord", "pillowfort", "dreamwidth", "kofi", "imagine", "see", "recipe", "links", "website", "wookiepedia", "explain", "describe", "description", "visualize", "hear", "smell", "taste", "understand", "know"), corlimit=0.3)

$twitter
numeric(0)

$tumblr
numeric(0)

$ao3
numeric(0)

$reading
numeric(0)

$writing
domesticity    exercise 
       0.41        0.41 

$learning
numeric(0)

$looking
              forward                  spun                 sugar 
                 0.53                  0.50                  0.50 
                glass               pattern                   saw 
                 0.41                  0.35                  0.35 
         streetdoccas                ending                 folks 
                 0.35                  0.35                  0.35 
                 woot             delighted shadowrunsupernatural 
                 0.35                  0.35                  0.35 
                 made 
                 0.32 

$art
   lucky   pimmys  amazing   ageing    amiss   amount   gotten indecent 
    0.43     0.43     0.38     0.30     0.30     0.30     0.30     0.30 
  mécène    spent  casting     map—    spell   flames trinkets 
    0.30     0.30     0.30     0.30     0.30     0.30     0.30 

$fanart
numeric(0)

$dresses
numeric(0)

$multimedia
numeric(0)

$meta
numeric(0)

$fanon
accepted   grumpy     meat   planet absolute  morning throwing   person 
    1.00     1.00     1.00     1.00     0.71     0.58     0.58     0.50 
   least 
    0.38 

$canon
    moose  squirrel recognize 
     0.41      0.41      0.33 

$discord
         ago conversation           ho        quite      stopped 
        1.00         1.00         1.00         0.71         0.71 
    watching          hot        broom      morning          boy 
        0.71         0.71         0.71         0.58         0.58 
         spn         damn       closet          ive         part 
        0.58         0.58         0.58         0.50         0.35 
        long         time 
        0.32         0.32 

$pillowfort
numeric(0)

$dreamwidth
numeric(0)

$kofi
numeric(0)

$imagine
     dealing      streets       within  environment interactions 
        1.00         1.00         1.00         0.71         0.71 
   brilliant       giving    perfectly           us     everyone 
        0.58         0.58         0.45         0.38         0.35 
      moving         cool 
        0.32         0.30 

$see
numeric(0)

$recipe
numeric(0)

$links
numeric(0)

$website
numeric(0)

$wookiepedia
numeric(0)

$explain
numeric(0)

$describe
numeric(0)

$description
warm 
0.33 

$visualize
numeric(0)

$hear
numeric(0)

$smell
numeric(0)

$taste
alcohol chasing mmmmmmm  spiced   woody    edge 
   1.00    1.00    1.00    1.00    1.00    0.58 

$understand
history 
   0.58 

$know
    itll tomorrow 
    0.31     0.31 

Establishing frequent terms. Taking a subset of terms that appear more than and equal to 15 times

(freq.terms <- findFreqTerms(tdm, lowfreq = 15)) 

 [1] "bit"       "world"     "glad"      "people"    "way"      
 [6] "just"      "didnt"     "know"      "still"     "said"     
[11] "chapter"   "well"      "youre"     "back"      "one"      
[16] "reading"   "amazing"   "made"      "poor"      "thats"    
[21] "story"     "done"      "things"    "good"      "even"     
[26] "now"       "little"    "oh"        "beautiful" "lot"      
[31] "think"     "great"     "man"      

bit

world

glad

people

way

just

didnt

know

still

said

chapter

well

youre

back

one

reading

amazing

made

poor

thats

story

done

things

good

even

now

little

oh

beautiful

lot

think

great

man

term.freq <- rowSums(as.matrix(tdm))  
term.freq <- subset(term.freq, term.freq >= 15) 

Converting frequent terms into a data frame format for easier plotting

df <- data.frame(term = names(term.freq), freq = term.freq) 
ggplot(df, aes(x = reorder(term, -freq), y = freq)) + geom_bar(stat = "identity") +  theme(axis.text.x=element_text(angle=45, hjust=1))

Establishing word frequency. Minimum term frequency for the word cloud plot is 20

wm <- as.matrix(tdm) 
word.freq <- sort(rowSums(wm), decreasing = T)
pal <- brewer.pal(9, "BuGn")[-(1:4)]
wordcloud(words = names(word.freq), freq = word.freq, min.freq = 20, random.order = F, colors = pal) 

Transposing the term document matrix and converting it into a document term matrix (documents as rows and colums as terms)

dtm <- as.DocumentTermMatrix(tdm) 

Summing frequencies for documents/individual comments and eliminating those that do not have any terms

row_total = apply(dtm, 1, sum) 
dtm.new = dtm[row_total>0,] 

Running topic modeling (Latent Dirichlet Allocation) on the document term matrix and requesting 8 topics. Specifying that each topic model should consist of 10 words.

lda <- LDA(dtm.new, k = 8) 
term <- terms(lda, 10) 
term

      Topic 1  Topic 2         Topic 3   Topic 4   Topic 5       Topic 6  
 [1,] "still"  "good"          "made"    "glad"    "know"        "said"   
 [2,] "lot"    "well"          "things"  "people"  "read"        "wanted" 
 [3,] "little" "beautiful"     "enough"  "worked"  "didnt"       "work"   
 [4,] "even"   "thats"         "think"   "bit"     "interesting" "kids"   
 [5,] "just"   "worldbuilding" "felt"    "many"    "world"       "bad"    
 [6,] "story"  "looked"        "looking" "write"   "deans"       "writing"
 [7,] "one"    "descriptions"  "scene"   "thing"   "voice"       "course" 
 [8,] "youre"  "chapter"       "need"    "nurse"   "brain"       "around" 
 [9,] "banter" "whole"         "line"    "creepy"  "djinn"       "eyes"   
[10,] "wasnt"  "just"          "go"      "tension" "yes"         "loved"  
      Topic 7   Topic 8  
 [1,] "oh"      "one"    
 [2,] "reading" "amazing"
 [3,] "now"     "art"    
 [4,] "great"   "back"   
 [5,] "got"     "imagery"
 [6,] "man"     "come"   
 [7,] "chapter" "fun"    
 [8,] "yeah"    "cool"   
 [9,] "okay"    "poor"   
[10,] "feels"   "take"   

still

lot

little

even

just

story

one

youre

banter

wasnt

good

well

beautiful

thats

worldbuilding

looked

descriptions

chapter

whole

just

made

things

enough

think

felt

looking

scene

need

line

go

glad

people

worked

bit

many

write

thing

nurse

creepy

tension

know

read

didnt

interesting

world

deans

voice

brain

djinn

yes

said

wanted

work

kids

bad

writing

course

around

eyes

loved

oh

reading

now

great

got

man

chapter

yeah

okay

feels

one

amazing

art

back

imagery

come

fun

cool

poor

take

50 most frequent words and their frequencies

findMostFreqTerms(dtm, n = 50, INDEX = rep(1, dtm$nrow))[[1]]

       just         one          oh        good        well        glad 
         38          38          38          33          30          29 
      world         lot      people     chapter       thats         way 
         27          23          22          22          22          20 
       know       still      things         bit      little        said 
         20          20          20          19          19          17 
    reading       story         man       youre        back        poor 
         17          17          17          16          16          16 
       done         now       didnt     amazing        made        even 
         16          16          15          15          15          15 
  beautiful       think       great       scene        need       going 
         15          15          15          14          14          14 
       read        work         can      worked description  everything 
         14          14          13          13          12          12 
       feel        line        best         got       voice         fun 
         12          12          12          12          12          12 
      whole       loved 
         11          11