Twitter Analysis

Extracting Tweets

Retrieve tweets from Twitter

# Load packages
library(rtweet)

# Twitter authentication
create_token(
  app             = "my_twitter_research_app",
  consumer_key    = consumer_key,
  consumer_secret = consumer_secret,
  access_token    = access_token,
  access_secret   = access_secret)

<Token>
<oauth_endpoint>
 request:   https://api.twitter.com/oauth/request_token
 authorize: https://api.twitter.com/oauth/authenticate
 access:    https://api.twitter.com/oauth/access_token
<oauth_app> my_twitter_research_app
  key:    koZFJbYVk7gagYEGLaUN6RmTb
  secret: <hidden>
<credentials> oauth_token, oauth_token_secret
---

# Retrieve tweets
tweets <- search_tweets("#MachineLearning", n = 10000, langs="en", tweet_mode="extended")

Searching for tweets...
Finished collecting tweets!

Tweets Description

## plot time series of tweets
ts_plot(tweets, "3 hours") +
  ggplot2::theme_minimal() +
  ggplot2::theme(plot.title = ggplot2::element_text(face = "bold")) +
  ggplot2::labs(
    x = NULL, y = NULL,
    title = "Frequency of #MachineLearning Twitter statuses from past 9 days",
    subtitle = "Twitter status (tweet) counts aggregated using three-hour intervals",
    caption = "\nSource: Data collected from Twitter's REST API via rtweet"
  )

head(tweets)

Text Cleaning

library(tm)

Build corpus

# build a corpus, and specify the source to be character vectors 
myCorpus <- Corpus(VectorSource(tweets$text))
# convert to lower case
myCorpus <- tm_map(myCorpus, content_transformer(tolower))

transformation drops documents

# remove URLs
removeURL <- function(x) gsub("http[^[:space:]]*", "", x)
myCorpus <- tm_map(myCorpus, content_transformer(removeURL))

transformation drops documents

# remove anything other than English letters or space 
removeNumPunct <- function(x) gsub("[^[:alpha:][:space:]]*", "", x) 
myCorpus <- tm_map(myCorpus, content_transformer(removeNumPunct))

transformation drops documents

# remove stopwords
myStopwords <- c(setdiff(stopwords('english'), c("r", "big")), "use", "see", "used", "via", "amp")
myCorpus <- tm_map(myCorpus, removeWords, myStopwords)

transformation drops documents

# remove extra whitespace
myCorpus <- tm_map(myCorpus, stripWhitespace)

transformation drops documents

# keep a copy for stem completion later
myCorpusCopy <- myCorpus

Frequent Words

Build Term Document Matrix

tdm <- TermDocumentMatrix(myCorpus, control = list(wordLengths = c(1, Inf)))

tdm

<<TermDocumentMatrix (terms: 8532, documents: 9737)>>
Non-/sparse entries: 175088/82900996
Sparsity           : 100%
Maximal term length: 36
Weighting          : term frequency (tf)

Top Frequent Terms

freq.terms <- findFreqTerms(tdm, lowfreq = 20)

freq.terms[1:50]

 [1] "ai"                     "algorithms"             "bigdata"                "datascience"            "deeplearning"          
 [6] "iot"                    "learning"               "machine"                "machinelearning"        "ml"                    
[11] "nlp"                    "read"                   "robots"                 "artificial"             "artificialintelligence"
[16] "causes"                 "conflict"               "decodes"                "intelligence"           "religious"             
[21] "system"                 "data"                   "global"                 "re"                     "reduce"                
[26] "risk"                   "boost"                  "million"                "science"                "approach"              
[31] "fear"                   "holds"                  "hope"                   "instead"                "promise"               
[36] "vast"                   "analytics"              "clinicalanalytics"      "decrease"               "healthtech"            
[41] "hospital"               "improve"                "increase"               "phuse"                  "physical"              
[46] "physicaltherapy"        "predictiveanalytics"    "readmissions"           "referrals"              "therapists"            

term.freq <- rowSums(as.matrix(tdm))
term.freq <- subset(term.freq, term.freq >= 1000)
df <- data.frame(term = names(term.freq), freq = term.freq)

ggplot2::ggplot(df, aes(x=term, y=freq)) + geom_bar(stat="identity") +
  xlab("Terms") + ylab("Count") + coord_flip() +
  theme(axis.text=element_text(size=7))

Wordcloud

Build Wordcloud

library(wordcloud)

m <- as.matrix(tdm)
# calculate the frequency of words and sort it by frequency 
word.freq <- sort(rowSums(m), decreasing = T)
# colors
pal <- brewer.pal(9, "BuGn")[-(1:4)]

wordcloud(words = names(word.freq), freq = word.freq, min.freq = 300,
    random.order = F, colors = pal)