Text Mining - by Rosikhu Ilmi

======================================================================================

TELKOMSEL

Extracting Tweets

Retrieve tweets from Twitter

# Retrieve tweets
tweets <- search_tweets("#Telkomsel", n = 8000, tweet_mode="extended")
tweets <- distinct(tweets, text, .keep_all=TRUE)

Tweets Description

tail(tweets, 20)

Text Cleaning

library(tm)

Build corpus

Frequent Words

Build Term Document Matrix

tdm1 <- TermDocumentMatrix(myCorpus, control = list(wordLengths = c(1, Inf)))

tdm1

Top Frequent Terms

freq.terms <- findFreqTerms(tdm1, lowfreq = 20)

freq.terms[1:50]

term.freq <- rowSums(as.matrix(tdm1))
term.freq <- subset(term.freq, term.freq >= 150)
df <- data.frame(term = names(term.freq), freq = term.freq)

ggplot(df, aes(x=term, y=freq)) + geom_bar(stat="identity") +
  xlab("Terms") + ylab("Count") + coord_flip() +
  theme(axis.text=element_text(size=7))

Wordcloud

Build Wordcloud

wordcloud(words = names(word.freq), freq = word.freq, min.freq = 100,
    random.order = F, colors = pal1)

INDOSAT

Extracting Tweets

Retrieve tweets from Twitter

# Retrieve tweets
tweets <- search_tweets("#Indosat", n = 8000, tweet_mode="extended")
tweets <- distinct(tweets, text, .keep_all=TRUE)

Tweets Description

ts_plot(tweets, "3 hours") +
  theme_minimal() +
  theme(plot.title = ggplot2::element_text(face = "bold")) +
  labs(
    x = NULL, y = NULL,
    title = "Frequency of indihome Twitter statuses from past 3 hours",
    subtitle = "Twitter status (tweet) counts aggregated using three-hour intervals",
    caption = "\nSource: Data collected from Twitter's REST API via rtweet"
  )

tail(tweets, 20)

Text Cleaning

library(tm)

Build corpus

# build a corpus, and specify the source to be character vectors 
myCorpus <- Corpus(VectorSource(tweets$text))
# convert to lower case
myCorpus <- tm_map(myCorpus, content_transformer(tolower))
# remove URLs
removeURL <- function(x) gsub("http[^[:space:]]*", "", x)
myCorpus <- tm_map(myCorpus, content_transformer(removeURL))
# remove anything other than English letters or space 
removeNumPunct <- function(x) gsub("[^[:alpha:][:space:]]*", "", x) 
myCorpus <- tm_map(myCorpus, content_transformer(removeNumPunct))
# remove stopwords
myStopwords <- c(setdiff(stopwords('english'), c("r", "big")), "use", "see", "used", "via", "amp", "indihome")
stopwords_id <- read.table("E://stopwords-id.txt", header = FALSE)
myStopwords <- c(myStopwords, as.matrix(stopwords_id$V1), "hi", "yg")
myCorpus <- tm_map(myCorpus, removeWords, myStopwords)
# remove extra whitespace
myCorpus <- tm_map(myCorpus, stripWhitespace)
# keep a copy for stem completion later
myCorpusCopy <- myCorpus

Frequent Words_2

Build Term Document Matrix_2

tdm <- TermDocumentMatrix(myCorpus, control = list(wordLengths = c(1, Inf)))

tdm

Top Frequent Terms_2

freq.terms <- findFreqTerms(tdm, lowfreq = 20)

freq.terms[1:50]

term.freq <- rowSums(as.matrix(tdm))
term.freq <- subset(term.freq, term.freq >= 150)
df <- data.frame(term = names(term.freq), freq = term.freq)

ggplot(df, aes(x=term, y=freq)) + geom_bar(stat="identity") +
  xlab("Terms") + ylab("Count") + coord_flip() +
  theme(axis.text=element_text(size=7))

Wordcloud_2

Build Wordcloud_2

library(wordcloud)

m <- as.matrix(tdm)
# calculate the frequency of words and sort it by frequency 
word.freq <- sort(rowSums(m), decreasing = T)
# colors
pal <- brewer.pal(9, "BuGn")[-(1:4)]

wordcloud(words = names(word.freq), freq = word.freq, min.freq = 100,
    random.order = F, colors = pal)