情緒分析、TF-idf、尺度縮減字詞共現

library(data.table)
library(dplyr)
library(stringr)

Amazon=fread('./data/Amazon_reviews.csv')

# pre-processing:
reviews=Amazon$artContent

#reviews=reviews[-2]
clean=function(review){
  reviews <- as.list(t(review))
  reviews <-str_trim(reviews) 
  reviews <- toupper(reviews) 
  reviews <- gsub("'", "", reviews)  # remove apostrophes
  reviews <- gsub("[[:punct:]]", " ", reviews)  # replace punctuation with space
  reviews <- gsub("[[:cntrl:]]", " ", reviews)  # replace control characters with space
  reviews <- gsub("[[:digit:]]"," ", reviews) # remove number
  reviews <- gsub("^[[:space:]]+", "", reviews) # remove whitespace at beginning of documents
  reviews <- gsub("[[:space:]]+$", "", reviews) # remove whitespace at end of documents
  reviews <- tolower(reviews)  # force to lowercase
  reviews
}

Sentiment analysis

#install.packages('syuzhet')
library(syuzhet)
X=data.frame()

split(Amazon,Amazon$tool_name) %>% sapply(.,function(v){
      txt=clean(v$artContent)
      senti = get_nrc_sentiment(txt)
      f = data.frame(t(colSums(senti)))
      X <<- rbind(X,f)
      rownames(X)[nrow(X)] <<- v$tool_name
     
  })

Visualization

#devtools::install_github("jbkunst/highcharter")
library(highcharter)

Highcharts (www.highcharts.com) is a Highsoft software product which is
not free for commercial and Governmental use

library(dplyr)
pmap = function(dx,st2="") { # perceptual map
  hchart(princomp(dx),cor=T) %>%
    hc_chart(zoomType = "xy") %>%
    hc_add_theme(hc_theme_flat())  %>%
    hc_tooltip(headerFormat="",pointFormat="{series.name}") %>%
    hc_title(text="Emotions on Brands of Hammers") %>%
    hc_subtitle(text=paste0("93 brands ",st2))}
pmap(X[,1:8]/(rowSums(X[,1:8])+1))

#save(X,file='0929.Rdata')

尺度縮減字詞共現

建立字頻表

library(tm)
library(stringr)
dtm = Amazon$artContent %>% 
  iconv(to = "utf-8", sub="") %>% 
  str_trim(.) %>%
  toupper() %>%
  VectorSource %>% Corpus %>% 
  tm_map(content_transformer(tolower)) %>% 
  tm_map(removePunctuation) %>% 
  tm_map(stemDocument) %>% 
  DocumentTermMatrix %>% 
  removeSparseTerms(0.998)
dtm  # (documents: 14156, terms: 1030)
dtm_tmp=dtm

TF-IDF 過濾

library(slam)
tfidf = tapply(dtm$v/row_sums(dtm)[dtm$i], dtm$j, mean) *
  log2(nrow(dtm)/col_sums(dtm > 0))
summary(tfidf)

dtm=dtm_tmp
dtm = dtm[, tfidf > 0.24 ]
dtm = dtm[,order(-col_sums(dtm))]
dim(dtm)

使用tSNE做尺度縮減

#install.packages('Rtsne')
library(Rtsne)
n = 636
tsne = dtm[, 1:n] %>% as.data.frame.matrix %>% 
  scale %>% t %>% Rtsne(
    check_duplicates = FALSE, theta=0.0, max_iter=3200)

層級式分群

Y = tsne$Y              # tSNE coordinates
d = dist(Y)             # distance matrix
hc = hclust(d)          # hi-clustering
K = 100                # number of clusters 
g = cutree(hc,K)        # cut into K clusters
table(g) %>% as.vector %>% sort         # sizes of clusters

  [1]  2  2  2  2  2  2  2  2  2  3  3  3  3  3  3  4  4  4  4  4  4  4  4
 [24]  4  4  4  4  5  5  5  5  5  5  5  5  5  5  5  5  5  5  5  6  6  6  6
 [47]  6  6  6  6  6  6  6  6  6  6  6  7  7  7  7  7  7  7  7  7  7  7  8
 [70]  8  8  8  8  8  8  8  8  9  9  9  9  9  9  9  9  9 10 10 10 10 10 10
 [93] 11 11 11 12 12 12 13 15

文字雲

# install.packages('randomcoloR')
library(randomcoloR)
library(wordcloud)

wc = col_sums(dtm[,1:n])
colors = distinctColorPalette(K)


png("./Amazon2.png", width=3200, height=1800)
textplot(
  Y[,1], Y[,2], colnames(dtm)[1:n], show=F, 
  col=colors[g],
  cex= 0.3 + 1.25 * sqrt(wc/mean(wc)),
  font=2)
dev.off()

文字雲