Exploratory Data Analysis. This script will perform some analysis of corpora resulting from unsupervised clustering. The results will be presented as graphs and tables. All of the code used to conduct the analysis are included in this script.
knitr::opts_chunk$set(echo = TRUE, cache= TRUE)
require(quanteda)
require(readtext)
base <- "/home/larsbun/som-mpi-py/experiments/20170206/31b624b4-8073-4f83-a1e2-ed3e26c18989/ward-partitioning-0/"
mystopwords <-c("<", ">", "/", "s", ":", ";")
plotdfm<-function(mydfm,order){
freq <- slam::col_sums(mydfm)
freq <- as.data.frame(freq)
freq$names <- rownames(freq)
rownames(freq) <- NULL
colnames(freq) <- c("freq","word")
freq <- freq[order(freq$freq,decreasing = TRUE),]
freq_top25 <- head(freq, n = 25)
ggplot(freq_top25, aes(x=reorder(word, freq), y=freq)) +
geom_bar(stat = "identity") +
coord_flip() +
xlab(paste(order, "-gram words")) +
ylab("Frequency") +
geom_text(aes(label=freq), hjust=-0.20) +
ggtitle(paste("Most Common", order, "-grams"))
}
createwordcloud<-function(mydfm){
textplot_wordcloud(mydfm, min.freq = 30, random.order = FALSE,
rot.per = .25, max.words=100,
colors = RColorBrewer::brewer.pal(8,"Dark2"))
}
library(ggplot2)
createtopmod<-function(mydfm){
if (require(topicmodels)) {
myLDAfit20 <- LDA(convert(mydfm, to = "topicmodels"), k = 20)
# get_terms(myLDAfit20, 5)
# topics(myLDAfit20, 3)
}
}
Iterate over the corpora in the partitioning given by base, save as a list of lists of corpus objects.
library(quanteda)
dfmarray<-list()
for (i in 1:3){
dfmlist<-list()
for (j in 1:3){
dfmlist<- c(dfmlist, dfm(clist[[i]], ngrams=j, removepunct=TRUE, remove = mystopwords))
}
dfmarray<-c(dfmarray, list(dfmlist))
}
## Warning in tokens.character(x, ...): Argument removepunct not used.
## Warning in tokens.character(x, ...): Argument removepunct not used.
## Warning in tokens.character(x, ...): Argument removepunct not used.
## Warning in tokens.character(x, ...): Argument removepunct not used.
## Warning in tokens.character(x, ...): Argument removepunct not used.
## Warning in tokens.character(x, ...): Argument removepunct not used.
## Warning in tokens.character(x, ...): Argument removepunct not used.
## Warning in tokens.character(x, ...): Argument removepunct not used.
## Warning in tokens.character(x, ...): Argument removepunct not used.
For each cluster, the top 25 n-grams up to order 3 will be displayed. Next, the 3-grams will be presente as wordclouds. Finally, the first
require(ggplot2)
## Loading required package: ggplot2
plotdfm(dfmarray[[1]][[1]],1)
## Loading required package: quanteda
## quanteda version 0.9.9.23
## Using 31 of 32 cores for parallel computing
##
## Attaching package: 'quanteda'
## The following object is masked from 'package:utils':
##
## View
## The following object is masked from 'package:base':
##
## sample
plotdfm(dfmarray[[1]][[2]],2)
plotdfm(dfmarray[[1]][[3]],3)
createwordcloud(dfmarray[[1]][[3]])
topmod1 <- createtopmod(dfmarray[[1]][[3]])
## Loading required package: topicmodels
t(as.data.frame(terms(topmod1,5)))
## [,1] [,2] [,3] [,4]
## Topic 1 "።_ነገር_ግን" "ነው_:_:" "።_‹_‹" "ግብረ_-_ሰዶማውያን"
## Topic 2 "!_!_!" "።_ነገር_ግን" "ነው_:_:" "ማለት_ነው_።"
## Topic 3 "፲_፱_፻" "ዓ_/_ም" "።_፲_፱" "/_ም_-"
## Topic 4 "._._." "ወ_/_ሮ" "ነው_:_:" "ያልታወቀ_፤_ወንድ"
## Topic 5 "።_ነገር_ግን" "ዶ_/_ር" "._._." "ንዑስ_አንቀጽ_("
## Topic 6 "-_-_-" "ደጋግሞ_ደጋግሞ_ደጋግሞ" "*_*_*" "!_!_!"
## Topic 7 "http_:_/" ":_/_/" "._._." "]_http_:"
## Topic 8 "*_*_*" "።_‹_‹" "።_ነገር_ግን" "ማለት_ነው_።"
## Topic 9 "._._." "ዓ_/_ም" "read_more_»" "._read_more"
## Topic 10 "._._." "-_-_-" "ዶ_/_ር" "/_ት_ብርቱካን"
## Topic 11 "._._." "ነበር_።_[" "ዓ_/_ም" "።_ነገር_ግን"
## Topic 12 "*_*_*" "።_ነገር_ግን" "ማለት_ነው_።" "._._."
## Topic 13 "ማለት_ነው_።" "።_ነገር_ግን" "ዶ_/_ር" "._._."
## Topic 14 "=_=_=" "._._." "ማለት_ነው_።" "።_‹_‹"
## Topic 15 "?_?_?" "…_…_…" "!_!_!" "._._."
## Topic 16 "._._." "-_-_-" "!_!_!" "።_ነገር_ግን"
## Topic 17 "ዶ_/_ር" "ነው_:_:" "።_‹_‹" "።_ነገር_ግን"
## Topic 18 "._._." "ዶ_/_ር" "።_ነገር_ግን" "ነው_።_ይህ"
## Topic 19 "._._." "-_-_-" "ማለት_ነው_።" "።_‹_‹"
## Topic 20 "።_‹_‹" "?_?_?" "._._." "!_!_!"
## [,5]
## Topic 1 "ማለት_ነው_።"
## Topic 2 "._._."
## Topic 3 "፱_፻_፷"
## Topic 4 "፤_ወንድ_፤"
## Topic 5 "አንቀጽ_ንዑስ_አንቀጽ"
## Topic 6 "._._."
## Topic 7 "ፍ_/_ቤት"
## Topic 8 "ጠ_/_ሚኒስትር"
## Topic 9 "._._read"
## Topic 10 "ማለት_ነው_።"
## Topic 11 "=_=_="
## Topic 12 "ነው_።_ይህ"
## Topic 13 "-_-_-"
## Topic 14 "-_-_-"
## Topic 15 "ነው_:_:"
## Topic 16 "?_\"_\""
## Topic 17 ":_:_\""
## Topic 18 "share_this_:"
## Topic 19 "[_._."
## Topic 20 "!_›_›"
plotdfm(dfmarray[[2]][[1]],1)
plotdfm(dfmarray[[2]][[2]],2)
plotdfm(dfmarray[[2]][[3]],3)
createwordcloud(dfmarray[[2]][[3]])
topmod2 <- createtopmod(dfmarray[[3]][[3]])
t(as.data.frame(terms(topmod2,5)))
## [,1] [,2] [,3] [,4]
## Topic 1 "=_=_=" "._._." "ብፁዕ_ወቅዱስ_አቡነ" "ወቅዱስ_አቡነ_ጳውሎስ"
## Topic 2 "._._." "read_more_»" "ዜና_:_-" "ኢሳት_ዜና_:"
## Topic 3 "ማለት_ነው_።" "?_ኦባንግ_፦" "'_ወያነ_'" "።_ጎልጉል_፦"
## Topic 4 "ዓ_/_ም" "-_-_-" "ኢሳት_ዜና_:" "ዜና_:_-"
## Topic 5 "._._." "!_!_!" "ቀን_፳_፻" "፳_፻_፭"
## Topic 6 "%_e1_%" "ወ_/_ሮ" "/_ሮ_አዜብ" "ዶ_/_ር"
## Topic 7 "!_!_!" "ማለት_ነው_።" "ሚ_/_ር" "ብቻ_ነው_።"
## Topic 8 "።_ነገር_ግን" "ቀን_2005_ዓ.ም" "2005_ዓ.ም_." "ማለት_ነው_።"
## Topic 9 "ንዑስ_አንቀጽ_/" "._._." "377_/_96" "._._read"
## Topic 10 "።_አዲስ_ዘመን" "።_ነገር_ግን" "አዲስ_ዘመን_፦" "._._."
## Topic 11 "._._." "!_!_!" "።_‹_‹" "ማለት_ነው_።"
## Topic 12 "._._." "^_^_^" "ጠ_/_ሚ" "/_ሚ_መለስ"
## Topic 13 "መለስ_/_ወያኔ" "ፀረ_-_ሰላም" "፲_፱_፻" "።_ነገር_ግን"
## Topic 14 "ት_/_ት" "rate_this_:" "ቀን_2005_ዓ.ም" "2005_ዓ.ም_."
## Topic 15 "._._." "።_ነገር_ግን" "ማለት_ነው_።" "._._\""
## Topic 16 "ላይፍ_፡_-" "።_ላይፍ_፡" "?_ብርሃኔ_፡" "ብርሃኔ_፡_-"
## Topic 17 "።_ነገር_ግን" "ማለት_ነው_።" "የኢትዮጵያ_ሰላም_አስከባሪ" "ሰላም_አስከባሪ_ሰራዊት"
## Topic 18 "?_?_?" "._._." "።_ነገር_ግን" "(_(_("
## Topic 19 "._._." "‹_ቢ_›" "ሊብሮ_፡_-" "?_አንዳርጋቸው_፡"
## Topic 20 "ዶ_/_ር" "ኢሳት_ዜና_:" "ዜና_:_-" "ነው_:_:"
## [,5]
## Topic 1 "ኦርቶዶክስ_ተዋሕዶ_ቤተ"
## Topic 2 "._._read"
## Topic 3 "ነው_።_\""
## Topic 4 "read_more_»"
## Topic 5 "ቀን_2003_ዓ.ም"
## Topic 6 "e1_%_88"
## Topic 7 "ነው_።_ይህ"
## Topic 8 "ሪፖርተር_፡_-"
## Topic 9 "._read_more"
## Topic 10 "ኢንጅነር_ይልቃል_፦"
## Topic 11 "ነው_:_:"
## Topic 12 "ሚ_መለስ_፦"
## Topic 13 "ነው_።_ስለዚህ"
## Topic 14 "/_ቤ_/"
## Topic 15 "ማዕድናት_ምርትና_ግብይት"
## Topic 16 "ኃ_/_ማሪያም"
## Topic 17 "ዶ_/_ር"
## Topic 18 "ኦርቶዶክስ_ተዋሕዶ_ቤተ"
## Topic 19 "አንዳርጋቸው_፡_-"
## Topic 20 "ኦርቶዶክስ_ተዋሕዶ_ቤተ"
plotdfm(dfmarray[[3]][[3]],1)
plotdfm(dfmarray[[3]][[3]],2)
plotdfm(dfmarray[[3]][[3]],3)
createwordcloud(dfmarray[[3]][[3]])
topmod3 <- createtopmod(dfmarray[[3]][[3]])
t(as.data.frame(terms(topmod3,5)))
## [,1] [,2] [,3] [,4]
## Topic 1 "%_e1_%" "._._." "።_‹_‹" "e1_%_88"
## Topic 2 "ቀን_2005_ዓ.ም" "2005_ዓ.ም_." "._._." "share_this_:"
## Topic 3 "-_-_-" "።_ነገር_ግን" "የት_አለ_?" "የኢትዮጵያ_ሰላም_አስከባሪ"
## Topic 4 "._._." "._._read" "read_more_»" "._read_more"
## Topic 5 "._._." "።_‹_‹" "/_1954_ዓ.ም" "ኢሳት_ዜና_:"
## Topic 6 "ንዑስ_አንቀጽ_/" "377_/_96" "ሺ_ይቀጣል_፣" "._._."
## Topic 7 "/_ሪት_ብርቱካን" "/_ት_ብርቱካን" "ዶ_/_ር" "._._."
## Topic 8 "._._." "like_this_:" "ሚ_/_ር" "ዶ_/_ር"
## Topic 9 "ዶ_/_ር" "'_ወያነ_'" "ማለት_ነው_።" "ዓ_/_ም"
## Topic 10 "ት_/_ት" "rate_this_:" "ማለት_ነው_።" "ኦርቶዶክስ_ተዋሕዶ_ቤተ"
## Topic 11 "._._." "?_ኦባንግ_፦" "።_ጎልጉል_፦" "።_ነገር_ግን"
## Topic 12 "._._." "!_!_!" "=_=_=" "?_?_?"
## Topic 13 "።_ነገር_ግን" "?_?_?" "…_._." "ማለት_ነው_።"
## Topic 14 "._._." "ዜና_:_-" "ኢሳት_ዜና_:" "read_more_»"
## Topic 15 "!_!_!" "።_‹_‹" "ኢትዮ_-_ቻናል" "?_አቶ_ተስፋዬ"
## Topic 16 "።_አዲስ_ዘመን" "አዲስ_ዘመን_፦" "ጠ_/_ሚ" "/_ሚ_መለስ"
## Topic 17 "ማለት_ነው_።" "ገ_/_መድኅን" "።_ነገር_ግን" "ፀረ_-_ሰላም"
## Topic 18 "._._." "^_^_^" "!_!_!" "ቀን_2003_ዓ.ም"
## Topic 19 "._._." "።_ነገር_ግን" "።_‹_‹" "ነው_።_ይህ"
## Topic 20 "=_=_=" "._._." "።_‹_‹" "መለስ_/_ወያኔ"
## [,5]
## Topic 1 "%_88_%"
## Topic 2 "ማለት_ነው_።"
## Topic 3 "ሰላም_አስከባሪ_ሰራዊት"
## Topic 4 "ዜና_:_-"
## Topic 5 "ዜና_:_-"
## Topic 6 "፣_‹_‹"
## Topic 7 "ወ_/_ሪት"
## Topic 8 "/_ሚ_/"
## Topic 9 "ወ_/_ሮ"
## Topic 10 "ብቻ_ነው_።"
## Topic 11 "read_more_»"
## Topic 12 "ብቻ_ነው_።"
## Topic 13 "ነው_።_ይህ"
## Topic 14 "ም_ኢሳት_ዜና"
## Topic 15 "አቶ_ተስፋዬ_፡"
## Topic 16 "ሚ_መለስ_፦"
## Topic 17 "ሂዩማን_ራይትስ_ዎች"
## Topic 18 "ጽ_/_ቤት"
## Topic 19 "ጽ_/_ቤት"
## Topic 20 "።_ነገር_ግን"