Tokenizing

` Cek Versi dari R yang dipergunakan dan packages yang diinstall

sessionInfo()
## R version 3.1.2 (2014-10-31)
## Platform: i386-w64-mingw32/i386 (32-bit)
## 
## locale:
## [1] LC_COLLATE=English_United States.1252 
## [2] LC_CTYPE=English_United States.1252   
## [3] LC_MONETARY=English_United States.1252
## [4] LC_NUMERIC=C                          
## [5] LC_TIME=English_United States.1252    
## 
## attached base packages:
## [1] stats     graphics  grDevices utils     datasets  methods   base     
## 
## loaded via a namespace (and not attached):
##  [1] digest_0.6.9    evaluate_0.8.3  htmltools_0.3.5 knitr_1.12.3   
##  [5] magrittr_1.5    Rcpp_0.12.4     rmarkdown_0.9.5 stringi_1.0-1  
##  [9] stringr_1.0.0   tools_3.1.2     yaml_2.1.13

Tokenizing

library(tm)
## Loading required package: NLP
Text=c("We will never know the real answer, before you try")
MC_tokenizer(Text)
##  [1] "We"     "will"   "never"  "know"   "the"    "real"   "answer"
##  [8] ""       "before" "you"    "try"
scan_tokenizer(Text)
##  [1] "We"      "will"    "never"   "know"    "the"     "real"    "answer,"
##  [8] "before"  "you"     "try"
strsplit_space_tokenizer <- function(x) unlist(strsplit(x, "[[:space:]]+"))
strsplit_space_tokenizer(Text)
##  [1] "We"      "will"    "never"   "know"    "the"     "real"    "answer,"
##  [8] "before"  "you"     "try"

Frase Tokenizing

library(tm)
library(ggplot2)
## Warning: package 'ggplot2' was built under R version 3.1.3
## 
## Attaching package: 'ggplot2'
## The following object is masked from 'package:NLP':
## 
##     annotate
library(lsa)
## Warning: package 'lsa' was built under R version 3.1.3
## Loading required package: SnowballC
if (Sys.getenv("JAVA_HOME")!="")
  Sys.setenv(JAVA_HOME="")
#install java machine
library(rJava)
library(RWeka)
NGramTokenizer(Text, Weka_control(min = 2, max = 2))
## [1] "We will"       "will never"    "never know"    "know the"     
## [5] "the real"      "real answer"   "answer before" "before you"   
## [9] "you try"

Term Doc Matrix

library(tm)
library(ggplot2)
library(lsa)
if (Sys.getenv("JAVA_HOME")!="")
  Sys.setenv(JAVA_HOME="")
library(rJava)
library(RWeka)
Text=as.character(Text)
corpus <- Corpus(VectorSource(Text))
BigramTokenizer <- function(x) NGramTokenizer(x, Weka_control(min = 2, max = 2))
tdm <- TermDocumentMatrix(corpus, control = list(tokenize = BigramTokenizer))
NGramTokenizer(Text, Weka_control(min = 2, max = 2))
## [1] "We will"       "will never"    "never know"    "know the"     
## [5] "the real"      "real answer"   "answer before" "before you"   
## [9] "you try"

Filtering

Text=c("We will never know the real answer, before you try !!!!!!")

corpus <- Corpus(VectorSource(Text))


h=c("we","you")
corpus <- tm_map(corpus, tolower)
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeWords,h)
Text=c("We zul dua will never know the real answer, before you try !!!!!!")
myStopwords <- c(stopwords('english'))
corpus <- Corpus(VectorSource(Text))
h=c("zul","dua")
corpus <- tm_map(corpus, tolower)
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeWords,myStopwords)
corpus <- tm_map(corpus, removeWords,h)

Stemming

Text=c("We will never know the real answer, before you try !!!!!!")

corpus <- Corpus(VectorSource(Text))


h=c("we","you")
corpus <- tm_map(corpus, tolower)
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeWords,h)
corpus = tm_map(corpus, stemDocument, language = "english")

Analyzing

library(tm)
if (Sys.getenv("JAVA_HOME")!="")
  Sys.setenv(JAVA_HOME="")
library(lsa)
text <- c("transporting food by cars will cause global warming. so we should go local.", 
    "we should try to convince our parents to stop using cars because it will cause global warming.", 
    "some food, such as mongo, requires a warm weather to grow. so they have to be transported to canada.", 
    "a typical electronic circuit can be built with a battery, a bulb, and a switch.", 
    "electricity flows from batteries to the bulb, just like water flows through a tube.", 
    "batteries have chemical energe in it. then electrons flow through a bulb to light it up.", 
    "birds can fly because they have feather and they are light.", "why some birds like pigeon can fly while some others like chicken cannot?", 
    "feather is important for birds' fly. if feather on a bird's wings is removed, this bird cannot fly.")
view <- factor(rep(c("view 1", "view 2", "view 3"), each = 3))
df <- data.frame(text, view, stringsAsFactors = FALSE)


corpus =Corpus(VectorSource(df$text))
td.mat <- as.matrix(TermDocumentMatrix(corpus))
td.mat.lsa <- lw_bintf(td.mat) * gw_idf(td.mat)  # weighting

Word Cloud

library(wordcloud)
## Warning: package 'wordcloud' was built under R version 3.1.3
## Loading required package: RColorBrewer
## Warning: package 'RColorBrewer' was built under R version 3.1.3
library(Rcpp)
## Warning: package 'Rcpp' was built under R version 3.1.3
library(RColorBrewer)
library(tm)
setwd("d:/")
data=read.csv("cloud.csv",header=TRUE,sep=",")
ap.d <- data.frame(data)
table(ap.d$freq)
## 
## 11 14 15 20 26 27 
##  2  1  2  1  1  1
pal2 <- brewer.pal(8,"Dark2")
wordcloud(ap.d$word,ap.d$freq,colors=pal2)