Data 607 - Project 4

1 · 2

library(kableExtra)
library(stringr)
library(tm)
library(RTextTools)
library(tidytext)
library(dplyr)
library(ggplot2)
library(wordcloud)
spam_dir<-"spam"
number_spam<-length(list.files(spam_dir, all.files = "FALSE", full.names = "TRUE"))
print(paste("There is a total of",number_spam,"spam emails"))
## [1] "There is a total of 501 spam emails"
list_spam<-list.files(spam_dir, all.files = "FALSE", full.names = "TRUE")

n<-0
for(i in 1:number_spam){
  tmp<-readLines(list_spam[i])
  if (length(tmp) != 0){
    #Subject
    subject_pos<-grep("Subject:", tmp, ignore.case = FALSE, value = FALSE)
    subject_tmp<-tmp[subject_pos][1]    
    subject_txt<-str_trim(str_sub(subject_tmp, start = 9))
    if (!is.na(subject_txt)) {
      n<-n+1
      #Date
      date_pos<-grep("Date:", tmp, ignore.case = FALSE, value = FALSE)
      date_tmp<-tmp[date_pos][1]         
      date_txt<-str_trim(str_sub(date_tmp, start = 6))
      #find tags to remove    
      content_type_pos<-grep("Content-Type:", tmp, ignore.case = FALSE, value = FALSE)
      content_type_tmp<-tmp[content_type_pos][1] 
      content_type_txt<-str_trim(str_sub(content_type_tmp, start = 14))
      content_trans_encod_pos<-grep("Content-Transfer-Encoding:", tmp, ignore.case = FALSE, value = FALSE)
      content_trans_encod_tmp<-tmp[content_trans_encod_pos][1] 
      content_trans_encod_txt<-str_trim(str_sub(content_trans_encod_tmp, start = 27))
      x_originalArraivelTime_pos<-grep("X-OriginalArrivalTime:", tmp, ignore.case = FALSE, value = FALSE)
      x_originalArraivelTime_tmp<-tmp[x_originalArraivelTime_pos][1] 
      x_originalArraivelTime_txt<-str_trim(str_sub(x_originalArraivelTime_tmp, start = 23))
      x_Beenthere_pos<-grep("X-Beenthere:", tmp, ignore.case = FALSE, value = FALSE)
      x_Beenthere_tmp<-tmp[x_Beenthere_pos][1] 
      x_Beenthere_txt<-str_trim(str_sub(x_Beenthere_tmp, start = 13))
      #Find where content is by eliminating tags
      start_line<-max(unlist(subject_pos), unlist(date_pos), unlist(content_type_pos), unlist(content_trans_encod_pos), unlist(x_originalArraivelTime_pos),unlist(x_Beenthere_pos)) + 1
      end_line<-length(tmp)
      #Content
      #We do some cleaning at the document level
      tmp<-tmp[start_line:end_line]
      tmp<-str_c(tmp, collapse = " ")
      tmp<-str_replace_all(tmp, pattern="<.*?>", replacement = " ")
      stopwords_regex<-paste(stopwords('en'), collapse = '\\b|\\b')
      stopwords_regex<-paste0('\\b', stopwords_regex, '\\b')
      tmp<-stringr::str_replace_all(tmp, stopwords_regex, '')
      if(n==1){
        mail_corpus<-VCorpus(VectorSource(tmp))
      } else{
        tmp_corpus<-VCorpus(VectorSource(tmp))
        mail_corpus<-c(mail_corpus, tmp_corpus)
      }
      meta(mail_corpus[[n]], "subject")<-subject_txt
      meta(mail_corpus[[n]], "date")<-date_txt
      meta(mail_corpus[[n]], "content_type")<-content_type_txt
      meta(mail_corpus[[n]], "class")<-"s"
    }
  }
}
spam_corpus<-mail_corpus
print(paste("There is a total of",n,"valid spam emails"))
## [1] "There is a total of 489 valid spam emails"
ham_dir<-"ham"
number_ham<-length(list.files(ham_dir, all.files = "FALSE", full.names = "TRUE"))
print(paste("There is a total of",number_ham,"ham emails"))
## [1] "There is a total of 2551 ham emails"
list_ham<-list.files(ham_dir, all.files = "FALSE", full.names = "TRUE")

m<-n
for(i in 1:number_ham){
  tmp<-readLines(list_ham[i])
  if (length(tmp) != 0){
    #Subject
    subject_pos<-grep("Subject:", tmp, ignore.case = FALSE, value = FALSE)
    subject_tmp<-tmp[subject_pos][1]    
    subject_txt<-str_trim(str_sub(subject_tmp, start = 9))
    if (!is.na(subject_txt)) {
      n<-n+1
      #Date
      date_pos<-grep("Date:", tmp, ignore.case = FALSE, value = FALSE)
      date_tmp<-tmp[date_pos][1]         
      date_txt<-str_trim(str_sub(date_tmp, start = 6))
      #find tags to remove    
      content_type_pos<-grep("Content-Type:", tmp, ignore.case = FALSE, value = FALSE)
      content_type_tmp<-tmp[content_type_pos][1] 
      content_type_txt<-str_trim(str_sub(content_type_tmp, start = 14))
      content_trans_encod_pos<-grep("Content-Transfer-Encoding:", tmp, ignore.case = FALSE, value = FALSE)
      content_trans_encod_tmp<-tmp[content_trans_encod_pos][1] 
      content_trans_encod_txt<-str_trim(str_sub(content_trans_encod_tmp, start = 27))
      x_originalArraivelTime_pos<-grep("X-OriginalArrivalTime:", tmp, ignore.case = FALSE, value = FALSE)
      x_originalArraivelTime_tmp<-tmp[x_originalArraivelTime_pos][1] 
      x_originalArraivelTime_txt<-str_trim(str_sub(x_originalArraivelTime_tmp, start = 23))
      x_Beenthere_pos<-grep("X-Beenthere:", tmp, ignore.case = FALSE, value = FALSE)
      x_Beenthere_tmp<-tmp[x_Beenthere_pos][1] 
      x_Beenthere_txt<-str_trim(str_sub(x_Beenthere_tmp, start = 13))
      #Find where content is by eliminating tags
      start_line<-max(unlist(subject_pos), unlist(date_pos), unlist(content_type_pos), unlist(content_trans_encod_pos), unlist(x_originalArraivelTime_pos),unlist(x_Beenthere_pos)) + 1
      end_line<-length(tmp)
      #Content
      #We do some cleaning at the document level
      tmp<-tmp[start_line:end_line]
      tmp<-str_c(tmp, collapse = " ")
      tmp<-str_replace_all(tmp, pattern="<.*?>", replacement = " ")
      stopwords_regex<-paste(stopwords('en'), collapse = '\\b|\\b')
      stopwords_regex<-paste0('\\b', stopwords_regex, '\\b')
      tmp<-stringr::str_replace_all(tmp, stopwords_regex, '')
      tmp_corpus<-VCorpus(VectorSource(tmp))
      mail_corpus<-c(mail_corpus, tmp_corpus)
      meta(mail_corpus[[n]], "subject")<-subject_txt
      meta(mail_corpus[[n]], "date")<-date_txt
      meta(mail_corpus[[n]], "content_type")<-content_type_txt
      meta(mail_corpus[[n]], "class")<-"h"
    }
  }
}
print(paste("There is a total of",n-m,"valid ham emails"))
## [1] "There is a total of 2545 valid ham emails"
mail_corpus<-sample(mail_corpus)
mail_corpus
## <<VCorpus>>
## Metadata:  corpus specific: 0, document level (indexed): 0
## Content:  documents: 3034
labels <- meta(mail_corpus, type = "local", tag = "class")
mail_corpus<-tm_map(mail_corpus, removeNumbers)
mail_corpus<-tm_map(mail_corpus, removeWords, words = stopwords("en"))
mail_corpus<-tm_map(mail_corpus, tolower)
mail_corpus<-tm_map(mail_corpus, stripWhitespace)
mail_corpus<-tm_map(mail_corpus, PlainTextDocument)
dtm_mail<-DocumentTermMatrix(mail_corpus)
dtm_mail<-removeSparseTerms(dtm_mail, 1-(20/length(mail_corpus)))
dtm_mail
## <<DocumentTermMatrix (documents: 3034, terms: 2150)>>
## Non-/sparse entries: 153572/6369528
## Sparsity           : 98%
## Maximal term length: 72
## Weighting          : term frequency (tf)
Terms(dtm_mail) %>% kable() %>% kable_styling() %>% scroll_box(height = "900px")
class_label <- unlist(labels)
training_size<-length(class_label)*.75
container <- create_container(dtm_mail, labels = class_label, trainSize = 1:training_size, testSize = (training_size+1):length(class_label), virgin = FALSE)
model <- train_model(container, "SVM")
model_out <- classify_model(container, model)
labels_out <- data.frame(correct_lable = class_label[(training_size+1):length(class_label)], svm = as.character(model_out[,1]), stringsAsFactors = FALSE)
print("Number of incorrect and correct classifications")
## [1] "Number of incorrect and correct classifications"
table(labels_out[, 1] == labels_out[, 2])
## 
## FALSE  TRUE 
##    17   741
print("Poportion of incorrect and correct classifications")
## [1] "Poportion of incorrect and correct classifications"
prop.table(table(labels_out[, 1] == labels_out[, 2]))
## 
##      FALSE       TRUE 
## 0.02242744 0.97757256
dtm_spam<-DocumentTermMatrix(spam_corpus)
dtm_spam<-removeSparseTerms(dtm_spam, 1-(20/length(spam_corpus)))
dtm_spam
## <<DocumentTermMatrix (documents: 489, terms: 453)>>
## Non-/sparse entries: 20552/200965
## Sparsity           : 91%
## Maximal term length: 41
## Weighting          : term frequency (tf)
Terms(dtm_spam) %>% kable() %>% kable_styling() %>% scroll_box(height = "900px")
tidy_spam<-tidy(dtm_spam)
tidy_spam<-select(tidy_spam,-document)
colnames(tidy_spam)<-c("word","n")
tidy_spam$n<-as.integer(tidy_spam$n)
tidy_spam %>% kable() %>% kable_styling() %>% scroll_box(height = "900px")
tidy_spam %>%
  count(word, sort = TRUE) %>%
  filter(nn > 100) %>%
  mutate(word = reorder(word, nn)) %>%
  ggplot(aes(word, nn)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()
tidy_spam %>%
  count(word) %>%
  with(wordcloud(word, nn, max.words = 30))

Data 607 - Project 4

Peter Kowalchuk

November 2, 2018

Document Classification

x
_______________________________________________
_________________________________________________________________
—
—–
——-
————-
—————–
————————
——————————————————-
———————————————————————~->
———————~–>
—–begin
—–end
–]
–huckleberry
–j.
-dave
;-)
:-(
:-)
:-).
:-/
::pm
:thinkgeek
!!!
..-acpi
…
….
.e.
’decline
’ll
’re
’ve
“…
“computers
“duncan”
“if
“in
“it
“the

x
…
’ll
’re
(un)subscription
&

=20
100
100%
2002
absolutely
accept
access
account
act
actually
address
addresses
advertising
affiliate
after
age
all
allow
already
also
always
amount
and
another
answer
any
anyone
anything
approved
are
around
assist
assistance
available

word	n
’ll	1
access	1
and	1
best	3
can	1
click	2
companies	1
country	1
dollars	1
e-mail	1
email	1
email.	1
even	1
family	1
financial	1
for	1
free	3
get	1
have	1
help	1
here	2
hundreds	1
insurance	1
investment	1
less	1
let	1
life	6
list,	1
lowest	1
more	1
new	1
our	1
please	2
provide	1
removed	1
save	3
security	1
service	1
simple	1
state	1