Introduction

It can be useful to be able to classify new “test” documents using already classified “training” documents. A common example is using a corpus of labeled spam and ham (non-spam) e-mails to predict whether or not a new document is spam.

For this project, you can start with a spam/ham dataset, then predict the class of new documents (either withheld from the training dataset or from another source such as your own spam folder).

Libraries Needed

I decieded to use Random Forest inorder to train and test my documents. I have never used Random Forest so it was an experience. Althought I did not figure out my problem in the end, I came to understand about data size, tables, and data.frames alot more. We begin by loading the libraries that are needed. Here I was introduced to tm, caret, and data.table.

library(tidyverse)
## Registered S3 methods overwritten by 'ggplot2':
##   method         from 
##   [.quosures     rlang
##   c.quosures     rlang
##   print.quosures rlang
## Registered S3 method overwritten by 'rvest':
##   method            from
##   read_xml.response xml2
## ── Attaching packages ────────────────────────────────── tidyverse 1.2.1 ──
## ✔ ggplot2 3.1.1       ✔ purrr   1.0.1  
## ✔ tibble  2.1.1       ✔ dplyr   0.8.0.1
## ✔ tidyr   0.8.3       ✔ stringr 1.4.0  
## ✔ readr   1.3.1       ✔ forcats 0.4.0
## ── Conflicts ───────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
library(openintro)
## Loading required package: airports
## Loading required package: cherryblossom
## Loading required package: usdata
library(randomForest)
## randomForest 4.6-14
## Type rfNews() to see new features/changes/bug fixes.
## 
## Attaching package: 'randomForest'
## The following object is masked from 'package:dplyr':
## 
##     combine
## The following object is masked from 'package:ggplot2':
## 
##     margin
library(tidytext)
library(tm)
## Loading required package: NLP
## 
## Attaching package: 'NLP'
## The following object is masked from 'package:ggplot2':
## 
##     annotate
library(caret)
## Loading required package: lattice
## 
## Attaching package: 'lattice'
## The following objects are masked from 'package:openintro':
## 
##     ethanol, lsegments
## 
## Attaching package: 'caret'
## The following object is masked from 'package:openintro':
## 
##     dotPlot
## The following object is masked from 'package:purrr':
## 
##     lift
library(data.table)
## 
## Attaching package: 'data.table'
## The following objects are masked from 'package:dplyr':
## 
##     between, first, last
## The following object is masked from 'package:purrr':
## 
##     transpose

Obtaining the Data

Obtaining the data was a little bit more direct that I planned, I planned on making functions but I only used them twice. I managed to open the email files and put them into a list.

set.seed(2121)

dirpath1 <- getwd()

dirpath2 <- paste0(dirpath1, "/spam_2/")
dirpath3 <- paste0(dirpath1, "/easy_ham/")

# List all the text files in the directory using list.files()
spam_textfiles <- list.files(dirpath2)
ham_textfiles <- list.files(dirpath3)

spam_emails <- list()
ham_emails <- list()

for (i in 1:length(spam_textfiles)) {
  filename <- paste0(dirpath2, spam_textfiles[i])
  email <- readLines(filename)
  spam_emails[[i]] <- email
}

for (i in 1:length(ham_textfiles)) {
  filename <- paste0(dirpath3, ham_textfiles[i])
  email <- readLines(filename)
  ham_emails[[i]] <- email
}

Making a list of Words

Next we had to break up the email files into words, so that I can use them inside the random forest classifier. Unfortunately, when I used all the spam2 and ham emails, and broke them up into words, later down the line, the computer told me that it was too big and that I was using alot of memory. So, I broke it down to the top 20 emails.

clean_words20 <- list()
spam_clean_words20 <- list()

for (i in 1:20 )  {
    email_text <- ham_emails[i][grep("From", ham_emails[i]):length(ham_emails[i])]
    
    # removing the headers
    email_text <- email_text[grep("^\\s*$", email_text, invert = TRUE)]
    
    email_text <- gsub("\\\\", "\\", email_text)
    
    email_text <- as.character(email_text)
    
    email_text <- iconv(email_text, from = "gb2312", to = "UTF-8")
    email_text <- gsub("=\\r?\\n", "", email_text)
    
    # remove url and remove email addresses
    email_text <- gsub("http\\S+", "", email_text)
    email_text <- gsub("\\S+@\\S+", "", email_text)
    
    email_text <- gsub("[^[:alnum:]\\s\\s]", " ", email_text)
    
    email_text <- tolower(email_text)
    
    words <- unlist(strsplit(email_text, split = " "))
    
    clean_words20[[i]] <- words[words != ""]
}
for (i in 1:20 )  {
    email_text <- spam_emails[i][grep("From", spam_emails[i]):length(spam_emails[i])]
    
    # removing the headers
    email_text <- email_text[grep("^\\s*$", email_text, invert = TRUE)]
    
    email_text <- gsub("\\\\", "\\", email_text)
    
    email_text <- as.character(email_text)
    
    email_text <- iconv(email_text, from = "gb2312", to = "UTF-8")
    email_text <- gsub("=\\r?\\n", "", email_text)
    
    # remove url and remove email addresses
    email_text <- gsub("http\\S+", "", email_text)
    email_text <- gsub("\\S+@\\S+", "", email_text)
    
    email_text <- gsub("[^[:alnum:]\\s\\s]", " ", email_text)
    
    email_text <- tolower(email_text)
    
    words <- unlist(strsplit(email_text, split = " "))
    
    spam_clean_words20[[i]] <- words[words != ""]
}

Preparing the Spam and Test Data

# label the ones from spam_2 as "spam" and the ones from easy_ham as "test"
spam_labels <- rep("spam", length(spam_clean_words20))
test_labels <- rep("test", length(clean_words20))

# combine the two lists into a single data frame
df_em <- data.frame(cbind(c(spam_clean_words20, clean_words20), c(spam_labels, test_labels)))

# name the columns
colnames(df_em) <- c("Words", "Label")

#flattened_col <- unlist(df_em$Words)
#df_em$Words <- flattened_col

Preparing our Data for Random Forrest

Here, unfortunately, I could not get the words to work with the random forest classifier. My errors jumped back and forth from the label has to be in as a factor to Error in cut.default(y, breaks, include.lowest = TRUE) I could not find the answer to this error.

train_data <- subset(df_em, Label == "spam")
test_data <- subset(df_em, Label == "test")

# make things vectors or factors
train_data$Label <- as.factor(train_data$Label)
test_data$Label <- as.factor(test_data$Label)





corpus <- Corpus(VectorSource(train_data$Words))
corpus <- tm_map(corpus, removePunctuation)
## Warning in tm_map.SimpleCorpus(corpus, removePunctuation): transformation
## drops documents
corpus <- tm_map(corpus, removeWords, stopwords("english"))
## Warning in tm_map.SimpleCorpus(corpus, removeWords, stopwords("english")):
## transformation drops documents
corpus <- tm_map(corpus, stemDocument)
## Warning in tm_map.SimpleCorpus(corpus, stemDocument): transformation drops
## documents
dtm <- DocumentTermMatrix(corpus)

dtm <- cbind(Label = train_data$Label, as.matrix(dtm))

ctrl <- trainControl(method = "cv", number = 10)
# rf_model <- train(Label ~ ., data = dtm, method = "rf", trControl = ctrl, ntree = 500, mtry = 3, importance = TRUE)

Conclusion

The random forest classifier is an incredibly powerful machine learning algorith that can be used for a variety of tasks. Once you get it working, it can be a valubale tool in your data science toolkit. However, preparing your data properly is a crucial step in the process. It is essential to ensure that your data is in the correct format, that missing values are handled correctly, and that the relevant features are selected.

Scratch Work for this project

# 
# email_text <- emails[1][grep("From", emails[1]):length(emails[1])]
# 
# # removing the headers
# email_text <- email_text[grep("^\\s*$", email_text, invert = TRUE)]
# 
# email_text <- as.character(email_text)
# 
# email_text <- iconv(email_text, from = "gb2312", to = "UTF-8")
# email_text <- gsub("=\\r?\\n", "", email_text)
# 
# # remove url and remove email addresses
# email_text <- gsub("http\\S+", "", email_text)
# email_text <- gsub("\\S+@\\S+", "", email_text)
# 
# email_text <- gsub("[^[:alnum:]\\s\\s]", " ", email_text)
# 
# email_text <- tolower(email_text)
# 
# words <- unlist(strsplit(email_text, split = " "))
# 
# clean_words <- words[words != ""]
# 
# 
# clean_words <- list()
# spam_clean_words <- list()
# 
# 
# for (i in 1:( length(ham_emails)-1) )  {
#     email_text <- ham_emails[i][grep("From", ham_emails[i]):length(ham_emails[i])]
#     
#     # removing the headers
#     email_text <- email_text[grep("^\\s*$", email_text, invert = TRUE)]
#     
#     email_text <- gsub("\\\\", "\\", email_text)
#     
#     email_text <- as.character(email_text)
#     
#     email_text <- iconv(email_text, from = "gb2312", to = "UTF-8")
#     email_text <- gsub("=\\r?\\n", "", email_text)
#     
#     # remove url and remove email addresses
#     email_text <- gsub("http\\S+", "", email_text)
#     email_text <- gsub("\\S+@\\S+", "", email_text)
#     
#     email_text <- gsub("[^[:alnum:]\\s\\s]", " ", email_text)
#     
#     email_text <- tolower(email_text)
#     
#     words <- unlist(strsplit(email_text, split = " "))
#     
#     clean_words[[i]] <- words[words != ""]
# }
# 
# for (i in 1:( length(spam_emails)-1) )  {
#     email_text <- spam_emails[i][grep("From", spam_emails[i]):length(spam_emails[i])]
#     
#     # removing the headers
#     email_text <- email_text[grep("^\\s*$", email_text, invert = TRUE)]
#     
#     email_text <- gsub("\\\\", "\\", email_text)
#     
#     email_text <- as.character(email_text)
#     
#     email_text <- iconv(email_text, from = "gb2312", to = "UTF-8")
#     email_text <- gsub("=\\r?\\n", "", email_text)
#     
#     # remove url and remove email addresses
#     email_text <- gsub("http\\S+", "", email_text)
#     email_text <- gsub("\\S+@\\S+", "", email_text)
#     
#     email_text <- gsub("[^[:alnum:]\\s\\s]", " ", email_text)
#     
#     email_text <- tolower(email_text)
#     
#     words <- unlist(strsplit(email_text, split = " "))
#     
#     spam_clean_words[[i]] <- words[words != ""]
# }
# library(data.table)
# 
# # create data table for spam words and labels
# spam_dt <- data.table(words = unlist(spam_clean_words100), label = "spam")
# 
# # create data table for test words and labels
# test_dt <- data.table(words = unlist(clean_words100), label = "test")
# 
# # combine the two data tables
# df <- rbind(spam_dt, test_dt)