UP_ProjectFour

# Load Libraries
install.packages("RTextTools", repos = "http://cran.us.r-project.org")

## 
## The downloaded binary packages are in
##  /var/folders/5m/4f5rvwrn5rngf6j4gpl2mc9w0000gn/T//RtmpGHLfdz/downloaded_packages

library(RTextTools)

## Loading required package: SparseM

## 
## Attaching package: 'SparseM'

## The following object is masked from 'package:base':
## 
##     backsolve

library(tm)

## Warning: package 'tm' was built under R version 4.3.2

## Loading required package: NLP

install.packages("e1071", repos = "http://cran.us.r-project.org")

## 
## The downloaded binary packages are in
##  /var/folders/5m/4f5rvwrn5rngf6j4gpl2mc9w0000gn/T//RtmpGHLfdz/downloaded_packages

library("e1071")

# Read spam and ham data into DF
ham_directory <- "https://spamassassin.apache.org/old/publiccorpus/20021010_easy_ham.tar.bz2"
ham_docs_list <- lapply(list.files(ham_directory), function(file) {
  text <- readLines(file.path(ham_directory, file))
  list(text = paste(text, collapse = "\n"), type = "ham")
})
hamDF <- as.data.frame(do.call(rbind, ham_docs_list), stringsAsFactors = FALSE)

spam_directory <- "https://spamassassin.apache.org/old/publiccorpus/20030228_spam.tar.bz2"
spam_docs_list <- lapply(list.files(spam_directory), function(file) {
  text <- readLines(file.path(spam_directory, file))
  list(text = paste(text, collapse = "\n"), type = "spam")
})
spamDF <- as.data.frame(do.call(rbind, spam_docs_list), stringsAsFactors = FALSE)

# Combine spam and ham data frames
spam_ham_dataframe <- rbind(hamDF, spamDF)

# Prepare the Corpus
Ccorpus <- Corpus(VectorSource(spam_ham_dataframe$text))
Ccorpus <- tm_map(Ccorpus, content_transformer(tolower))
Ccorpus <- tm_map(Ccorpus, removeNumbers)
Ccorpus <- tm_map(Ccorpus, removePunctuation)
Ccorpus <- tm_map(Ccorpus, removeWords, stopwords("en"))
Ccorpus <- tm_map(Ccorpus, stripWhitespace)

# Create Document-term matrix
edtm <- DocumentTermMatrix(Ccorpus)

# Prepare test and train data
sample_size <- floor(0.70 * nrow(spam_ham_dataframe))
set.seed(7535)
train_ind <- sample(seq_len(nrow(spam_ham_dataframe)), size = sample_size)
train_spam_ham <- spam_ham_dataframe[train_ind, ]
test_spam_ham <- spam_ham_dataframe[-train_ind, ]

# Convert to DocumentTermMatrix
train_clean_corpus <- Corpus(VectorSource(train_spam_ham$text))
train_clean_corpus <- tm_map(train_clean_corpus, removeNumbers)
train_clean_corpus <- tm_map(train_clean_corpus, removePunctuation)
train_clean_corpus <- tm_map(train_clean_corpus, removeWords, stopwords("en"))
train_clean_corpus <- tm_map(train_clean_corpus, stripWhitespace)
train_edtm <- DocumentTermMatrix(train_clean_corpus)

test_clean_corpus <- Corpus(VectorSource(test_spam_ham$text))
test_clean_corpus <- tm_map(test_clean_corpus, removeNumbers)
test_clean_corpus <- tm_map(test_clean_corpus, removePunctuation)
test_clean_corpus <- tm_map(test_clean_corpus, removeWords, stopwords("en"))
test_clean_corpus <- tm_map(test_clean_corpus, stripWhitespace)
test_edtm <- DocumentTermMatrix(test_clean_corpus)


# Define a function to preprocess new documents
preprocess_new_documents <- function(new_documents) {
  new_corpus <- Corpus(VectorSource(new_documents))
  new_corpus <- tm_map(new_corpus, content_transformer(tolower))
  new_corpus <- tm_map(new_corpus, removeNumbers)
  new_corpus <- tm_map(new_corpus, removePunctuation)
  new_corpus <- tm_map(new_corpus, removeWords, stopwords("en"))
  new_corpus <- tm_map(new_corpus, stripWhitespace)
  return(new_corpus)
}

# Obtain new documents (for example)
new_docs <- c("This is a new email with some spam words",
              "Another new email that looks like ham")

# Preprocess the new documents
new_clean_corpus <- preprocess_new_documents(new_docs)

## Warning in tm_map.SimpleCorpus(new_corpus, content_transformer(tolower)):
## transformation drops documents

## Warning in tm_map.SimpleCorpus(new_corpus, removeNumbers): transformation drops
## documents

## Warning in tm_map.SimpleCorpus(new_corpus, removePunctuation): transformation
## drops documents

## Warning in tm_map.SimpleCorpus(new_corpus, removeWords, stopwords("en")):
## transformation drops documents

## Warning in tm_map.SimpleCorpus(new_corpus, stripWhitespace): transformation
## drops documents

# Convert to binary format
train_sms <- apply(train_edtm, 2, function(x) ifelse(x > 0, 1, 0))
test_sms <- apply(test_edtm, 2, function(x) ifelse(x > 0, 1, 0))