data_dir = 'data' # compressed archives
dest_dir = 'data/unzip' # destination for unzipped files

# unzip automatic - run first time only
for (file in list.files(data_dir, pattern='.*\\.bz2', full.names=TRUE)){
  untar(file,exdir=dest_dir)
}

# load data automatically
spam_email <- vector() # init
ham_email <- vector() # init
df <- data.frame(source=character(), label=character(), email=character()) # init

# load spam
for (path in list.files(dest_dir, pattern='.*spam.*', full.names=TRUE)){
  for (f in grep(list.files(path,full.names=TRUE), pattern='.*cmd.*', invert=TRUE, value=TRUE)){
    df[nrow(df)+1,] = c(path,'spam',read_file(f))
  }
}

# load ham
for (path in list.files(dest_dir, pattern='.*ham.*', full.names=TRUE)){
  for (f in grep(list.files(path,full.names=TRUE), pattern='.*cmd.*', invert=TRUE, value=TRUE)){
    df[nrow(df)+1,] = c(path,'ham',read_file(f))
  }
}

# convert to latin1
df$email <- str_conv(df$email, 'latin1')

# separate header
df$header <- substr(df$email,1,str_locate(df$email,'\n\n')) 

# separate body
df$body <- substr(df$email,str_locate(df$email,'\n\n'),str_length(df$email))

# strip html tags and line breaks
strip_html <- function(x){gsub('</?[^>]+>|\\n|=','',x)}
df$body <- sapply(df$body,strip_html) # works

df <- df %>% select(!email) # drop column

# tokenize
df_train_words <- df_train %>%
  unnest_tokens(word,body) %>%
  group_by(label, word) %>%
  summarize(count=n())

# clean up
df_train_words_tidy <- df_train_words %>%
  anti_join(stop_words) %>% # filter out stop words
  filter(grepl('^[:alpha:]',word)) %>% # filter out numeric / non-alpha entries
  filter(count>5) %>% # filter out low-frequency terms
  filter(str_length(word)>2) %>% # filter out 1-2 charater terms
  arrange(desc(count),word)

# get top words for spam/ham
df_spam_words <- df_train_words_tidy %>%
  filter(label=='spam')

df_ham_words <- df_train_words_tidy %>%
  filter(label=='ham')

# spam words only
df_spam_words_unique <- df_spam_words %>%
  anti_join(df_ham_words, by=c('word') )

# ham words only
df_ham_words_unique <- df_ham_words %>%
  anti_join(df_spam_words, by=c('word') )

# tokenize
df_test_words <- df_test %>%
  select(!label) %>%
  unnest_tokens(word,body) %>%
  group_by(word,email_id) %>%
  summarize(count=n())

# clean up
df_test_words_tidy <- df_test_words %>%
  anti_join(stop_words) %>% 
  filter(grepl('^[:alpha:]',word)) %>% 
  filter(str_length(word)>2)

# join with SPAM words list and get count
df_test_words_spam <- df_test_words_tidy %>%
  inner_join(y = df_spam_words_unique, by=c('word')) %>%
  select(word,email_id,count.x) %>%
  rename(spam = count.x)

# join with HAM words list and get count
df_test_words_ham <- df_test_words_tidy %>%
  inner_join(y = df_ham_words_unique, by=c('word')) %>%
  select(word,email_id,count.x) %>%
  rename(ham = count.x)

# total up SPAM score per email
df_test_spam_scores <- df_test_words_spam %>%
  group_by(email_id) %>%
  summarize(spam = sum(spam))

# total up HAM score per email
df_test_ham_scores <- df_test_words_ham %>%
  group_by(email_id) %>%
  summarize(ham = sum(ham))

# join back to df
df_results <- df %>%
  filter(segment==1) %>% # just the test segment
  left_join(df_test_spam_scores, by=c('email_id')) %>%
  left_join(df_test_ham_scores, by=c('email_id'))

# fix NAs
df_results[is.na(df_results)] <- 0 # so we can calculate wihout NA
  
# if spam score > ham score, flag the email as 'spam'. success=1 if predicted correctly.
df_results <- df_results %>%
  mutate(flag=if_else(spam>ham,'spam','ham')) %>%
  mutate(success=if_else(label==flag,1,0))

# accuracy level
accuracy <- df_results %>%
  filter(segment==1) %>%
  select(success) %>%
  summarize(accuracy=sum(success)/length(success)) %>%
  pull()

# confusion matrix
conf_matrix <- df_results %>%
  filter(segment==1) %>%
  select(label,success) %>% 
  group_by(label,success) %>%
  summarize(count=n())

library(randomForest)

df_train_rf <- df_train %>%
  mutate(label=as.factor(label)) # convert to factor for classification

rf <- randomForest(label ~ ., data = df_train_rf, importance=TRUE, 
                   proximity=TRUE)

rf_predict <- predict(object=rf, newdata=df_test)

rf_results <- df_test %>%
  cbind(flag=rf_predict) %>%
  mutate(success=if_else(label==flag,1,0)) %>%
  select(!body)

# accuracy level
rf_accuracy <- rf_results %>%
  select(success) %>%
  summarize(accuracy=sum(success)/length(success)) %>%
  pull()

# confusion matrix
rf_conf_matrix <- rf_results %>%
  select(label,success) %>% 
  group_by(label,success) %>%
  summarize(count=n())