R Markdown

This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see http://rmarkdown.rstudio.com.

When you click the Knit button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this:

library(e1071)
library(cwhmisc)
## Loading required package: lattice
## Loading required package: grid
library(ggplot2)
library(caret)
library(stringr)
library(party)
## Loading required package: mvtnorm
## Loading required package: modeltools
## Loading required package: stats4
## Loading required package: strucchange
## Loading required package: zoo
## 
## Attaching package: 'zoo'
## The following objects are masked from 'package:base':
## 
##     as.Date, as.Date.numeric
## Loading required package: sandwich
## 
## Attaching package: 'strucchange'
## The following object is masked from 'package:stringr':
## 
##     boundary
library(randomForest)
## randomForest 4.6-14
## Type rfNews() to see new features/changes/bug fixes.
## 
## Attaching package: 'randomForest'
## The following object is masked from 'package:ggplot2':
## 
##     margin
library(ROCR)
## Loading required package: gplots
## 
## Attaching package: 'gplots'
## The following object is masked from 'package:stats':
## 
##     lowess
library(pROC)
## Type 'citation("pROC")' for a citation.
## 
## Attaching package: 'pROC'
## The following objects are masked from 'package:stats':
## 
##     cov, smooth, var
record_performance <- function(df, name, model, test) {
  svm.pred <- predict(model, test)
  svm.table <- table(pred = svm.pred, true=test$corr)
  df <- rbind(df, data.frame(model=c(name), score=c(classAgreement(svm.table)$diag)))
  return(df)
}

paren_match <- function(page, text) {
  start <- cpos(page, "(")
  
  end <- cpos(page, ")")
  if (!is.na(start) && !is.na(end)) {
    search <- substring(page, start + 1, end - 1)
    return(grepl(tolower(search), tolower(text), fixed=TRUE))
  } else {
    return(FALSE)
  }
}


setwd("C:/Users/yoges/Desktop/HW3 ENPM808W")
full <- read.csv("C:/Users/yoges/Desktop/HW3 ENPM808W/qb.train.csv")

full$obs_len <- apply(full, 1, function(x) {nchar(x['text'])})
full$scale_len <- scale(full$obs_len)

full$scale_score <- scale(full$body_score)

full$paren_match <- apply(full, 1, function(x) {paren_match(x['page'], x['text'])})

full$log_links <- scale(log(as.numeric(full$inlinks) + 1))

index <- 1:nrow(full)
testindex <- sample(index, trunc(length(index)/5))
testset <- full[testindex,]
trainset <- full[-testindex,]

# Get the most frequent baseline
mfc_baseline <- sum(testset$corr == "False") / nrow(testset)
results <- data.frame(model=c("MFC"), score=c(mfc_baseline))

results <- record_performance(results, "body_score", svm(corr ~ body_score, data=trainset), testset)
results <- record_performance(results, "scale_score", svm(corr ~ scale_score, data=trainset), testset)
results <- record_performance(results, "obs_len", svm(corr ~ obs_len, data=trainset), testset)
results <- record_performance(results, "score+len", svm(corr ~ obs_len + body_score, data=trainset), testset)
results <- record_performance(results, "paren+len", svm(corr ~ obs_len + paren_match, data=trainset), testset)
results <- record_performance(results, "paren_match", svm(corr ~ paren_match, data=trainset), testset)
results <- record_performance(results, "score+paren_match", svm(corr ~ scale_score + paren_match, data=trainset), testset)
results <- record_performance(results, "score+len+paren_match", svm(corr ~ scale_len + scale_score + paren_match, data=trainset), testset)
results <- record_performance(results, "links", svm(corr ~ inlinks, data=trainset), testset)
results <- record_performance(results, "loglinks", svm(corr ~ log_links, data=trainset), testset)
results <- record_performance(results, "score+len+links+paren_match", svm(corr ~ scale_len + scale_score + log_links + paren_match, data=trainset), testset)
results <- record_performance(results, "score+links+paren_match", svm(corr ~ scale_len + scale_score + paren_match, data=trainset), testset)


#("Modeling based on SVM Radial Kernel")
results
##                          model     score
## 1                          MFC 0.5640867
## 2                   body_score 0.7634675
## 3                  scale_score 0.7634675
## 4                      obs_len 0.5640867
## 5                    score+len 0.8160991
## 6                    paren+len 0.5795666
## 7                  paren_match 0.5795666
## 8            score+paren_match 0.7690402
## 9        score+len+paren_match 0.8092879
## 10                       links 0.6018576
## 11                    loglinks 0.6421053
## 12 score+len+links+paren_match 0.8105263
## 13     score+links+paren_match 0.8092879
## SVM by default uses radial kernel
## Making classifiers using logistic Regression
record_performance_logistic <- function(df, name, model, test) {
  glm.pred <- predict(model, newdata = test, type= "response")
  glm.pred <- factor(ifelse(glm.pred > 0.5,"TRUE","FALSE"))
  glm.table <- table(pred = glm.pred, true=test$corr)
  df <- rbind(df, data.frame(model=c(name), score=c(classAgreement(glm.table)$diag)))
  return(df)
}


mfc_baseline <- sum(testset$corr == "False") / nrow(testset)
results_logistic <- data.frame(model=c("MFC"), score=c(mfc_baseline))

results_logistic <- record_performance_logistic(results_logistic, "body_score", glm(corr ~ body_score, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "scale_score", glm(corr ~ scale_score, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "obs_len", glm(corr ~ obs_len, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "score+len", glm(corr ~ obs_len + body_score, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "paren+len", glm(corr ~ obs_len + paren_match, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "paren_match", glm(corr ~ paren_match, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "score+paren_match", glm(corr ~ scale_score + paren_match, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "score+len+paren_match", glm(corr ~ scale_len + scale_score + paren_match, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "links", glm(corr ~ inlinks, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "loglinks", glm(corr ~ log_links, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "score+len+links+paren_match", glm(corr ~ scale_len + scale_score + log_links + paren_match, data=trainset,family = "binomial"), testset)
results_logistic <- record_performance_logistic(results_logistic, "score+links+paren_match", glm(corr ~ scale_len + scale_score + paren_match, data=trainset,family = "binomial"), testset)

#("Modeling based on Logistic Regression")
results_logistic
##                          model     score
## 1                          MFC 0.5640867
## 2                   body_score 0.6817337
## 3                  scale_score 0.6817337
## 4                      obs_len 0.5572755
## 5                    score+len 0.7263158
## 6                    paren+len 0.5789474
## 7                  paren_match 0.5795666
## 8            score+paren_match 0.6916409
## 9        score+len+paren_match 0.7337461
## 10                       links 0.5640867
## 11                    loglinks 0.6309598
## 12 score+len+links+paren_match 0.6897833
## 13     score+links+paren_match 0.7337461
library(party)
record_performance_tree <- function(df, name, model, test) {
  tree.pred <- predict(model,newdata = test)
  tree.table <- table(pred = tree.pred, true=test$corr)
  df <- rbind(df, data.frame(model=c(name), score=c(classAgreement(tree.table)$diag)))
  return(df)
}
mfc_baseline <- sum(testset$corr == "False") / nrow(testset)
results_tree <- data.frame(model=c("MFC"), score=c(mfc_baseline))

results_tree <- record_performance_tree(results_tree , "body_score", ctree(corr ~ body_score, data=trainset), testset)
results_tree <- record_performance_tree(results_tree, "scale_score", ctree(corr ~ scale_score, data=trainset), testset)
results_tree <- record_performance_tree(results_tree, "obs_len", ctree(corr ~ obs_len, data=trainset), testset)
results_tree <- record_performance_tree(results_tree, "score+len", ctree(corr ~ obs_len + body_score, data=trainset), testset)
results_tree <- record_performance_tree(results_tree, "paren+len", ctree(corr ~ obs_len + paren_match, data=trainset), testset)
results_tree <- record_performance_tree(results_tree, "paren_match", ctree(corr ~ paren_match, data=trainset), testset)
results_tree<- record_performance_tree(results_tree, "score+paren_match", ctree(corr ~ scale_score + paren_match, data=trainset), testset)
results_tree<- record_performance_tree(results_tree, "score+len+paren_match", ctree(corr ~ scale_len + scale_score + paren_match, data=trainset), testset)
results_tree <- record_performance_tree(results_tree, "links", ctree(corr ~ inlinks, data=trainset), testset)
results_tree <- record_performance_tree(results_tree, "loglinks", ctree(corr ~ log_links, data=trainset), testset)
results_tree<- record_performance_tree(results_tree, "score+len+links+paren_match", ctree(corr ~ scale_len + scale_score + log_links + paren_match, data=trainset), testset)
results_tree <- record_performance_tree(results_tree, "score+links+paren_match", ctree(corr ~ scale_len + scale_score + paren_match, data=trainset), testset)

#("Modeling based on Decision tree")
results_tree
##                          model     score
## 1                          MFC 0.5640867
## 2                   body_score 0.7702786
## 3                  scale_score 0.7702786
## 4                      obs_len 0.5640867
## 5                    score+len 0.7826625
## 6                    paren+len 0.5795666
## 7                  paren_match 0.5795666
## 8            score+paren_match 0.7727554
## 9        score+len+paren_match 0.7839009
## 10                       links 0.6328173
## 11                    loglinks 0.6328173
## 12 score+len+links+paren_match 0.8111455
## 13     score+links+paren_match 0.7839009
### with sVM linear kernel model
mfc_baseline <- sum(testset$corr == "False") / nrow(testset)
results_svm_linear <- data.frame(model=c("MFC"), score=c(mfc_baseline))

results_svm_linear <- record_performance(results_svm_linear, "body_score", svm(corr ~ body_score, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "scale_score", svm(corr ~ scale_score, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "obs_len", svm(corr ~ obs_len, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "score+len", svm(corr ~ obs_len + body_score, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "paren+len", svm(corr ~ obs_len + paren_match, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "paren_match", svm(corr ~ paren_match, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "score+paren_match", svm(corr ~ scale_score + paren_match, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "score+len+paren_match", svm(corr ~ scale_len + scale_score + paren_match, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "links", svm(corr ~ inlinks, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "loglinks", svm(corr ~ log_links, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "score+len+links+paren_match", svm(corr ~ scale_len + scale_score + log_links + paren_match, data=trainset,kernel = "linear"), testset)
results_svm_linear <- record_performance(results_svm_linear, "score+links+paren_match", svm(corr ~ scale_len + scale_score + paren_match, data=trainset,kernel = "linear"), testset)

#("Modeling based on SVM linear Kernel")
results_svm_linear
##                          model     score
## 1                          MFC 0.5640867
## 2                   body_score 0.6965944
## 3                  scale_score 0.6965944
## 4                      obs_len 0.5640867
## 5                    score+len 0.7201238
## 6                    paren+len 0.5795666
## 7                  paren_match 0.5795666
## 8            score+paren_match 0.6984520
## 9        score+len+paren_match 0.7201238
## 10                       links 0.5640867
## 11                    loglinks 0.6328173
## 12 score+len+links+paren_match 0.7287926
## 13     score+links+paren_match 0.7201238
## SVM with Ploynomial Kernel
mfc_baseline <- sum(testset$corr == "False") / nrow(testset)
results_svm_poly <- data.frame(model=c("MFC"), score=c(mfc_baseline))

results_svm_poly <- record_performance(results_svm_poly, "body_score", svm(corr ~ body_score, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "scale_score", svm(corr ~ scale_score, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "obs_len", svm(corr ~ obs_len, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "score+len", svm(corr ~ obs_len + body_score, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "paren+len", svm(corr ~ obs_len + paren_match, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "paren_match", svm(corr ~ paren_match, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "score+paren_match", svm(corr ~ scale_score + paren_match, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "score+len+paren_match", svm(corr ~ scale_len + scale_score + paren_match, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "links", svm(corr ~ inlinks, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "loglinks", svm(corr ~ log_links, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "score+len+links+paren_match", svm(corr ~ scale_len + scale_score + log_links + paren_match, data=trainset,kernel = "polynomial"), testset)
results_svm_poly <- record_performance(results_svm_poly, "score+links+paren_match", svm(corr ~ scale_len + scale_score + paren_match, data=trainset,kernel = "polynomial"), testset)

#("Modeling based on SVM polynomial Kernel")
results_svm_poly
##                          model     score
## 1                          MFC 0.5640867
## 2                   body_score 0.6575851
## 3                  scale_score 0.6575851
## 4                      obs_len 0.5640867
## 5                    score+len 0.6712074
## 6                    paren+len 0.5795666
## 7                  paren_match 0.5795666
## 8            score+paren_match 0.7678019
## 9        score+len+paren_match 0.7993808
## 10                       links 0.5634675
## 11                    loglinks 0.5640867
## 12 score+len+links+paren_match 0.7975232
## 13     score+links+paren_match 0.7993808
##SVM with sigmoid kernel
mfc_baseline <- sum(testset$corr == "False") / nrow(testset)
results_svm_sigmoid <- data.frame(model=c("MFC"), score=c(mfc_baseline))

results_svm_sigmoid <- record_performance(results_svm_sigmoid, "body_score", svm(corr ~ body_score, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "scale_score", svm(corr ~ scale_score, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "obs_len", svm(corr ~ obs_len, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "score+len", svm(corr ~ obs_len + body_score, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "paren+len", svm(corr ~ obs_len + paren_match, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "paren_match", svm(corr ~ paren_match, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "score+paren_match", svm(corr ~ scale_score + paren_match, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "score+len+paren_match", svm(corr ~ scale_len + scale_score + paren_match, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "links", svm(corr ~ inlinks, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "loglinks", svm(corr ~ log_links, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "score+len+links+paren_match", svm(corr ~ scale_len + scale_score + log_links + paren_match, data=trainset,kernel = "sigmoid"), testset)
results_svm_sigmoid <- record_performance(results_svm_sigmoid, "score+links+paren_match", svm(corr ~ scale_len + scale_score + paren_match, data=trainset,kernel = "sigmoid"), testset)


#"Modeling based on SVM sigmoid Kernel"
results_svm_sigmoid
##                          model     score
## 1                          MFC 0.5640867
## 2                   body_score 0.5473684
## 3                  scale_score 0.5473684
## 4                      obs_len 0.4972136
## 5                    score+len 0.5622291
## 6                    paren+len 0.4934985
## 7                  paren_match 0.5795666
## 8            score+paren_match 0.4383901
## 9        score+len+paren_match 0.4854489
## 10                       links 0.6142415
## 11                    loglinks 0.5504644
## 12 score+len+links+paren_match 0.5238390
## 13     score+links+paren_match 0.4854489
logistic_best <-  glm(corr ~ scale_len + scale_score + paren_match, data=trainset,family = "binomial")
pred_logistic_best <- predict(logistic_best,newdata = testset,type = "response")
pred_try_round <- round(pred_logistic_best)
conf_matrix_logistic_best <- table(pred_try_round,testset$corr)
accuracy_logistic_best <-  sum(diag(conf_matrix_logistic_best))/sum(conf_matrix_logistic_best)
accuracy_logistic_best
## [1] 0.7337461
conf_matrix_logistic_best
##               
## pred_try_round False True
##              0   844  363
##              1    67  341
svm_best <- svm(corr ~ obs_len + body_score, data=trainset)
pred_svm_best <- predict(svm_best,newdata = testset)
conf_matrix_svm_best <- table(pred_svm_best,testset$corr)
accuracy_svm_best <- sum(diag(conf_matrix_svm_best))/sum(conf_matrix_svm_best)
accuracy_svm_best
## [1] 0.8160991
conf_matrix_svm_best
##              
## pred_svm_best False True
##         False   860  246
##         True     51  458
decision_tree_best <- ctree(corr ~ scale_len + scale_score + paren_match, data=trainset)
pred_dt_best <- predict(decision_tree_best,testset)
conf_best_dt <- table(pred_dt_best,testset$corr)
accuracy_best_dt <- sum(diag(conf_best_dt))/sum(conf_best_dt)
accuracy_best_dt
## [1] 0.7839009
conf_best_dt
##             
## pred_dt_best False True
##        False   849  287
##        True     62  417
## corr with respect to various varaibales used for classifying corr
category_table <- prop.table(table(full$corr,full$category))
## From this one can definitely conclude that there is just one factor level in category and is not that usefull feature for classification
prop.table(table(full$corr,full$paren_match))
##        
##               FALSE        TRUE
##   False 0.551182077 0.003094442
##   True  0.426785493 0.018937987
## From this table one can conclude that paren_match feature is good at classifying false but bad at classifying true for corr 
str(full$answer_type)
##  Factor w/ 9 levels "abs","anim","char",..: 9 5 5 7 5 5 7 9 9 9 ...
## Show that answer_type has 9 factors and lets plot them with facet grid to find if any of the factors are substantially more in one class than other
answer_typeVscorr.plot <- ggplot(full,aes(x = answer_type))
answer_typeVscorr.plot + geom_bar(stat = "count")+ facet_grid(.~corr)+ labs(title = "Answer_type Vs corr")

## One takeaway one can conclude from the graph Answer_type is that the answer type abs anim and event occur more in False category of the corr than True one so one can possibly rule that occurence of abs,anim and event is associated with corr being classified as FALSE

#Plotting body_score vs corr to find any pattern if there exists any
body_score.plot <- ggplot(data = trainset,aes(x = body_score, y = corr))+geom_point()+ labs(title = "Body_Score Vs corr")
body_score.plot+geom_vline(xintercept = 125,linetype = "dashed",color = "red")

## This body_score plot shows that if body_score is greater than 125 than it is categorized as TRUE for corr with only 3 outliers which make this rule wrong

test_accuracy <- as.data.frame(trainset)
## Making predicted model based on the inferences made above
test_accuracy$predicted <- ifelse(test_accuracy$body_score > 125, "True","False")
conf_try <- table(test_accuracy$predicted,test_accuracy$corr) 
accuracy_try <- sum(diag(conf_try))/sum(conf_try)
accuracy_try
## [1] 0.6274752
log_links_tab.plot <- ggplot(data = trainset,aes(x = log_links, y = corr))+geom_point()+ labs(title = "log_links Vs corr")
log_links_tab.plot

scale_score.plot <- ggplot(data = trainset,aes(x = scale_score, y = corr))+geom_point()+ labs(title = "scale_score Vs corr")
scale_score.plot+ geom_vline(xintercept = 2, linetype = "dashed", color = "blue")

test_accuracy$body_score_made <- ifelse(test_accuracy$body_score > 125,"True","False")
test_accuracy$scale_score_made <- ifelse(test_accuracy$scale_score < 2,"False","True")
test_accuracy$answer_type_made <- ifelse((test_accuracy$answer_type == "abs"| test_accuracy$answer_type == "anim"|test_accuracy$answer_type == "event"),"True","False")

test_accuracy$body_score_made <- as.factor(test_accuracy$body_score_made)
test_accuracy$scale_score_made <- as.factor(test_accuracy$scale_score_made)
test_accuracy$answer_type_made <- as.factor(test_accuracy$answer_type_made)

testing_accuracy_rf_model <- randomForest(corr ~ body_score_made + scale_score_made + answer_type_made,test_accuracy)
development_try_data <- testset
development_try_data$body_score_made <- ifelse(development_try_data$body_score > 125,"True","False")
development_try_data$scale_score_made <- ifelse(development_try_data$scale_score < 2,"False","True")
development_try_data$answer_type_made <- ifelse((development_try_data$answer_type == "abs"| development_try_data$answer_type == "anim"|development_try_data$answer_type == "event"),"True","False")

development_try_data$body_score_made <- as.factor(development_try_data$body_score_made)
development_try_data$scale_score_made <- as.factor(development_try_data$scale_score_made)
development_try_data$answer_type_made <- as.factor(development_try_data$answer_type_made)


try_development_pred <- predict(testing_accuracy_rf_model,development_try_data)
conf_development <- table(try_development_pred,development_try_data$corr)
accuracy_development <- sum(diag(conf_development))/sum(conf_development)
accuracy_development
## [1] 0.6408669
## making random Forest Classifiers for the given problem
## obs_len+body_score - 18.81% error
library(randomForest)
 record_performance_rf <- function(df, name, model, test) {
    rf.pred <- predict(model, test)
    rf.table <- table(pred = rf.pred, true=test$corr)
    df <- rbind(df, data.frame(model=c(name), score=c(classAgreement(rf.table)$diag)))
    return(df)
 }
 
mfc_baseline <- sum(testset$corr == "False") / nrow(testset)
results_rf <- data.frame(model=c("MFC"), score=c(mfc_baseline))
 
 
results_rf <- record_performance_rf(results_rf, "body_score", randomForest(corr ~ body_score, data=trainset), testset)
results_rf <- record_performance(results_rf, "scale_score", randomForest(corr ~ scale_score, data=trainset), testset)
results_rf <- record_performance(results_rf, "obs_len", randomForest(corr ~ obs_len, data=trainset), testset)
results_rf <- record_performance(results_rf, "score+len", randomForest(corr ~ obs_len + body_score, data=trainset), testset)
results_rf <- record_performance(results_rf, "paren+len", randomForest(corr ~ obs_len + paren_match, data=trainset), testset)
results_rf <- record_performance(results_rf, "paren_match", randomForest(corr ~ paren_match, data=trainset), testset)
results_rf <- record_performance(results_rf, "score+paren_match", randomForest(corr ~ scale_score + paren_match, data=trainset), testset)
results_rf <- record_performance(results_rf, "score+len+paren_match", randomForest(corr ~ scale_len + scale_score + paren_match, data=trainset), testset)
results_rf <- record_performance(results_rf, "links", randomForest(corr ~ inlinks, data=trainset), testset)
results_rf <- record_performance(results_rf, "loglinks", randomForest(corr ~ log_links, data=trainset), testset)
results_rf <- record_performance(results_rf, "score+len+links+paren_match", randomForest(corr ~ scale_len + scale_score + log_links + paren_match, data=trainset), testset)
results_rf <- record_performance(results_rf, "score+links+paren_match", randomForest(corr ~ scale_len + scale_score + paren_match, data=trainset), testset)

results_rf
##                          model     score
## 1                          MFC 0.5640867
## 2                   body_score 0.6637771
## 3                  scale_score 0.6637771
## 4                      obs_len 0.5151703
## 5                    score+len 0.8074303
## 6                    paren+len 0.5795666
## 7                  paren_match 0.5795666
## 8            score+paren_match 0.7640867
## 9        score+len+paren_match 0.7690402
## 10                       links 0.6148607
## 11                    loglinks 0.6160991
## 12 score+len+links+paren_match 0.8043344
## 13     score+links+paren_match 0.7690402
# Accuracy using Random Forest classifier
## try1 <- corr ~ scale_len + scale_score + log_links + paren_match = 79% accuracy 
## try2 <- corr ~ scale_len + scale_score + log_links + paren_match+ obs_len = 77% accuracy
## try3 <- try 1 but with full dataset = 80% accuracy
final_test <- read.csv("C:/Users/yoges/Desktop/HW3 ENPM808W/qb.test.csv")

final_test$obs_len <- apply(final_test, 1, function(x) {nchar(x['text'])})

final_test$scale_len <- scale(final_test$obs_len)

final_test$scale_score <- scale(final_test$body_score)

final_test$paren_match <- apply(final_test, 1, function(x) {paren_match(x['page'], x['text'])})

final_test$log_links <- scale(log(as.numeric(final_test$inlinks) + 1))

rf_final <- randomForest(corr ~ scale_len + scale_score + log_links + paren_match, data=full)
rf_final_pred <- predict(rf_final,final_test)
submit_file <- as.data.frame(as.double(rf_final_pred))
#submit_file <- as.double(submit_file)
submit_file$row <- final_test$row
submit_file <- submit_file[c("row","as.double(rf_final_pred)")]
colnames(submit_file) <- c("row","corr")
head(submit_file)
##   row corr
## 1  31    1
## 2  32    2
## 3  33    2
## 4  88    2
## 5  91    2
## 6 139    1
write.csv(submit_file, file = "Yogesh_try3_rf.csv", row.names = FALSE)
library(ROCR)
library(pROC)
rf_try <- randomForest(corr ~ scale_len + scale_score + log_links + paren_match, data=trainset)
rf_try_predict <- predict(rf_try,testset,type = "prob")
rf_try_pred <- predict(rf_try,testset)
pred <- prediction(rf_try_predict[,2], testset$corr)
roc <- performance(pred,"tpr","fpr")
plot(roc,
     colorize = T,
     main = "ROC Curve",
     xlab = "1 - Specificity",
     ylab = "Sensitivity")
abline(a = 0,b= 1)

## sensitivity = True Positive Rate
## 1-Specificty = False Positive Rate
## Finding True positive , True negative , False positives and False Negatives for random forest classifier made
error_confusion_matrix <- (table(rf_try_pred,testset$corr))
error_analysis_accuracy <- sum(diag(error_confusion_matrix))/sum(error_confusion_matrix)
error_confusion_matrix
##            
## rf_try_pred False True
##       False   788  190
##       True    123  514
error_analysis_accuracy
## [1] 0.806192
error_analysis_misclassification_rate <- 1 - error_analysis_accuracy
error_analysis_misclassification_rate
## [1] 0.193808
Type_1_error_False_Positives <- error_confusion_matrix[2,1]
Type_2_error_False_Negatives <- error_confusion_matrix[1,2]
True_Positives <- error_confusion_matrix[2,2]
True_Negatives <- error_confusion_matrix[1,1]

error_essesntials <- c(Type_1_error_False_Positives,Type_2_error_False_Negatives,True_Positives,True_Negatives ) 
error_pdf <- as.data.frame(error_essesntials)
error_pdf$error_type <- c("Type_1_error_False_Positives","Type_2_error_False_Negatives","True_Positives","True_Negatives" )
colnames(error_pdf) <- c("Value","error_type") 

#error_plot <- ggplot(error_pdf, aes(x = error_type,y = Value , fill = error_type,color = error_type))+geom_bar(stat = "identity") + labs(title = "Error_plot")
#error_plot

plot(rf_try)

error_pdf$model <- "Random Forest"
error_pdf[nrow(error_pdf)+1,] <- c(conf_matrix_logistic_best[2,1],"Type_1_error_False_Positives","Logistic Regression")
error_pdf[nrow(error_pdf)+1,] <- c(conf_matrix_logistic_best[1,2],"Type_2_error_False_Negatives","Logistic Regression")
error_pdf[nrow(error_pdf)+1,] <- c(conf_matrix_logistic_best[2,2],"True_Positives","Logistic Regression")
error_pdf[nrow(error_pdf)+1,] <- c(conf_matrix_logistic_best[1,1],"True_Negatives","Logistic Regression")

error_pdf[nrow(error_pdf)+1,] <- c(conf_matrix_svm_best[2,1],"Type_1_error_False_Positives","SVM Radial")
error_pdf[nrow(error_pdf)+1,] <- c(conf_matrix_svm_best[1,2],"Type_2_error_False_Negatives","SVM Radial")
error_pdf[nrow(error_pdf)+1,] <- c(conf_matrix_svm_best[2,2],"True_Positives","SVM Radial")
error_pdf[nrow(error_pdf)+1,] <- c(conf_matrix_svm_best[1,1],"True_Negatives","SVM Radial")

error_pdf[nrow(error_pdf)+1,] <- c(conf_best_dt[2,1],"Type_1_error_False_Positives","Decision Tree")
error_pdf[nrow(error_pdf)+1,] <- c(conf_best_dt[1,2],"Type_2_error_False_Negatives","Decision Tree")
error_pdf[nrow(error_pdf)+1,] <- c(conf_best_dt[2,2],"True_Positives","Decision Tree")
error_pdf[nrow(error_pdf)+1,] <- c(conf_best_dt[1,1],"True_Negatives","Decision Tree")

error_pdf_models.plot <- ggplot(error_pdf,aes(x= error_type,y = Value, fill = error_type,color = error_type))+geom_bar(stat = "identity") + labs(title = "Error_plot")+ facet_grid(.~model)
error_pdf_models.plot