Data Mining Final Project
Michal Greenwood, Ashley Mata, Andrew Padro, John Thomas and Manuel Valles
May 13th, 2019
Getting and Fixing Data Set
Libraries
Looking at the Votes Across Parties
Cross Validated Logistic Regression
K-fold LDA
Random Forest Across Different mtry, all 16 Variables
rf.v<- list()
yhat.rf<-list()
rf.error<-list()
for ( i in 1:16 ) {
set.seed(24)
rf.v[[i]]<-randomForest(V1 ~ ., data = v.train, mtry = i, importance = TRUE)
yhat.rf[[i]]<-predict(rf.v[[i]], newdata = v.test)
rf.error[[i]]<-caret::confusionMatrix(yhat.rf[[i]], v.test$V1)$overall[1]
}
rf.airz<-data.frame(accuracy=unlist(rf.error))
rownames(rf.airz)<-paste0("mtry=",1:16)
par(mfrow=c(1,1))
matplot(1:16, rf.error, xlab = 'mtry', ylab = 'Accuracy', main = "What # of predictors sampled at each split \n gives the lowest error?")
lines(1:16, rf.error, type = "o")
Closer Look at Most Important Variables, Across Best mtrys, 2:4,8
Do randomForest across all 16 possible mtrys, using best n (1:16) predictors, WITH 10 different seeds
Organizing the Best rf models
and our best Random Forrest goes to:
Now with 10 different seeds, these are the best n seeds of LDA across different predictors.
npreds_lda<-
function(x){
errors<-list()
for( seed in 1:10){
set.seed(seed)
tr<-sample(nrow(v),nrow(v)*.8)
v.tr<-v[tr,]
v.te<-v[-tr,]
errors[[seed]]<-1-confusionMatrix(predict(lda(V1~., data = v.tr[,c("V1",best_n[[x]])] , cv = FALSE), v.te, type='response')$class, v.te$V1)$overall[1]
}
unlist(errors)
}
best_npreds_lda<-list()
for( i in 1:16){
best_npreds_lda[[i]]<-mean(npreds_lda(i))
}
matplot(1:16, best_npreds_lda, xlab = "best n predictors", ylab ="error", main = 'Best n Predictors for LDA' , pch = 19)
lines(1:16, best_npreds_lda)
abline(h = min(unlist(best_npreds_lda))+.2*sd(unlist(best_npreds_lda)))
min(unlist(best_npreds_lda))
[1] 0.03678161
data.frame(n_preds = which(as.matrix(best_npreds_lda)== min(unlist(best_npreds_lda)), arr.ind = TRUE)[,1], error = min(unlist(best_npreds_lda)) )
n_preds error
row 3 0.03678161
Now let's attempt Gradient Boosting
v.boost<-gbm(V1~., data = v.train, distribution = 'gaussian',n.trees=5000,interaction.depth=4)
yhat.boost=predict(v.boost ,newdata =v.test, n.trees=5000)
v.boostn<-function(x) {
as.numeric(caret::confusionMatrix(as.factor(ifelse(yhat.boost> x , 1, 0)), v.test$V1)[[3]][1])
}
cutoffs<-seq(1,2,.05)
bbb<-as.data.frame(cutoffs)
bbb$accuracy<-apply(bbb,1,v.boostn)
bbb
cutoffs accuracy
1 1.00 0.6436782
2 1.05 0.8160920
3 1.10 0.9080460
4 1.15 0.9310345
5 1.20 0.9310345
6 1.25 0.9310345
7 1.30 0.9425287
8 1.35 0.9425287
9 1.40 0.9425287
10 1.45 0.9425287
11 1.50 0.9080460
12 1.55 0.9080460
13 1.60 0.9080460
14 1.65 0.9080460
15 1.70 0.9080460
16 1.75 0.9080460
17 1.80 0.9080460
18 1.85 0.8735632
19 1.90 0.8735632
20 1.95 0.8505747
21 2.00 0.7471264
plot(bbb)
lines(bbb[,1],bbb[,2])
best_gbm<-data.frame(best_n=1:16, error = rbind(mean(npreds_gbm(1)), mean(npreds_gbm(2)), mean(npreds_gbm(3)), mean(npreds_gbm(4)), mean(npreds_gbm(5)), mean(npreds_gbm(6)), mean(npreds_gbm(7)), mean(npreds_gbm(8)), mean(npreds_gbm(9)), mean(npreds_gbm(10)), mean(npreds_gbm(11)), mean(npreds_gbm(12)), mean(npreds_gbm(13)), mean(npreds_gbm(14)), mean(npreds_gbm(15)), mean(npreds_gbm(16))))
matplot(best_gbm$best_n, best_gbm$error, xlab = "best n predictors", ylab ="error", main = 'Best n Predictors for GBM' , pch = 19)
lines(best_gbm$best_n, best_gbm$error)
abline(h = min(unlist( best_gbm[2]))+.2*sd(unlist( best_gbm[2])))
best_gbm[which(best_gbm$error== min(best_gbm$error), arr.ind = TRUE),]
best_n error
2 2 0.04597701
3 3 0.04597701
Our Best SVM is as follows:
best_svm<-data.frame( n_var = 1:16,
linear = unlist(best_npreds_svm_lin),
radial = unlist(best_npreds_svm_rad),
polynomial = unlist(best_npreds_svm_pol)
)
matplot(as.matrix(best_svm)[,2:4], type = c('b'), pch = 20, col = c(1,2,4), xlab='best n variables', ylab = 'error', main = "Best SVM" )
abline(h=min(as.matrix(best_svm)[,2:4]) + .2*sd(as.matrix(best_svm)[,2:4]))
best_svm_model= data.frame(which(as.matrix(best_svm)==min(as.matrix(best_svm)), arr.ind = TRUE), best_svm[which(as.matrix(best_svm)==min(as.matrix(best_svm)), arr.ind = TRUE)])
names(best_svm_model)<-c("best_n", "model","error")
best_svm_model[1,2]<-names(best_svm)[best_svm_model[1,2]]
best_svm_model
best_n model error
1 7 radial 0.03448276