Question 5a

x1=runif(500)-0.5
x2=runif(500)-0.5
y=1*(x1^2-x2^2 > 0 )

Question 5b

plot(x1,x2,col=ifelse(y,'red','blue'))

## Question 5c

glm.fit=glm(y~. ,family='binomial', data=data.frame(x1,x2,y))
glm.fit
## 
## Call:  glm(formula = y ~ ., family = "binomial", data = data.frame(x1, 
##     x2, y))
## 
## Coefficients:
## (Intercept)           x1           x2  
##     0.02722     -0.20846     -0.36839  
## 
## Degrees of Freedom: 499 Total (i.e. Null);  497 Residual
## Null Deviance:       693 
## Residual Deviance: 691.1     AIC: 697.1

Question 5d

glm.pred=predict(glm.fit,data.frame(x1,x2))
plot(x1,x2,col=ifelse(glm.pred>0,'red','blue'),pch=ifelse(as.integer(glm.pred>0) == y,1,4))

Question 5e

glm.fit=glm(y~poly(x1,2)+poly(x2,2) ,family='binomial', data=data.frame(x1,x2,y))
## Warning: glm.fit: algorithm did not converge
## Warning: glm.fit: fitted probabilities numerically 0 or 1 occurred
summary(glm.fit)
## 
## Call:
## glm(formula = y ~ poly(x1, 2) + poly(x2, 2), family = "binomial", 
##     data = data.frame(x1, x2, y))
## 
## Deviance Residuals: 
##        Min          1Q      Median          3Q         Max  
## -2.348e-03  -2.000e-08   2.000e-08   2.000e-08   2.910e-03  
## 
## Coefficients:
##                Estimate Std. Error z value Pr(>|z|)
## (Intercept)       87.79   11487.89   0.008    0.994
## poly(x1, 2)1   -6513.11  198756.93  -0.033    0.974
## poly(x1, 2)2   77795.45 2954461.24   0.026    0.979
## poly(x2, 2)1    2156.21  305428.90   0.007    0.994
## poly(x2, 2)2  -76372.99 2770224.86  -0.028    0.978
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 6.9302e+02  on 499  degrees of freedom
## Residual deviance: 2.0439e-05  on 495  degrees of freedom
## AIC: 10
## 
## Number of Fisher Scoring iterations: 25

All the variables are not significant.

Question 5f

glm.pred=predict(glm.fit,data.frame(x1,x2))     # returns the log-odds.
plot(x1,x2,col=ifelse(glm.pred>0,'red','blue'),pch=ifelse(as.integer(glm.pred>0) == y,1,4))

Question 5g

svm.fit=svm(y~.,data=data.frame(x1,x2,y=as.factor(y)),kernel='linear')
svm.pred=predict(svm.fit,data.frame(x1,x2),type='response')
plot(x1,x2,col=ifelse(svm.pred!=0,'red','blue'),pch=ifelse(svm.pred == y,1,4))

Question 5h

svm.fit=svm(y~.,data=data.frame(x1,x2,y=as.factor(y)),kernel='polynomial',degree=2)
svm.pred=predict(svm.fit,data.frame(x1,x2),type='response')
plot(x1,x2,col=ifelse(svm.pred!=0,'red','blue'),pch=ifelse(svm.pred == y,1,4))

## Question 5i

Logit model outperforms the SVM model by far.

Question 7a

Auto$mpg=ifelse(Auto$mpg>median(Auto$mpg),1,0)
table(Auto$mpg)
## 
##   0   1 
## 196 196

Question 7b

costs=data.frame(cost=seq(0.05,100,length.out = 15))
svm.tune=tune(svm,mpg~.,data=Auto,ranges=costs,kernel='linear')
svm.tune
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##      cost
##  7.189286
## 
## - best performance: 0.1000433
plot(svm.tune$performance[,c(1,2)],type='l')

plot(svm.tune$performance[,c(1,2)],type='l')

The plot and parameter tunning both suggest the best parameter is cost 0.05

Question 7c

params=data.frame(cost=seq(0.05,100,length.out = 5),degree=seq(1,100,length.out = 5))
svm.poly=tune(svm,mpg~.,data=Auto,ranges=params,kernel='polynomial')
svm.poly
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost degree
##   100      1
## 
## - best performance: 0.09731409
params=data.frame(cost=seq(0.05,100,length.out = 5),gamma=seq(0.1,100,length.out = 5))
svm.radial=tune(svm,mpg~.,data=Auto,ranges=params,kernel='radial')
svm.radial
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##     cost gamma
##  25.0375   0.1
## 
## - best performance: 0.07254304

Question 7d

The best performance is of the polynomial.

Question 8a

set.seed(42)
train=sample(1:1070,800)
test=(1:1070)[-train]

tb=c()
res=c()

Question 8b

svm.fit=svm(Purchase~.,data=OJ,subset=train,cost=0.01,kernel='linear')
summary(svm.fit)
## 
## Call:
## svm(formula = Purchase ~ ., data = OJ, cost = 0.01, kernel = "linear", 
##     subset = train)
## 
## 
## Parameters:
##    SVM-Type:  C-classification 
##  SVM-Kernel:  linear 
##        cost:  0.01 
## 
## Number of Support Vectors:  432
## 
##  ( 215 217 )
## 
## 
## Number of Classes:  2 
## 
## Levels: 
##  CH MM

432 observations used in the support vectors with almost identical split with the classes (215 and 217)

Question 8c

svm.pred=predict(svm.fit,OJ[train,])
kable(table(OJ[train,'Purchase'],svm.pred))
CH MM
CH 432 60
MM 77 231 
mean(OJ$Purchase[train] != svm.pred)
## [1] 0.17125
res=cbind(res,'train'=mean(OJ$Purchase[train] != svm.pred))
svm.pred=predict(svm.fit,OJ[test,])
kable(table(OJ[test,'Purchase'],svm.pred))
CH MM
CH 142 19
MM 25 84 
mean(OJ$Purchase[test] != svm.pred)
## [1] 0.162963
res=cbind(res,'test'=mean(OJ$Purchase[test] != svm.pred))

Training error = 0.17125 Test error = 0.162963

Question 8d

svm.tune=tune(svm,Purchase~.,data=OJ[train,],ranges=data.frame(cost=seq(0.01,10,25)),kernel='linear')
summary(svm.tune)
## 
## Error estimation of 'svm' using 10-fold cross validation: 0.1825
res=cbind(res,'CV'=svm.tune$best.performance)

Question 8e

svm.pred=predict(svm.tune$best.model,OJ[train,])
kable(table(OJ[train,'Purchase'],svm.pred))
CH MM
CH 432 60
MM 77 231 
mean(OJ$Purchase[train] != svm.pred)
## [1] 0.17125
res=cbind(res,'train.tuned'=mean(OJ$Purchase[train] != svm.pred))
svm.pred=predict(svm.tune$best.model,OJ[test,])
kable(table(OJ[test,'Purchase'],svm.pred))
CH MM
CH 142 19
MM 25 84 
mean(OJ$Purchase[test] != svm.pred)
## [1] 0.162963

Training = 0.17125 Testing = 0.162963

Question 8f

svm.fit=svm(Purchase~.,data=OJ,subset=train,cost=0.01,kernel='radial')
summary(svm.fit)
## 
## Call:
## svm(formula = Purchase ~ ., data = OJ, cost = 0.01, kernel = "radial", 
##     subset = train)
## 
## 
## Parameters:
##    SVM-Type:  C-classification 
##  SVM-Kernel:  radial 
##        cost:  0.01 
## 
## Number of Support Vectors:  621
## 
##  ( 308 313 )
## 
## 
## Number of Classes:  2 
## 
## Levels: 
##  CH MM
svm.pred=predict(svm.fit,OJ[train,])
kable(table(OJ[train,'Purchase'],svm.pred))
CH MM
CH 492 0
MM 308 0 
mean(OJ$Purchase[train] != svm.pred)
## [1] 0.385
res=cbind(res,'train'=mean(OJ$Purchase[train] != svm.pred))
svm.pred=predict(svm.fit,OJ[test,])
kable(table(OJ[test,'Purchase'],svm.pred))
CH MM
CH 161 0
MM 109 0 
mean(OJ$Purchase[test] != svm.pred)
## [1] 0.4037037
res=cbind(res,'train'=mean(OJ$Purchase[test] != svm.pred))
svm.tune=tune(svm,Purchase~.,data=OJ[train,],ranges=data.frame(cost=seq(0.01,10,25)))
summary(svm.tune)
## 
## Error estimation of 'svm' using 10-fold cross validation: 0.385
res=cbind(res,'CV'=svm.tune$best.performance)
svm.pred=predict(svm.tune$best.model,OJ[train,])
kable(table(OJ[train,'Purchase'],svm.pred))
CH MM
CH 492 0
MM 308 0 
mean(OJ$Purchase[train] != svm.pred)
## [1] 0.385
res=cbind(res,'train.tuned'=mean(OJ$Purchase[train] != svm.pred))
svm.pred=predict(svm.tune$best.model,OJ[test,])
kable(table(OJ[test,'Purchase'],svm.pred))
CH MM
CH 161 0
MM 109 0 
mean(OJ$Purchase[test] != svm.pred)
## [1] 0.4037037
res=cbind(res,'test.tuned'=mean(OJ$Purchase[test] != svm.pred))
tb=rbind(tb,res)
res=c()

Question 8g

svm.fit=svm(Purchase~.,data=OJ,subset=train,cost=0.01,kernel='polynomial')
summary(svm.fit)
## 
## Call:
## svm(formula = Purchase ~ ., data = OJ, cost = 0.01, kernel = "polynomial", 
##     subset = train)
## 
## 
## Parameters:
##    SVM-Type:  C-classification 
##  SVM-Kernel:  polynomial 
##        cost:  0.01 
##      degree:  3 
##      coef.0:  0 
## 
## Number of Support Vectors:  614
## 
##  ( 303 311 )
## 
## 
## Number of Classes:  2 
## 
## Levels: 
##  CH MM
svm.pred=predict(svm.fit,OJ[train,])
kable(table(OJ[train,'Purchase'],svm.pred))
CH MM
CH 486 6
MM 287 21 
mean(OJ$Purchase[train] != svm.pred)
## [1] 0.36625
res=cbind(res,'train'=mean(OJ$Purchase[train] != svm.pred))
svm.pred=predict(svm.fit,OJ[test,])
kable(table(OJ[test,'Purchase'],svm.pred))
CH MM
CH 160 1
MM 101 8 
mean(OJ$Purchase[test] != svm.pred)
## [1] 0.3777778
res=cbind(res,'test'=mean(OJ$Purchase[test] != svm.pred))
svm.tune=tune(svm,Purchase~.,data=OJ[train,],ranges=data.frame(cost=seq(0.01,10,25)),kernel='polynomial')
summary(svm.tune)
## 
## Error estimation of 'svm' using 10-fold cross validation: 0.36625
res=cbind(res,'CV'=svm.tune$best.performance)
svm.pred=predict(svm.tune$best.model,OJ[train,])
kable(table(OJ[train,'Purchase'],svm.pred))
CH MM
CH 486 6
MM 287 21 
mean(OJ$Purchase[train] != svm.pred)
## [1] 0.36625
res=cbind(res,'train.tuned'=mean(OJ$Purchase[train] != svm.pred))
svm.pred=predict(svm.tune$best.model,OJ[test,])
kable(table(OJ[test,'Purchase'],svm.pred))
CH MM
CH 160 1
MM 101 8 
mean(OJ$Purchase[test] != svm.pred)
## [1] 0.3777778
res=cbind(res,'test.tuned'=mean(OJ$Purchase[test] != svm.pred))

Question 8h

Linear SVM is the best.