Classification analysis on German Credit Risk Data
Avinash Vashishtha
June 25, 2019
1.Introduction
1.Objective
The objective of this exercise is to apply various classification models to categorical response variable and compare their model results
2.German Credit Risk Data Introduction
The original dataset contains 1000 entries with 20 categorial/symbolic attributes prepared by Prof. Hofmann. In this dataset, each entry represents a person who takes a credit by a bank. Each person is classified as good or bad credit risks according to the set of attributes. The link to the original dataset can be found below.
set.seed(2019)
library(ROCR) #Creating ROC curve
library(PRROC) #Precision-recall curve
library(glmnet) #Lasso
library(tidyverse)
library(DT)
library(glmnet)
library(rpart)
library(rpart.plot)
library(caret)
library(knitr)
library(mgcv)
library(nnet)
library(NeuralNetTools)## Warning: package 'NeuralNetTools' was built under R version 3.5.3library(e1071)
library(verification)
#library(leaps) #best subset
#library(glmnet) #lasso
#library(rpart) #Regression Tree
#library(rpart.plot)
#library(mgcv) # GAM
#library(ipred) #Bagging
#library(randomForest) #Random Forest
#library(gbm) #Boosting
#library(neuralnet) #Neural Networkgerman.data <- read.table("http://archive.ics.uci.edu/ml/machine-learning-databases/statlog/german/german.data")
head(german.data)## V1 V2 V3 V4 V5 V6 V7 V8 V9 V10 V11 V12 V13 V14 V15 V16 V17
## 1 A11 6 A34 A43 1169 A65 A75 4 A93 A101 4 A121 67 A143 A152 2 A173
## 2 A12 48 A32 A43 5951 A61 A73 2 A92 A101 2 A121 22 A143 A152 1 A173
## 3 A14 12 A34 A46 2096 A61 A74 2 A93 A101 3 A121 49 A143 A152 1 A172
## 4 A11 42 A32 A42 7882 A61 A74 2 A93 A103 4 A122 45 A143 A153 1 A173
## 5 A11 24 A33 A40 4870 A61 A73 3 A93 A101 4 A124 53 A143 A153 2 A173
## 6 A14 36 A32 A46 9055 A65 A73 2 A93 A101 4 A124 35 A143 A153 1 A172
## V18 V19 V20 V21
## 1 1 A192 A201 1
## 2 1 A191 A201 2
## 3 2 A191 A201 1
## 4 2 A191 A201 1
## 5 2 A191 A201 2
## 6 2 A192 A201 1dim(german.data)## [1] 1000 21colnames(german.data) <- c("chk_acct", "duration", "credit_his", "purpose",
"amount", "saving_acct", "present_emp", "installment_rate", "sex", "other_debtor",
"present_resid", "property", "age", "other_install", "housing", "n_credits",
"job", "n_people", "telephone", "foreign", "response")
#orginal response coding 1= good, 2 = bad
#we need 0 = good, 1 = bad
german.data$response <- german.data$response - 1
german.data$response <- as.factor(german.data$response)3.Preparation of Dataset
3.1 Splitting the data to train and test dataset
set.seed(12871014)
trainrows <- sample(nrow(german.data), nrow(german.data) * 0.75)
germandata.train <- german.data[trainrows, ]
germandata.test <- german.data[-trainrows,]2.Logistic Regression
2.1.Running Logistic Regression on all variables
germandata.train.glm0 <- glm(response~., family = binomial, germandata.train)
summary(germandata.train.glm0)##
## Call:
## glm(formula = response ~ ., family = binomial, data = germandata.train)
##
## Deviance Residuals:
## Min 1Q Median 3Q Max
## -2.5894 -0.6851 -0.4018 0.6628 2.5169
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 6.805e-01 1.196e+00 0.569 0.569493
## chk_acctA12 -3.802e-01 2.552e-01 -1.490 0.136321
## chk_acctA13 -1.234e+00 4.299e-01 -2.871 0.004095 **
## chk_acctA14 -1.652e+00 2.713e-01 -6.088 1.15e-09 ***
## duration 2.458e-02 1.051e-02 2.338 0.019402 *
## credit_hisA31 6.555e-01 6.261e-01 1.047 0.295126
## credit_hisA32 -4.092e-01 4.822e-01 -0.849 0.396055
## credit_hisA33 -7.157e-01 5.239e-01 -1.366 0.171877
## credit_hisA34 -1.255e+00 4.907e-01 -2.558 0.010513 *
## purposeA41 -1.550e+00 4.235e-01 -3.660 0.000253 ***
## purposeA410 -1.620e+00 7.862e-01 -2.061 0.039290 *
## purposeA42 -7.977e-01 3.096e-01 -2.576 0.009990 **
## purposeA43 -7.996e-01 2.838e-01 -2.818 0.004836 **
## purposeA44 -5.545e-01 9.876e-01 -0.561 0.574463
## purposeA45 -1.257e-01 6.035e-01 -0.208 0.834982
## purposeA46 4.374e-02 4.690e-01 0.093 0.925687
## purposeA48 -1.915e+00 1.265e+00 -1.514 0.130101
## purposeA49 -6.075e-01 3.816e-01 -1.592 0.111352
## amount 1.053e-04 5.094e-05 2.066 0.038815 *
## saving_acctA62 -7.859e-01 3.661e-01 -2.147 0.031831 *
## saving_acctA63 -3.296e-01 4.357e-01 -0.757 0.449317
## saving_acctA64 -9.624e-01 5.562e-01 -1.730 0.083569 .
## saving_acctA65 -8.169e-01 2.977e-01 -2.744 0.006070 **
## present_empA72 1.594e-01 4.823e-01 0.330 0.741057
## present_empA73 1.121e-02 4.547e-01 0.025 0.980340
## present_empA74 -7.015e-01 4.979e-01 -1.409 0.158835
## present_empA75 -3.260e-01 4.609e-01 -0.707 0.479442
## installment_rate 2.777e-01 1.041e-01 2.667 0.007659 **
## sexA92 -2.200e-01 4.436e-01 -0.496 0.619997
## sexA93 -5.738e-01 4.372e-01 -1.313 0.189349
## sexA94 -2.376e-01 5.298e-01 -0.448 0.653845
## other_debtorA102 7.650e-01 4.772e-01 1.603 0.108888
## other_debtorA103 -7.653e-01 4.615e-01 -1.658 0.097286 .
## present_resid 6.789e-03 1.006e-01 0.067 0.946190
## propertyA122 1.923e-01 2.951e-01 0.652 0.514644
## propertyA123 1.600e-01 2.778e-01 0.576 0.564564
## propertyA124 7.342e-01 4.806e-01 1.528 0.126602
## age -2.539e-03 1.055e-02 -0.241 0.809834
## other_installA142 6.955e-03 4.920e-01 0.014 0.988720
## other_installA143 -6.925e-01 2.771e-01 -2.499 0.012460 *
## housingA152 -4.346e-01 2.688e-01 -1.617 0.105924
## housingA153 -5.221e-01 5.316e-01 -0.982 0.326033
## n_credits 1.456e-01 2.242e-01 0.649 0.516089
## jobA172 -3.963e-01 7.354e-01 -0.539 0.590007
## jobA173 -1.646e-01 7.051e-01 -0.233 0.815451
## jobA174 -1.396e-01 7.171e-01 -0.195 0.845686
## n_people 2.456e-01 2.913e-01 0.843 0.399207
## telephoneA192 -2.916e-01 2.343e-01 -1.244 0.213373
## foreignA202 -1.261e+00 6.276e-01 -2.008 0.044603 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 902.33 on 749 degrees of freedom
## Residual deviance: 676.61 on 701 degrees of freedom
## AIC: 774.61
##
## Number of Fisher Scoring iterations: 52.2.Model diagnostic
germandata.train.glm0$deviance## [1] 676.6139AIC(germandata.train.glm0)## [1] 774.6139BIC(germandata.train.glm0)## [1] 1000.9972.3.In-Sample Prediction
#Response variable split
summary(germandata.train$response)## 0 1
## 533 217#Summary of predicted values
germandata.train.pred<-predict(germandata.train.glm0,type="response")
summary(germandata.train.pred)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 0.006941 0.089434 0.205306 0.289333 0.450436 0.967894hist(predict(germandata.train.glm0,type="response"))
predict<-ifelse(predict(germandata.train.glm0,type="response")>0.4,1,0)
Misclassification_rate<-mean(predict!= germandata.train$response)
Misclassification_rate## [1] 0.2213333FPR<- sum(germandata.train$response==0 & predict==1)/sum(germandata.train$response==0)
FNR<- sum(germandata.train$response==1 & predict==0)/sum(germandata.train$response==1)2.3.1.Finding optimal Threshold FPR=TPR
2.3.1.1.FPR=TPR
step<-seq(0,1,by=0.001)
y<-0
diff<-0
count<-0
for(i in step)
{
predict<-ifelse(predict(germandata.train.glm0,type="response")>i,1,0)
FPR<- sum(germandata.train$response==0 & predict==1)/sum(germandata.train$response==0)
FNR<- sum(germandata.train$response==1 & predict==0)/sum(germandata.train$response==1)
y[count]<-i
diff[count]<-ifelse(FPR-FNR<0,FNR-FPR,FPR-FNR)
count=count+1
}
indices<-min(diff)==diff
Threshold<-y[indices]
predict<-ifelse(predict(germandata.train.glm0,type="response")>Threshold,1,0)
Misclassification_rate<-mean(predict!= germandata.train$response)
FPR<- sum(germandata.train$response==0 & predict==1)/sum(germandata.train$response==0)
FNR<- sum(germandata.train$response==1 & predict==0)/sum(germandata.train$response==1)
Misclassification_rate## [1] 0.2453333FPR## [1] 0.2457786FNR## [1] 0.24423962.3.1.2.Naive Choice of Cut-off probability
summary(germandata.train$response)## 0 1
## 533 217pcut1<- mean(as.numeric(as.character(germandata.train$response)))
pcut1## [1] 0.2893333# get binary prediction
class.glm0.train<- (as.numeric(germandata.train.pred)>pcut1)*1
summary(class.glm0.train)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 0.000 0.000 0.000 0.412 1.000 1.000# get confusion matrix
table(as.numeric(germandata.train$response), class.glm0.train, dnn = c("True", "Predicted"))## Predicted
## True 0 1
## 1 394 139
## 2 47 1702.3.1.3.Grid search using Asymmetric cost function
# define a cost function with input "obs" being observed response
# and "pi" being predicted probability, and "pcut" being the threshold.
costfunc = function(obs, pred.p, pcut){
weight1 = 5 # define the weight for "true=1 but pred=0" (FN)
weight0 = 1 # define the weight for "true=0 but pred=1" (FP)
c1 = (obs==1)&(pred.p<pcut) # count for "true=1 but pred=0" (FN)
c0 = (obs==0)&(pred.p>=pcut) # count for "true=0 but pred=1" (FP)
cost = mean(weight1*c1 + weight0*c0) # misclassification with weight
return(cost) # you have to return to a value when you write R functions
} # end of the function
# define a sequence from 0.01 to 1 by 0.01
p.seq = seq(0.01, 1, 0.01)
# write a loop for all p-cut to see which one provides the smallest cost
# first, need to define a 0 vector in order to save the value of cost from all pcut
cost = rep(0, length(p.seq))
for(i in 1:length(p.seq)){
cost[i] = costfunc(obs = germandata.train$response, pred.p = as.numeric(germandata.train.pred), pcut = p.seq[i])
} # end of the loop
optimal.pcut = p.seq[which(cost==min(cost))][1]
pcut<-optimal.pcut
optimal.pcut.asymmetric<-optimal.pcut
optimal.pcut## [1] 0.19Plotting the misclassfication rate vs range of probability cutoffs
plot(p.seq, cost)
Defining cost function
# Asymmetric Misclassification Rate, using 5:1 asymmetric cost
# r - actual response
# pi - predicted response
cost <- function(r, pi){
weight1 = 5
weight0 = 1
c1 = (r==1)&(pi==0) #logical vector - true if actual 1 but predict 0
c0 = (r==0)&(pi==1) #logical vector - true if actual 0 but predict 1
return(mean(weight1*c1+weight0*c0))
}
# pcut <- 1/6 ## Bayes estimate
pcut <- optimal.pcut.asymmetricROC curve
pred <- prediction(germandata.train.pred, germandata.train$response)
perf <- performance(pred, "tpr", "fpr")
plot(perf, colorize=TRUE)
Finding area under the curve to understand the prediction power of the model
#Get the AUC
unlist(slot(performance(pred, "auc"), "y.values"))## [1] 0.8264843Precision Recall curve - another way to find out AUC
score1= germandata.train.pred[germandata.train$response==1]
score0= germandata.train.pred[germandata.train$response==0]
roc= roc.curve(score1, score0, curve = T)
roc$auc## [1] 0.8264843Precision recall curve diagnostics
pr= pr.curve(score1, score0, curve = T)
pr##
## Precision-recall curve
##
## Area under curve (Integral):
## 0.6652113
##
## Area under curve (Davis & Goadrich):
## 0.6651141
##
## Curve for scores from 0.006941132 to 0.9678944
## ( can be plotted with plot(x) )Plotting PR curve
plot(pr)
Out of Sample Prediction
pred.glm0.test<- predict(germandata.train.glm0, newdata = germandata.test, type="response")3.Stepwise
credit.glm.back <- step(germandata.train.glm0) # backward selection (if you don't specify anything)## Start: AIC=774.61
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + sex + other_debtor +
## present_resid + property + age + other_install + housing +
## n_credits + job + n_people + telephone + foreign
##
## Df Deviance AIC
## - job 3 677.45 769.45
## - property 3 678.94 770.94
## - sex 3 679.74 771.74
## - present_resid 1 676.62 772.62
## - age 1 676.67 772.67
## - n_credits 1 677.03 773.03
## - n_people 1 677.32 773.32
## - housing 2 679.42 773.42
## - present_emp 4 684.17 774.17
## - telephone 1 678.18 774.18
## <none> 676.61 774.61
## - other_debtor 2 682.66 776.66
## - amount 1 680.90 776.90
## - foreign 1 681.38 777.38
## - duration 1 682.08 778.08
## - other_install 2 684.17 778.17
## - saving_acct 4 689.33 779.33
## - installment_rate 1 683.90 779.90
## - purpose 9 701.00 781.00
## - credit_his 4 694.69 784.69
## - chk_acct 3 723.20 815.20
##
## Step: AIC=769.45
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + sex + other_debtor +
## present_resid + property + age + other_install + housing +
## n_credits + n_people + telephone + foreign
##
## Df Deviance AIC
## - property 3 680.10 766.10
## - sex 3 680.78 766.78
## - present_resid 1 677.45 767.45
## - age 1 677.54 767.54
## - n_credits 1 677.94 767.94
## - n_people 1 677.98 767.98
## - housing 2 680.44 768.44
## - telephone 1 678.80 768.80
## - present_emp 4 685.08 769.08
## <none> 677.45 769.45
## - other_debtor 2 683.59 771.59
## - amount 1 682.15 772.15
## - foreign 1 682.49 772.49
## - other_install 2 684.82 772.82
## - duration 1 683.28 773.28
## - saving_acct 4 690.13 774.13
## - installment_rate 1 685.11 775.11
## - purpose 9 702.17 776.17
## - credit_his 4 695.76 779.76
## - chk_acct 3 723.75 809.75
##
## Step: AIC=766.1
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + sex + other_debtor +
## present_resid + age + other_install + housing + n_credits +
## n_people + telephone + foreign
##
## Df Deviance AIC
## - sex 3 683.08 763.08
## - present_resid 1 680.10 764.10
## - age 1 680.25 764.25
## - n_credits 1 680.52 764.52
## - n_people 1 680.54 764.54
## - telephone 1 681.07 765.07
## - present_emp 4 687.77 765.77
## - housing 2 684.03 766.03
## <none> 680.10 766.10
## - foreign 1 685.12 769.12
## - other_debtor 2 687.15 769.15
## - amount 1 685.66 769.66
## - saving_acct 4 692.26 770.26
## - other_install 2 688.36 770.36
## - duration 1 686.43 770.43
## - installment_rate 1 688.29 772.29
## - purpose 9 706.22 774.22
## - credit_his 4 698.63 776.63
## - chk_acct 3 727.81 807.81
##
## Step: AIC=763.08
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + other_debtor +
## present_resid + age + other_install + housing + n_credits +
## n_people + telephone + foreign
##
## Df Deviance AIC
## - present_resid 1 683.08 761.08
## - n_people 1 683.12 761.12
## - age 1 683.17 761.17
## - n_credits 1 683.51 761.51
## - telephone 1 684.03 762.03
## <none> 683.08 763.08
## - housing 2 687.47 763.47
## - present_emp 4 693.05 765.05
## - amount 1 687.83 765.83
## - other_debtor 2 689.86 765.86
## - foreign 1 688.36 766.36
## - other_install 2 690.55 766.55
## - saving_acct 4 695.33 767.33
## - installment_rate 1 689.87 767.87
## - duration 1 689.92 767.92
## - purpose 9 708.82 770.82
## - credit_his 4 702.51 774.51
## - chk_acct 3 731.34 805.34
##
## Step: AIC=761.08
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + other_debtor +
## age + other_install + housing + n_credits + n_people + telephone +
## foreign
##
## Df Deviance AIC
## - n_people 1 683.12 759.12
## - age 1 683.17 759.17
## - n_credits 1 683.52 759.52
## - telephone 1 684.03 760.03
## <none> 683.08 761.08
## - housing 2 687.90 761.90
## - present_emp 4 693.25 763.25
## - amount 1 687.83 763.83
## - other_debtor 2 689.86 763.86
## - foreign 1 688.37 764.37
## - other_install 2 690.55 764.55
## - saving_acct 4 695.36 765.36
## - installment_rate 1 689.87 765.87
## - duration 1 689.95 765.95
## - purpose 9 708.83 768.83
## - credit_his 4 702.51 772.51
## - chk_acct 3 731.51 803.51
##
## Step: AIC=759.12
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + other_debtor +
## age + other_install + housing + n_credits + telephone + foreign
##
## Df Deviance AIC
## - age 1 683.21 757.21
## - n_credits 1 683.57 757.57
## - telephone 1 684.09 758.09
## <none> 683.12 759.12
## - housing 2 687.97 759.97
## - present_emp 4 693.29 761.29
## - other_debtor 2 689.87 761.87
## - amount 1 687.89 761.89
## - foreign 1 688.40 762.40
## - other_install 2 690.60 762.60
## - saving_acct 4 695.37 763.37
## - installment_rate 1 689.87 763.87
## - duration 1 689.95 763.95
## - purpose 9 709.05 767.05
## - credit_his 4 702.63 770.63
## - chk_acct 3 731.60 801.60
##
## Step: AIC=757.21
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + other_debtor +
## other_install + housing + n_credits + telephone + foreign
##
## Df Deviance AIC
## - n_credits 1 683.65 755.65
## - telephone 1 684.26 756.26
## <none> 683.21 757.21
## - housing 2 688.17 758.17
## - present_emp 4 693.93 759.93
## - amount 1 687.95 759.95
## - other_debtor 2 690.00 760.00
## - foreign 1 688.52 760.52
## - other_install 2 690.63 760.63
## - saving_acct 4 695.47 761.47
## - installment_rate 1 689.94 761.94
## - duration 1 690.22 762.22
## - purpose 9 709.07 765.07
## - credit_his 4 702.83 768.83
## - chk_acct 3 731.76 799.76
##
## Step: AIC=755.65
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + other_debtor +
## other_install + housing + telephone + foreign
##
## Df Deviance AIC
## - telephone 1 684.68 754.68
## <none> 683.65 755.65
## - housing 2 688.72 756.72
## - present_emp 4 694.25 758.25
## - amount 1 688.40 758.40
## - other_debtor 2 690.56 758.56
## - foreign 1 689.20 759.20
## - other_install 2 691.40 759.40
## - saving_acct 4 695.87 759.87
## - installment_rate 1 690.24 760.24
## - duration 1 690.49 760.49
## - purpose 9 709.62 763.62
## - credit_his 4 704.34 768.34
## - chk_acct 3 732.05 798.05
##
## Step: AIC=754.68
## response ~ chk_acct + duration + credit_his + purpose + amount +
## saving_acct + present_emp + installment_rate + other_debtor +
## other_install + housing + foreign
##
## Df Deviance AIC
## <none> 684.68 754.68
## - housing 2 689.64 755.64
## - amount 1 688.65 756.65
## - other_debtor 2 691.56 757.56
## - present_emp 4 695.71 757.71
## - foreign 1 689.96 757.96
## - other_install 2 692.41 758.41
## - saving_acct 4 696.81 758.81
## - installment_rate 1 690.88 758.88
## - duration 1 692.28 760.28
## - purpose 9 711.17 763.17
## - credit_his 4 705.80 767.80
## - chk_acct 3 733.98 797.98summary(credit.glm.back)##
## Call:
## glm(formula = response ~ chk_acct + duration + credit_his + purpose +
## amount + saving_acct + present_emp + installment_rate + other_debtor +
## other_install + housing + foreign, family = binomial, data = germandata.train)
##
## Deviance Residuals:
## Min 1Q Median 3Q Max
## -2.7087 -0.6833 -0.4110 0.6952 2.5669
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 8.628e-01 7.242e-01 1.191 0.23350
## chk_acctA12 -3.968e-01 2.481e-01 -1.600 0.10966
## chk_acctA13 -1.232e+00 4.175e-01 -2.951 0.00316 **
## chk_acctA14 -1.670e+00 2.656e-01 -6.289 3.2e-10 ***
## duration 2.771e-02 1.005e-02 2.757 0.00583 **
## credit_hisA31 5.908e-01 5.963e-01 0.991 0.32182
## credit_hisA32 -5.404e-01 4.531e-01 -1.193 0.23301
## credit_hisA33 -8.349e-01 5.151e-01 -1.621 0.10507
## credit_hisA34 -1.273e+00 4.822e-01 -2.641 0.00828 **
## purposeA41 -1.525e+00 4.128e-01 -3.695 0.00022 ***
## purposeA410 -1.677e+00 7.537e-01 -2.225 0.02608 *
## purposeA42 -7.376e-01 2.980e-01 -2.476 0.01330 *
## purposeA43 -8.027e-01 2.765e-01 -2.903 0.00369 **
## purposeA44 -7.163e-01 9.803e-01 -0.731 0.46497
## purposeA45 -4.519e-02 5.934e-01 -0.076 0.93930
## purposeA46 1.327e-01 4.592e-01 0.289 0.77268
## purposeA48 -1.908e+00 1.233e+00 -1.548 0.12173
## purposeA49 -6.731e-01 3.746e-01 -1.797 0.07235 .
## amount 9.191e-05 4.608e-05 1.995 0.04606 *
## saving_acctA62 -7.475e-01 3.577e-01 -2.090 0.03665 *
## saving_acctA63 -3.191e-01 4.281e-01 -0.745 0.45598
## saving_acctA64 -8.390e-01 5.396e-01 -1.555 0.11998
## saving_acctA65 -8.003e-01 2.901e-01 -2.758 0.00581 **
## present_empA72 1.874e-01 4.250e-01 0.441 0.65918
## present_empA73 -7.298e-02 3.923e-01 -0.186 0.85242
## present_empA74 -8.224e-01 4.476e-01 -1.837 0.06618 .
## present_empA75 -4.376e-01 4.097e-01 -1.068 0.28550
## installment_rate 2.437e-01 9.892e-02 2.464 0.01375 *
## other_debtorA102 8.057e-01 4.675e-01 1.724 0.08479 .
## other_debtorA103 -7.844e-01 4.443e-01 -1.766 0.07747 .
## other_installA142 -1.056e-01 4.906e-01 -0.215 0.82951
## other_installA143 -7.089e-01 2.713e-01 -2.614 0.00896 **
## housingA152 -5.159e-01 2.484e-01 -2.077 0.03785 *
## housingA153 -1.338e-01 3.789e-01 -0.353 0.72407
## foreignA202 -1.299e+00 6.173e-01 -2.104 0.03539 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 902.33 on 749 degrees of freedom
## Residual deviance: 684.68 on 715 degrees of freedom
## AIC: 754.68
##
## Number of Fisher Scoring iterations: 54.Lasso
germandata.train<-germandata.train[,1:21]
dummy<- model.matrix(~ ., data = germandata.train)
credit.data.lasso<- data.frame(dummy[,-1])
head(credit.data.lasso)## chk_acctA12 chk_acctA13 chk_acctA14 duration credit_hisA31
## 711 0 0 1 18 0
## 267 0 0 1 36 0
## 865 0 0 1 10 0
## 144 0 0 0 18 0
## 1 0 0 0 6 0
## 421 0 0 1 15 0
## credit_hisA32 credit_hisA33 credit_hisA34 purposeA41 purposeA410
## 711 0 0 1 0 0
## 267 0 0 1 0 0
## 865 1 0 0 0 0
## 144 1 0 0 0 0
## 1 0 0 1 0 0
## 421 1 0 0 0 0
## purposeA42 purposeA43 purposeA44 purposeA45 purposeA46 purposeA48
## 711 0 1 0 0 0 0
## 267 0 0 0 0 0 0
## 865 1 0 0 0 0 0
## 144 1 0 0 0 0 0
## 1 0 1 0 0 0 0
## 421 0 0 0 0 0 0
## purposeA49 amount saving_acctA62 saving_acctA63 saving_acctA64
## 711 0 629 0 1 0
## 267 1 6304 0 0 0
## 865 0 2210 0 0 0
## 144 0 2462 0 0 0
## 1 0 1169 0 0 0
## 421 0 3186 0 0 1
## saving_acctA65 present_empA72 present_empA73 present_empA74
## 711 0 0 0 0
## 267 1 0 0 0
## 865 0 0 1 0
## 144 0 0 1 0
## 1 1 0 0 0
## 421 0 0 0 1
## present_empA75 installment_rate sexA92 sexA93 sexA94 other_debtorA102
## 711 1 4 0 1 0 0
## 267 1 4 0 1 0 0
## 865 0 2 0 1 0 0
## 144 0 2 0 1 0 0
## 1 1 4 0 1 0 0
## 421 0 2 1 0 0 0
## other_debtorA103 present_resid propertyA122 propertyA123 propertyA124
## 711 0 3 1 0 0
## 267 0 4 0 0 0
## 865 0 2 0 0 0
## 144 0 2 0 1 0
## 1 0 4 0 0 0
## 421 0 3 0 1 0
## age other_installA142 other_installA143 housingA152 housingA153
## 711 32 0 0 1 0
## 267 36 0 1 1 0
## 865 25 0 0 0 0
## 144 22 0 1 1 0
## 1 67 0 1 1 0
## 421 20 0 1 0 0
## n_credits jobA172 jobA173 jobA174 n_people telephoneA192 foreignA202
## 711 2 0 0 1 1 1 0
## 267 2 0 1 0 1 0 0
## 865 1 1 0 0 1 0 0
## 144 1 0 1 0 1 0 0
## 1 2 0 1 0 1 1 0
## 421 1 0 1 0 1 0 0
## response1
## 711 0
## 267 0
## 865 1
## 144 1
## 1 0
## 421 0colnames(credit.data.lasso)## [1] "chk_acctA12" "chk_acctA13" "chk_acctA14"
## [4] "duration" "credit_hisA31" "credit_hisA32"
## [7] "credit_hisA33" "credit_hisA34" "purposeA41"
## [10] "purposeA410" "purposeA42" "purposeA43"
## [13] "purposeA44" "purposeA45" "purposeA46"
## [16] "purposeA48" "purposeA49" "amount"
## [19] "saving_acctA62" "saving_acctA63" "saving_acctA64"
## [22] "saving_acctA65" "present_empA72" "present_empA73"
## [25] "present_empA74" "present_empA75" "installment_rate"
## [28] "sexA92" "sexA93" "sexA94"
## [31] "other_debtorA102" "other_debtorA103" "present_resid"
## [34] "propertyA122" "propertyA123" "propertyA124"
## [37] "age" "other_installA142" "other_installA143"
## [40] "housingA152" "housingA153" "n_credits"
## [43] "jobA172" "jobA173" "jobA174"
## [46] "n_people" "telephoneA192" "foreignA202"
## [49] "response1"#Data Preparation for Lasso
index <- trainrows
#credit.train.X = as.matrix(select(credit.data.lasso, -response1)[index,])
#credit.test.X = as.matrix(select(credit.data.lasso, -response1)[-index,])
#credit.train.Y = credit.data.lasso[index, "response1"]
#credit.test.Y = credit.data.lasso[-index, "response1"]
#credit.lasso<- glmnet(x=credit.train.X, y=credit.train.Y, family = "binomial")
#credit.lasso.cv<- cv.glmnet(x=credit.train.X, y=credit.train.Y, family = "binomial", type.measure = "class")
#plot(credit.lasso.cv)Coefficients of lambda min
#coef(credit.lasso, s=credit.lasso.cv$lambda.min)Coefficients of lambda of 1 standard error
#coef(credit.lasso, s=credit.lasso.cv$lambda.1se)5.Classification Tree
Building and plotting a Classificaion Tree using all variables Cost function with 1:5 asymmetric cost
germandata.largetree <- rpart(formula = response~., data = germandata.train,method = "class", parms = list(loss = matrix(c(0, 5, 1, 0), nrow = 2)))
prp(germandata.largetree, extra = 1, nn.font=40,box.palette = "green")
Creating crosstabs of true and predicted value
pred0<- predict(germandata.largetree, type="class")
table(germandata.train$response, pred0, dnn = c("True", "Pred"))## Pred
## True 0 1
## 0 334 199
## 1 13 204Checking relative error for all tree sizes
plotcp(germandata.largetree)
printcp(germandata.largetree)##
## Classification tree:
## rpart(formula = response ~ ., data = germandata.train, method = "class",
## parms = list(loss = matrix(c(0, 5, 1, 0), nrow = 2)))
##
## Variables actually used in tree construction:
## [1] amount chk_acct duration other_install predict
## [6] present_emp present_resid purpose saving_acct
##
## Root node error: 533/750 = 0.71067
##
## n= 750
##
## CP nsplit rel error xerror xstd
## 1 0.257036 0 1.00000 5.0000 0.116495
## 2 0.026266 1 0.74296 1.3283 0.096758
## 3 0.022514 5 0.61914 2.1932 0.116669
## 4 0.015009 6 0.59662 2.2251 0.117325
## 5 0.014071 7 0.58161 2.1820 0.116558
## 6 0.013133 9 0.55347 2.1820 0.116558
## 7 0.012195 10 0.54034 2.1351 0.115826
## 8 0.010319 12 0.51595 2.0507 0.114433
## 9 0.010000 14 0.49531 2.1107 0.115312Pruning the tree using optimal cp and then plotting the optimal tree
german.prunedtree <- rpart(response~., data = germandata.train, method = "class",
parms = list(loss = matrix(c(0, 5, 1, 0), nrow = 2)),cp=0.015009)
prp(german.prunedtree, extra = 1, nn.font=500,box.palette = "green")
In-sample Prediction
credit.train.pred.tree1<- predict(german.prunedtree, germandata.train, type="class")
table(germandata.train$response, credit.train.pred.tree1, dnn=c("Truth","Predicted"))## Predicted
## Truth 0 1
## 0 333 200
## 1 22 195Out-sample Prediction
#credit.test.pred.tree1<- predict(german.prunedtree, newdata=germandata.test, type="class")
#table(germandata.test$response, credit.test.pred.tree1, dnn=c("Truth","Predicted"))
#pred.tree.gtrain <- predict(german.prunedtree, type = "prob")[,2]
#pred.tree.gtest <- predict(german.prunedtree, newdata=germandata.test, type = "prob")[,2]
#credit.train.pred.tree1<- predict(german.prunedtree, data=germandata.test, type="class")
#head(credit.train.pred.tree1[[,2]])
#table(germandata.test$response, credit.test.pred.tree1, dnn=c("Truth","Predicted"))
#pred.tree.gtrain <- predict(german.prunedtree, type = "prob")[,2]
#pred.tree.gtest <- predict(german.prunedtree, data=germandata.test, type = "prob")[,2]
#pred.train <- as.numeric(pred.tree.gtrain > optimal.pcut)
#pred.test <- as.numeric(pred.tree.gtest > optimal.pcut)
#confusion_matrix_train <- table(germandata.train$response, pred.train)
#confusion_matrix_test <- table(germandata.test$response, pred.test)
#misclassification_rate_train <- round((confusion_matrix_train[2]+confusion_matrix_train[3])/sum(confusion_matrix_train), 2)
#misclassification_rate_test <- round((confusion_matrix_test[2]+confusion_matrix_test[3])/sum(confusion_matrix_test), 2)
#cat("train misclassfication rate:", misclassification_rate_train, "| test misclassfication rate:", misclassification_rate_test)6.GAMs
Building a Generalized Additive Model
germandata.gam <- gam(as.factor(response)~chk_acct+s(duration)+credit_his+purpose+s(amount)+saving_acct+present_emp+installment_rate+sex+other_debtor+present_resid+property
+s(age)+other_install+housing+n_credits+telephone+foreign , family=binomial,data=germandata.train)
summary(germandata.gam)##
## Family: binomial
## Link function: logit
##
## Formula:
## as.factor(response) ~ chk_acct + s(duration) + credit_his + purpose +
## s(amount) + saving_acct + present_emp + installment_rate +
## sex + other_debtor + present_resid + property + s(age) +
## other_install + housing + n_credits + telephone + foreign
##
## Parametric coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 1.73499 0.97295 1.783 0.074550 .
## chk_acctA12 -0.39872 0.25495 -1.564 0.117836
## chk_acctA13 -1.25695 0.42700 -2.944 0.003243 **
## chk_acctA14 -1.64115 0.27004 -6.077 1.22e-09 ***
## credit_hisA31 0.67822 0.62572 1.084 0.278403
## credit_hisA32 -0.41406 0.48379 -0.856 0.392070
## credit_hisA33 -0.74087 0.52470 -1.412 0.157954
## credit_hisA34 -1.23285 0.49098 -2.511 0.012039 *
## purposeA41 -1.50127 0.42849 -3.504 0.000459 ***
## purposeA410 -1.77146 0.81426 -2.176 0.029589 *
## purposeA42 -0.77717 0.30986 -2.508 0.012138 *
## purposeA43 -0.80479 0.28289 -2.845 0.004443 **
## purposeA44 -0.59062 0.98369 -0.600 0.548233
## purposeA45 -0.14647 0.60786 -0.241 0.809590
## purposeA46 0.01392 0.46918 0.030 0.976330
## purposeA48 -2.05678 1.28478 -1.601 0.109404
## purposeA49 -0.58042 0.38068 -1.525 0.127331
## saving_acctA62 -0.76531 0.36447 -2.100 0.035750 *
## saving_acctA63 -0.34857 0.43022 -0.810 0.417823
## saving_acctA64 -0.91498 0.54794 -1.670 0.094946 .
## saving_acctA65 -0.79432 0.29608 -2.683 0.007301 **
## present_empA72 0.08586 0.43745 0.196 0.844392
## present_empA73 -0.06766 0.40001 -0.169 0.865688
## present_empA74 -0.78556 0.45429 -1.729 0.083770 .
## present_empA75 -0.42007 0.41898 -1.003 0.316051
## installment_rate 0.22439 0.10535 2.130 0.033175 *
## sexA92 -0.23545 0.44499 -0.529 0.596728
## sexA93 -0.50671 0.43443 -1.166 0.243463
## sexA94 -0.29413 0.53269 -0.552 0.580831
## other_debtorA102 0.76429 0.47913 1.595 0.110677
## other_debtorA103 -0.78319 0.46169 -1.696 0.089819 .
## present_resid -0.01038 0.09985 -0.104 0.917207
## propertyA122 0.21695 0.29202 0.743 0.457526
## propertyA123 0.20147 0.27349 0.737 0.461333
## propertyA124 0.80037 0.48109 1.664 0.096179 .
## other_installA142 -0.01103 0.49221 -0.022 0.982114
## other_installA143 -0.66704 0.27766 -2.402 0.016291 *
## housingA152 -0.46127 0.26904 -1.715 0.086434 .
## housingA153 -0.50695 0.53286 -0.951 0.341420
## n_credits 0.14560 0.22456 0.648 0.516749
## telephoneA192 -0.24913 0.22129 -1.126 0.260245
## foreignA202 -1.31608 0.64223 -2.049 0.040439 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Approximate significance of smooth terms:
## edf Ref.df Chi.sq p-value
## s(duration) 1.951 2.470 8.870 0.0249 *
## s(amount) 2.552 3.237 7.037 0.0767 .
## s(age) 1.000 1.000 0.027 0.8695
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## R-sq.(adj) = 0.249 Deviance explained = 25.5%
## UBRE = 0.022901 Scale est. = 1 n = 750Plotting the non-linear terms in Generalized Additive Model
plot(germandata.gam, shade=TRUE)
Moving age to partially linear term
# Move age to partially linear term and refit gam() model
germandata.gam <- gam(as.factor(response)~chk_acct+s(duration)+credit_his+purpose+s(amount)+saving_acct+present_emp+installment_rate+sex+other_debtor+present_resid+property
+(age)+other_install+housing+n_credits+telephone+foreign , family=binomial,data=germandata.train)
summary(germandata.gam)##
## Family: binomial
## Link function: logit
##
## Formula:
## as.factor(response) ~ chk_acct + s(duration) + credit_his + purpose +
## s(amount) + saving_acct + present_emp + installment_rate +
## sex + other_debtor + present_resid + property + (age) + other_install +
## housing + n_credits + telephone + foreign
##
## Parametric coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 1.796336 1.037565 1.731 0.083398 .
## chk_acctA12 -0.398722 0.254950 -1.564 0.117836
## chk_acctA13 -1.256953 0.426997 -2.944 0.003243 **
## chk_acctA14 -1.641154 0.270044 -6.077 1.22e-09 ***
## credit_hisA31 0.678224 0.625717 1.084 0.278403
## credit_hisA32 -0.414061 0.483791 -0.856 0.392071
## credit_hisA33 -0.740865 0.524697 -1.412 0.157954
## credit_hisA34 -1.232850 0.490978 -2.511 0.012039 *
## purposeA41 -1.501274 0.428488 -3.504 0.000459 ***
## purposeA410 -1.771460 0.814261 -2.176 0.029589 *
## purposeA42 -0.777174 0.309864 -2.508 0.012138 *
## purposeA43 -0.804785 0.282887 -2.845 0.004443 **
## purposeA44 -0.590617 0.983691 -0.600 0.548234
## purposeA45 -0.146467 0.607864 -0.241 0.809592
## purposeA46 0.013921 0.469181 0.030 0.976329
## purposeA48 -2.056777 1.284780 -1.601 0.109404
## purposeA49 -0.580420 0.380676 -1.525 0.127331
## saving_acctA62 -0.765308 0.364474 -2.100 0.035750 *
## saving_acctA63 -0.348568 0.430223 -0.810 0.417823
## saving_acctA64 -0.914980 0.547936 -1.670 0.094946 .
## saving_acctA65 -0.794324 0.296082 -2.683 0.007301 **
## present_empA72 0.085861 0.437445 0.196 0.844393
## present_empA73 -0.067658 0.400010 -0.169 0.865687
## present_empA74 -0.785564 0.454288 -1.729 0.083770 .
## present_empA75 -0.420075 0.418982 -1.003 0.316049
## installment_rate 0.224385 0.105347 2.130 0.033175 *
## sexA92 -0.235448 0.444989 -0.529 0.596730
## sexA93 -0.506713 0.434434 -1.166 0.243463
## sexA94 -0.294132 0.532684 -0.552 0.580832
## other_debtorA102 0.764294 0.479134 1.595 0.110677
## other_debtorA103 -0.783190 0.461690 -1.696 0.089819 .
## present_resid -0.010380 0.099853 -0.104 0.917209
## propertyA122 0.216951 0.292022 0.743 0.457526
## propertyA123 0.201469 0.273493 0.737 0.461333
## propertyA124 0.800372 0.481088 1.664 0.096178 .
## age -0.001719 0.010456 -0.164 0.869392
## other_installA142 -0.011034 0.492214 -0.022 0.982115
## other_installA143 -0.667039 0.277662 -2.402 0.016291 *
## housingA152 -0.461272 0.269039 -1.715 0.086433 .
## housingA153 -0.506948 0.532864 -0.951 0.341419
## n_credits 0.145597 0.224560 0.648 0.516749
## telephoneA192 -0.249134 0.221292 -1.126 0.260244
## foreignA202 -1.316076 0.642226 -2.049 0.040438 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Approximate significance of smooth terms:
## edf Ref.df Chi.sq p-value
## s(duration) 1.951 2.470 8.870 0.0249 *
## s(amount) 2.552 3.237 7.037 0.0767 .
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## R-sq.(adj) = 0.249 Deviance explained = 25.5%
## UBRE = 0.0229 Scale est. = 1 n = 750Plotting the non-linear terms in Generalized Additive Model
plot(germandata.gam, shade=TRUE)
Train and Test Predictions
pred.glm.gtrain.gam <- predict(germandata.gam, type = "response")
pred.glm.gtest.gam <- predict(germandata.gam, newdata=germandata.test,type = "response")
pred.train <- as.numeric(pred.glm.gtrain.gam > optimal.pcut)
pred.test <- as.numeric(pred.glm.gtest.gam > optimal.pcut)
confusion_matrix_train <- table(germandata.train$response, pred.train)
confusion_matrix_test <- table(germandata.test$response, pred.test)
misclassification_rate_train <- round((confusion_matrix_train[2]+confusion_matrix_train[3])/sum(confusion_matrix_train), 2)
misclassification_rate_test <- round((confusion_matrix_test[2]+confusion_matrix_test[3])/sum(confusion_matrix_test), 2)
cat("train misclassfication rate:", misclassification_rate_train, "| test misclassfication rate:", misclassification_rate_test)## train misclassfication rate: 0.33 | test misclassfication rate: 0.3Test/Train Confusion Matrix
confusion_matrix_train## pred.train
## 0 1
## 0 320 213
## 1 31 186confusion_matrix_test## pred.test
## 0 1
## 0 101 66
## 1 10 73ROC Curve - Train
par(mfrow=c(1,1))
roc.logit <- roc.plot(x=(germandata.train$response == "1"), pred =pred.glm.gtrain.gam)
AUC Train
roc.logit$roc.vol[2]## Area
## 1 0.8294326ROC Curve - Test
par(mfrow=c(1,1))
roc.logit.test <- roc.plot(x=(germandata.test$response == "1"), pred =pred.glm.gtest.gam)
AUC - Test
roc.logit.test$roc.vol[2]## Area
## 1 0.8374576Train and Test Asymmetric Misclassfication Rate or Asymmetric Misclassification Cost
class.pred.train.gam <- (pred.glm.gtrain.gam>pcut)*1
cost.train <- round(cost(r = germandata.train$response, pi = class.pred.train.gam),2)
class.pred.test.gam<- (pred.glm.gtest.gam>pcut)*1
cost.test <- round(cost(r = germandata.test$response, pi = class.pred.test.gam),2)
cat("total train cost:", cost.train, "| total test cost:", cost.test)## total train cost: 0.49 | total test cost: 0.467.Neural Network
Building a Neural Network Model
par(mfrow=c(1,1))
germandata.nnet <- train(response~., data=germandata.train,method="nnet")## # weights: 51
## initial value 595.549357
## final value 452.060645
## converged
## # weights: 151
## initial value 604.536597
## final value 452.060645
## converged
## # weights: 251
## initial value 763.720935
## final value 452.060645
## converged
## # weights: 51
## initial value 454.203971
## iter 10 value 452.100528
## final value 452.100521
## converged
## # weights: 151
## initial value 596.518658
## iter 10 value 452.094331
## iter 20 value 444.949850
## iter 30 value 425.030038
## iter 40 value 392.283981
## iter 50 value 351.296580
## iter 60 value 313.216978
## iter 70 value 306.224949
## iter 80 value 305.736336
## iter 90 value 305.225960
## iter 100 value 304.356608
## final value 304.356608
## stopped after 100 iterations
## # weights: 251
## initial value 692.821596
## iter 10 value 452.204716
## iter 20 value 444.957274
## iter 30 value 410.652759
## iter 40 value 388.208093
## iter 50 value 331.656923
## iter 60 value 308.447830
## iter 70 value 304.046808
## iter 80 value 281.978600
## iter 90 value 247.428214
## iter 100 value 243.190588
## final value 243.190588
## stopped after 100 iterations
## # weights: 51
## initial value 471.946598
## final value 452.061530
## converged
## # weights: 151
## initial value 560.788742
## final value 452.062988
## converged
## # weights: 251
## initial value 686.883092
## final value 452.065046
## converged
## # weights: 51
## initial value 524.462787
## final value 447.518697
## converged
## # weights: 151
## initial value 504.291769
## final value 447.518697
## converged
## # weights: 251
## initial value 458.981830
## iter 10 value 443.288533
## iter 20 value 423.613853
## iter 30 value 405.022502
## iter 40 value 320.819643
## iter 50 value 295.147469
## iter 60 value 280.032495
## iter 70 value 266.260061
## iter 80 value 265.975431
## final value 265.974966
## converged
## # weights: 51
## initial value 673.117113
## iter 10 value 447.561538
## final value 447.561528
## converged
## # weights: 151
## initial value 457.102413
## iter 10 value 442.998424
## iter 20 value 434.054731
## iter 30 value 428.560415
## iter 40 value 426.215613
## iter 50 value 406.502152
## iter 60 value 378.055857
## iter 70 value 342.870024
## iter 80 value 302.350739
## iter 90 value 291.046109
## iter 100 value 290.349498
## final value 290.349498
## stopped after 100 iterations
## # weights: 251
## initial value 467.834282
## iter 10 value 447.703162
## iter 20 value 447.536973
## iter 30 value 447.536195
## iter 30 value 447.536192
## iter 30 value 447.536192
## final value 447.536192
## converged
## # weights: 51
## initial value 799.826765
## final value 447.519655
## converged
## # weights: 151
## initial value 580.609884
## final value 447.521189
## converged
## # weights: 251
## initial value 461.056819
## final value 447.522877
## converged
## # weights: 51
## initial value 501.711848
## final value 453.832031
## converged
## # weights: 151
## initial value 469.594181
## final value 453.832031
## converged
## # weights: 251
## initial value 502.533459
## iter 10 value 444.858699
## iter 20 value 441.187992
## iter 30 value 440.891721
## iter 40 value 440.880638
## iter 50 value 440.880413
## iter 50 value 440.880409
## iter 50 value 440.880405
## final value 440.880405
## converged
## # weights: 51
## initial value 610.606111
## iter 10 value 453.909217
## iter 20 value 450.128665
## iter 30 value 444.181625
## iter 40 value 440.877623
## iter 50 value 439.957603
## iter 60 value 439.696916
## final value 439.696906
## converged
## # weights: 151
## initial value 459.595609
## iter 10 value 453.854531
## iter 20 value 446.583561
## iter 30 value 374.314363
## iter 40 value 328.184645
## iter 50 value 308.043720
## iter 60 value 305.515443
## iter 70 value 299.186242
## iter 80 value 277.781324
## iter 90 value 269.942604
## iter 100 value 258.767268
## final value 258.767268
## stopped after 100 iterations
## # weights: 251
## initial value 684.596489
## iter 10 value 453.572114
## iter 20 value 448.853726
## iter 30 value 430.222718
## iter 40 value 404.455966
## iter 50 value 344.423462
## iter 60 value 308.150515
## iter 70 value 274.030872
## iter 80 value 255.901757
## iter 90 value 253.324543
## iter 100 value 250.706153
## final value 250.706153
## stopped after 100 iterations
## # weights: 51
## initial value 467.285291
## final value 453.832914
## converged
## # weights: 151
## initial value 585.425496
## final value 453.834357
## converged
## # weights: 251
## initial value 460.034523
## final value 453.836426
## converged
## # weights: 51
## initial value 489.778945
## final value 453.832031
## converged
## # weights: 151
## initial value 537.049198
## final value 453.832031
## converged
## # weights: 251
## initial value 487.384416
## final value 453.832031
## converged
## # weights: 51
## initial value 468.001110
## iter 10 value 453.278484
## iter 20 value 414.915813
## iter 30 value 375.340836
## iter 40 value 352.367160
## iter 50 value 343.746226
## iter 60 value 342.888613
## iter 70 value 342.865975
## final value 342.865183
## converged
## # weights: 151
## initial value 602.698326
## iter 10 value 453.901223
## iter 20 value 453.248979
## iter 30 value 429.095229
## iter 40 value 415.208742
## iter 50 value 398.350866
## iter 60 value 362.085023
## iter 70 value 332.722043
## iter 80 value 328.321506
## iter 90 value 328.190623
## iter 100 value 328.175274
## final value 328.175274
## stopped after 100 iterations
## # weights: 251
## initial value 655.797383
## iter 10 value 451.408937
## iter 20 value 427.548828
## iter 30 value 392.106591
## iter 40 value 365.225579
## iter 50 value 362.182785
## iter 60 value 352.912717
## iter 70 value 336.375646
## iter 80 value 330.252908
## iter 90 value 327.565502
## iter 100 value 326.978025
## final value 326.978025
## stopped after 100 iterations
## # weights: 51
## initial value 674.271603
## final value 453.832853
## converged
## # weights: 151
## initial value 456.416088
## final value 453.834379
## converged
## # weights: 251
## initial value 524.035157
## final value 453.835723
## converged
## # weights: 51
## initial value 597.139815
## final value 452.060645
## converged
## # weights: 151
## initial value 486.333657
## final value 452.060645
## converged
## # weights: 251
## initial value 1471.194535
## final value 452.060645
## converged
## # weights: 51
## initial value 462.723673
## iter 10 value 451.762385
## iter 20 value 420.426485
## iter 30 value 372.314281
## iter 40 value 328.751738
## iter 50 value 319.026851
## iter 60 value 317.701481
## iter 70 value 317.664996
## iter 80 value 317.663268
## iter 80 value 317.663267
## iter 80 value 317.663267
## final value 317.663267
## converged
## # weights: 151
## initial value 503.970792
## iter 10 value 451.668136
## iter 20 value 448.368636
## iter 30 value 435.569263
## iter 40 value 405.541041
## iter 50 value 401.130031
## iter 60 value 374.892084
## iter 70 value 344.279361
## iter 80 value 331.213357
## iter 90 value 314.425742
## iter 100 value 305.488965
## final value 305.488965
## stopped after 100 iterations
## # weights: 251
## initial value 456.737019
## iter 10 value 452.355427
## iter 20 value 439.770902
## iter 30 value 418.371790
## iter 40 value 384.290546
## iter 50 value 335.172159
## iter 60 value 322.045038
## iter 70 value 309.794798
## iter 80 value 307.317275
## iter 90 value 298.551047
## iter 100 value 287.075929
## final value 287.075929
## stopped after 100 iterations
## # weights: 51
## initial value 473.530615
## final value 452.061548
## converged
## # weights: 151
## initial value 470.107630
## final value 452.063142
## converged
## # weights: 251
## initial value 459.911514
## final value 452.064884
## converged
## # weights: 51
## initial value 672.249926
## final value 438.928465
## converged
## # weights: 151
## initial value 475.282245
## final value 438.928465
## converged
## # weights: 251
## initial value 487.389535
## iter 10 value 438.588919
## iter 20 value 436.373137
## final value 436.372757
## converged
## # weights: 51
## initial value 678.660715
## iter 10 value 439.025595
## iter 20 value 438.978478
## final value 438.977005
## converged
## # weights: 151
## initial value 454.982055
## iter 10 value 437.956549
## iter 20 value 405.840220
## iter 30 value 363.454587
## iter 40 value 337.939548
## iter 50 value 308.751342
## iter 60 value 301.866131
## iter 70 value 274.885072
## iter 80 value 260.301301
## iter 90 value 246.448249
## iter 100 value 237.584764
## final value 237.584764
## stopped after 100 iterations
## # weights: 251
## initial value 557.753906
## iter 10 value 438.982658
## iter 20 value 438.903319
## iter 30 value 434.026130
## iter 40 value 431.463059
## iter 50 value 423.135176
## iter 60 value 403.601714
## iter 70 value 390.768144
## iter 80 value 336.816366
## iter 90 value 295.024427
## iter 100 value 274.451806
## final value 274.451806
## stopped after 100 iterations
## # weights: 51
## initial value 545.768440
## final value 438.929217
## converged
## # weights: 151
## initial value 501.672281
## final value 438.930960
## converged
## # weights: 251
## initial value 514.322779
## final value 438.932175
## converged
## # weights: 51
## initial value 454.387463
## final value 453.832031
## converged
## # weights: 151
## initial value 686.179005
## final value 453.832031
## converged
## # weights: 251
## initial value 478.907733
## final value 453.832031
## converged
## # weights: 51
## initial value 498.456414
## iter 10 value 454.001200
## iter 20 value 453.870816
## final value 453.870766
## converged
## # weights: 151
## initial value 719.261515
## iter 10 value 453.870183
## iter 20 value 433.272411
## iter 30 value 367.393684
## iter 40 value 328.654371
## iter 50 value 325.211608
## iter 60 value 314.148913
## iter 70 value 295.232038
## iter 80 value 266.387690
## iter 90 value 254.153918
## iter 100 value 249.817369
## final value 249.817369
## stopped after 100 iterations
## # weights: 251
## initial value 554.167339
## iter 10 value 453.913617
## iter 20 value 453.846083
## iter 30 value 453.845501
## iter 30 value 453.845499
## iter 30 value 453.845499
## final value 453.845499
## converged
## # weights: 51
## initial value 622.068701
## final value 453.833067
## converged
## # weights: 151
## initial value 455.417686
## final value 453.834592
## converged
## # weights: 251
## initial value 465.347970
## final value 453.835824
## converged
## # weights: 51
## initial value 516.699962
## final value 436.945973
## converged
## # weights: 151
## initial value 549.790548
## final value 436.945973
## converged
## # weights: 251
## initial value 437.110845
## iter 10 value 435.671300
## iter 20 value 435.632393
## final value 435.632357
## converged
## # weights: 51
## initial value 528.283239
## iter 10 value 429.381290
## iter 20 value 413.199469
## iter 30 value 393.023019
## iter 40 value 341.423795
## iter 50 value 320.213491
## iter 60 value 309.069817
## iter 70 value 304.527022
## iter 80 value 304.182365
## iter 90 value 304.153392
## final value 304.152943
## converged
## # weights: 151
## initial value 790.969272
## iter 10 value 436.982476
## iter 20 value 434.786701
## iter 30 value 373.531593
## iter 40 value 335.644462
## iter 50 value 329.977048
## iter 60 value 321.954221
## iter 70 value 320.715305
## iter 80 value 318.879589
## iter 90 value 317.889918
## iter 100 value 314.639967
## final value 314.639967
## stopped after 100 iterations
## # weights: 251
## initial value 463.825324
## iter 10 value 437.220905
## iter 20 value 435.133199
## iter 30 value 413.295146
## iter 40 value 398.851067
## iter 50 value 347.148743
## iter 60 value 317.515259
## iter 70 value 289.900611
## iter 80 value 278.183112
## iter 90 value 269.220092
## iter 100 value 267.114213
## final value 267.114213
## stopped after 100 iterations
## # weights: 51
## initial value 531.711222
## final value 436.946879
## converged
## # weights: 151
## initial value 462.476651
## final value 436.948320
## converged
## # weights: 251
## initial value 473.386471
## final value 436.950410
## converged
## # weights: 51
## initial value 597.992519
## final value 456.440911
## converged
## # weights: 151
## initial value 555.263284
## final value 456.440911
## converged
## # weights: 251
## initial value 459.613668
## final value 456.440911
## converged
## # weights: 51
## initial value 768.917595
## iter 10 value 456.478215
## final value 456.477982
## converged
## # weights: 151
## initial value 535.696536
## iter 10 value 456.472134
## iter 20 value 455.754847
## iter 30 value 452.067052
## iter 40 value 450.110471
## iter 50 value 429.105143
## iter 60 value 396.955527
## iter 70 value 359.021732
## iter 80 value 330.183354
## iter 90 value 325.011137
## iter 100 value 321.039484
## final value 321.039484
## stopped after 100 iterations
## # weights: 251
## initial value 576.296654
## iter 10 value 456.444228
## iter 20 value 442.056739
## iter 30 value 428.455749
## iter 40 value 421.637095
## iter 50 value 368.239218
## iter 60 value 341.510750
## iter 70 value 310.716560
## iter 80 value 279.778774
## iter 90 value 256.733645
## iter 100 value 243.008571
## final value 243.008571
## stopped after 100 iterations
## # weights: 51
## initial value 461.053945
## final value 456.441740
## converged
## # weights: 151
## initial value 474.739680
## final value 456.443554
## converged
## # weights: 251
## initial value 536.002893
## final value 456.444779
## converged
## # weights: 51
## initial value 514.164713
## final value 471.647400
## converged
## # weights: 151
## initial value 556.458886
## final value 471.647400
## converged
## # weights: 251
## initial value 572.803244
## final value 471.647400
## converged
## # weights: 51
## initial value 555.363092
## iter 10 value 471.625026
## iter 20 value 447.593708
## iter 30 value 423.461192
## iter 40 value 415.615550
## iter 50 value 366.085294
## iter 60 value 343.551186
## iter 70 value 329.846043
## iter 80 value 324.549137
## iter 90 value 324.291788
## iter 100 value 324.270940
## final value 324.270940
## stopped after 100 iterations
## # weights: 151
## initial value 603.443075
## iter 10 value 471.696068
## iter 20 value 448.490162
## iter 20 value 448.490158
## final value 448.490158
## converged
## # weights: 251
## initial value 476.057041
## iter 10 value 471.783295
## iter 20 value 471.670984
## iter 30 value 465.486322
## iter 40 value 463.266091
## iter 50 value 459.039689
## iter 60 value 449.877936
## iter 70 value 440.905962
## iter 80 value 390.020487
## iter 90 value 329.582336
## iter 100 value 304.321523
## final value 304.321523
## stopped after 100 iterations
## # weights: 51
## initial value 497.651584
## final value 471.648310
## converged
## # weights: 151
## initial value 607.446574
## final value 471.649670
## converged
## # weights: 251
## initial value 699.540471
## final value 471.651383
## converged
## # weights: 51
## initial value 454.170887
## final value 449.355019
## converged
## # weights: 151
## initial value 981.187239
## final value 449.355019
## converged
## # weights: 251
## initial value 506.067116
## final value 449.355019
## converged
## # weights: 51
## initial value 524.741380
## iter 10 value 448.859495
## iter 20 value 442.274978
## iter 30 value 439.875461
## final value 439.663721
## converged
## # weights: 151
## initial value 472.508521
## iter 10 value 449.401899
## iter 20 value 449.231158
## iter 30 value 435.520322
## iter 40 value 428.299209
## iter 50 value 422.320956
## iter 60 value 419.739116
## iter 70 value 395.576131
## iter 80 value 340.597063
## iter 90 value 323.420889
## iter 100 value 311.327218
## final value 311.327218
## stopped after 100 iterations
## # weights: 251
## initial value 542.556481
## iter 10 value 449.250853
## iter 20 value 436.092722
## iter 30 value 392.426550
## iter 40 value 340.305985
## iter 50 value 329.039326
## iter 60 value 314.552661
## iter 70 value 271.114177
## iter 80 value 250.066821
## iter 90 value 241.400758
## iter 100 value 240.836012
## final value 240.836012
## stopped after 100 iterations
## # weights: 51
## initial value 500.043694
## final value 449.355864
## converged
## # weights: 151
## initial value 712.609473
## final value 449.357508
## converged
## # weights: 251
## initial value 911.512564
## final value 449.358939
## converged
## # weights: 51
## initial value 636.276573
## final value 448.440127
## converged
## # weights: 151
## initial value 552.044246
## final value 448.440127
## converged
## # weights: 251
## initial value 491.770483
## final value 448.440127
## converged
## # weights: 51
## initial value 593.992414
## iter 10 value 448.482350
## iter 10 value 448.482349
## iter 10 value 448.482349
## final value 448.482349
## converged
## # weights: 151
## initial value 567.273829
## iter 10 value 448.485135
## iter 20 value 448.236531
## iter 30 value 446.658209
## iter 40 value 446.457470
## iter 50 value 440.573978
## iter 60 value 431.843746
## iter 70 value 423.238879
## iter 80 value 415.695465
## iter 90 value 405.428671
## iter 100 value 346.908783
## final value 346.908783
## stopped after 100 iterations
## # weights: 251
## initial value 736.305853
## iter 10 value 447.478598
## iter 20 value 441.072262
## iter 30 value 414.742568
## iter 40 value 378.105017
## iter 50 value 318.381802
## iter 60 value 293.830985
## iter 70 value 286.500595
## iter 80 value 278.587809
## iter 90 value 273.760420
## iter 100 value 265.435587
## final value 265.435587
## stopped after 100 iterations
## # weights: 51
## initial value 552.500462
## final value 448.440983
## converged
## # weights: 151
## initial value 636.827894
## final value 448.442490
## converged
## # weights: 251
## initial value 733.254876
## final value 448.444745
## converged
## # weights: 51
## initial value 482.153819
## final value 436.945973
## converged
## # weights: 151
## initial value 453.809003
## final value 436.945973
## converged
## # weights: 251
## initial value 440.495983
## final value 436.945973
## converged
## # weights: 51
## initial value 480.753430
## iter 10 value 436.995843
## iter 10 value 436.995842
## iter 10 value 436.995842
## final value 436.995842
## converged
## # weights: 151
## initial value 658.296261
## iter 10 value 437.005724
## iter 20 value 428.320109
## iter 30 value 381.152397
## iter 40 value 314.002203
## iter 50 value 299.156999
## iter 60 value 295.410464
## iter 70 value 293.295343
## iter 80 value 285.016882
## iter 90 value 269.622885
## iter 100 value 244.128000
## final value 244.128000
## stopped after 100 iterations
## # weights: 251
## initial value 467.813952
## iter 10 value 436.998397
## iter 20 value 433.245812
## iter 30 value 381.820986
## iter 40 value 350.144952
## iter 50 value 309.468135
## iter 60 value 296.018324
## iter 70 value 292.248671
## iter 80 value 285.188488
## iter 90 value 279.829391
## iter 100 value 269.684456
## final value 269.684456
## stopped after 100 iterations
## # weights: 51
## initial value 447.922890
## final value 436.946843
## converged
## # weights: 151
## initial value 580.798990
## final value 436.948367
## converged
## # weights: 251
## initial value 823.951366
## final value 436.949868
## converged
## # weights: 51
## initial value 587.380133
## final value 418.982539
## converged
## # weights: 151
## initial value 619.529986
## final value 418.982539
## converged
## # weights: 251
## initial value 420.350562
## final value 418.982539
## converged
## # weights: 51
## initial value 465.888854
## iter 10 value 417.012947
## iter 20 value 382.436288
## iter 30 value 312.819949
## iter 40 value 284.933034
## iter 50 value 276.545621
## iter 60 value 274.920256
## iter 70 value 273.629653
## iter 80 value 271.272628
## iter 90 value 266.062722
## iter 100 value 264.603402
## final value 264.603402
## stopped after 100 iterations
## # weights: 151
## initial value 635.015308
## iter 10 value 419.057388
## iter 20 value 415.190810
## iter 30 value 385.569399
## iter 40 value 351.091409
## iter 50 value 297.027339
## iter 60 value 270.865877
## iter 70 value 264.670298
## iter 80 value 262.779990
## iter 90 value 259.796473
## iter 100 value 252.905421
## final value 252.905421
## stopped after 100 iterations
## # weights: 251
## initial value 849.842962
## iter 10 value 418.950300
## iter 20 value 418.776576
## iter 30 value 399.659645
## iter 40 value 371.224564
## iter 50 value 332.850157
## iter 60 value 297.107173
## iter 70 value 275.960846
## iter 80 value 257.682068
## iter 90 value 248.152524
## iter 100 value 235.785113
## final value 235.785113
## stopped after 100 iterations
## # weights: 51
## initial value 434.737329
## final value 418.983419
## converged
## # weights: 151
## initial value 478.581813
## final value 418.984978
## converged
## # weights: 251
## initial value 488.592774
## final value 418.986282
## converged
## # weights: 51
## initial value 485.875446
## final value 460.661621
## converged
## # weights: 151
## initial value 718.848394
## final value 460.661621
## converged
## # weights: 251
## initial value 559.493527
## final value 460.661621
## converged
## # weights: 51
## initial value 513.959958
## iter 10 value 460.731395
## iter 20 value 460.139889
## iter 30 value 441.979573
## iter 40 value 438.873065
## iter 50 value 401.530319
## iter 60 value 346.047427
## iter 70 value 336.686798
## iter 80 value 333.041298
## iter 90 value 325.532902
## iter 100 value 315.497336
## final value 315.497336
## stopped after 100 iterations
## # weights: 151
## initial value 480.072881
## iter 10 value 460.699886
## iter 20 value 460.683731
## iter 30 value 460.555843
## iter 40 value 443.612762
## iter 50 value 396.692341
## iter 60 value 337.602445
## iter 70 value 317.442248
## iter 80 value 310.042545
## iter 90 value 309.702518
## iter 100 value 309.580203
## final value 309.580203
## stopped after 100 iterations
## # weights: 251
## initial value 646.427677
## iter 10 value 458.193917
## iter 20 value 427.731355
## iter 30 value 362.406169
## iter 40 value 349.144104
## iter 50 value 328.261820
## iter 60 value 304.050111
## iter 70 value 296.237077
## iter 80 value 272.796618
## iter 90 value 247.555011
## iter 100 value 237.341425
## final value 237.341425
## stopped after 100 iterations
## # weights: 51
## initial value 573.822051
## final value 460.662824
## converged
## # weights: 151
## initial value 475.000745
## final value 460.664136
## converged
## # weights: 251
## initial value 567.527118
## final value 460.665282
## converged
## # weights: 51
## initial value 451.317041
## final value 444.714988
## converged
## # weights: 151
## initial value 546.285181
## final value 444.714988
## converged
## # weights: 251
## initial value 477.045724
## final value 444.714988
## converged
## # weights: 51
## initial value 554.285355
## final value 444.804071
## converged
## # weights: 151
## initial value 550.986913
## iter 10 value 444.762354
## iter 20 value 430.453956
## iter 30 value 409.211367
## iter 40 value 372.279456
## iter 50 value 343.865990
## iter 60 value 331.910965
## iter 70 value 330.290005
## iter 80 value 328.070751
## iter 90 value 323.796768
## iter 100 value 313.721814
## final value 313.721814
## stopped after 100 iterations
## # weights: 251
## initial value 556.710000
## iter 10 value 444.763462
## iter 20 value 444.023923
## iter 30 value 404.757665
## iter 40 value 355.191079
## iter 50 value 341.362373
## iter 60 value 318.881549
## iter 70 value 302.839472
## iter 80 value 292.228027
## iter 90 value 285.131461
## iter 100 value 279.994673
## final value 279.994673
## stopped after 100 iterations
## # weights: 51
## initial value 554.925009
## final value 444.715818
## converged
## # weights: 151
## initial value 470.283011
## iter 10 value 433.381883
## iter 20 value 426.297648
## iter 30 value 421.543746
## iter 40 value 418.812708
## iter 50 value 416.830103
## iter 60 value 411.581808
## iter 70 value 407.469391
## iter 80 value 403.262546
## iter 90 value 390.200809
## iter 100 value 388.702191
## final value 388.702191
## stopped after 100 iterations
## # weights: 251
## initial value 491.817643
## final value 444.719289
## converged
## # weights: 51
## initial value 438.085386
## final value 437.940596
## converged
## # weights: 151
## initial value 945.325873
## final value 437.940596
## converged
## # weights: 251
## initial value 796.985842
## final value 437.940596
## converged
## # weights: 51
## initial value 534.368728
## iter 10 value 437.991574
## final value 437.989790
## converged
## # weights: 151
## initial value 687.798846
## iter 10 value 437.850128
## iter 20 value 432.041319
## iter 30 value 398.975845
## iter 40 value 384.777957
## iter 50 value 333.239273
## iter 60 value 314.553709
## iter 70 value 301.593657
## iter 80 value 300.970723
## iter 90 value 295.572788
## iter 100 value 290.714301
## final value 290.714301
## stopped after 100 iterations
## # weights: 251
## initial value 451.448542
## iter 10 value 437.905883
## iter 20 value 435.539715
## iter 30 value 413.833950
## iter 40 value 376.669464
## iter 50 value 353.773733
## iter 60 value 330.398627
## iter 70 value 319.181988
## iter 80 value 306.258381
## iter 90 value 292.311266
## iter 100 value 282.306549
## final value 282.306549
## stopped after 100 iterations
## # weights: 51
## initial value 540.823807
## final value 437.941431
## converged
## # weights: 151
## initial value 438.051687
## final value 437.942822
## converged
## # weights: 251
## initial value 449.758857
## final value 437.944925
## converged
## # weights: 51
## initial value 593.019877
## final value 442.812812
## converged
## # weights: 151
## initial value 644.726434
## final value 442.812812
## converged
## # weights: 251
## initial value 461.555479
## iter 10 value 442.813427
## final value 442.812813
## converged
## # weights: 51
## initial value 724.569040
## iter 10 value 442.858954
## final value 442.858750
## converged
## # weights: 151
## initial value 799.798303
## iter 10 value 442.908090
## iter 20 value 442.782724
## iter 30 value 437.275031
## iter 40 value 403.870134
## iter 50 value 361.425075
## iter 60 value 315.322700
## iter 70 value 307.462025
## iter 80 value 306.062543
## iter 90 value 292.144497
## iter 100 value 278.860834
## final value 278.860834
## stopped after 100 iterations
## # weights: 251
## initial value 472.345790
## iter 10 value 442.836543
## iter 20 value 430.778981
## iter 30 value 387.836847
## iter 40 value 313.725624
## iter 50 value 299.734612
## iter 60 value 298.504848
## iter 70 value 292.057828
## iter 80 value 289.223176
## iter 90 value 288.666285
## iter 100 value 287.608153
## final value 287.608153
## stopped after 100 iterations
## # weights: 51
## initial value 536.898256
## final value 442.813656
## converged
## # weights: 151
## initial value 737.242669
## final value 442.815589
## converged
## # weights: 251
## initial value 488.036796
## final value 442.816836
## converged
## # weights: 51
## initial value 489.840263
## final value 452.949563
## converged
## # weights: 151
## initial value 543.694507
## final value 452.949563
## converged
## # weights: 251
## initial value 669.099236
## final value 452.949561
## converged
## # weights: 51
## initial value 462.928865
## iter 10 value 453.013747
## iter 20 value 446.406059
## iter 30 value 443.737186
## iter 40 value 430.682657
## iter 50 value 405.252391
## iter 60 value 376.543176
## iter 70 value 362.472348
## iter 80 value 348.682490
## iter 90 value 336.052838
## iter 100 value 333.871417
## final value 333.871417
## stopped after 100 iterations
## # weights: 151
## initial value 480.004532
## iter 10 value 452.977992
## iter 20 value 440.037588
## iter 30 value 411.403522
## iter 40 value 356.033442
## iter 50 value 340.674486
## iter 60 value 328.429408
## iter 70 value 324.668604
## iter 80 value 323.934093
## iter 90 value 317.083017
## iter 100 value 305.317065
## final value 305.317065
## stopped after 100 iterations
## # weights: 251
## initial value 787.518242
## iter 10 value 456.317299
## iter 20 value 453.727297
## iter 30 value 446.783302
## iter 40 value 418.899144
## iter 50 value 376.092126
## iter 60 value 350.734409
## iter 70 value 336.611773
## iter 80 value 328.340770
## iter 90 value 325.122795
## iter 100 value 324.407741
## final value 324.407741
## stopped after 100 iterations
## # weights: 51
## initial value 485.656516
## final value 452.950448
## converged
## # weights: 151
## initial value 493.627854
## final value 452.951611
## converged
## # weights: 251
## initial value 452.955612
## final value 452.953896
## converged
## # weights: 51
## initial value 591.054276
## final value 453.832031
## converged
## # weights: 151
## initial value 482.198463
## final value 453.832031
## converged
## # weights: 251
## initial value 463.493651
## iter 10 value 448.501789
## iter 20 value 418.108918
## iter 30 value 398.868254
## iter 40 value 386.637008
## iter 50 value 368.207874
## iter 60 value 360.446454
## iter 70 value 359.797702
## iter 80 value 359.385330
## iter 90 value 356.626082
## iter 100 value 347.502770
## final value 347.502770
## stopped after 100 iterations
## # weights: 51
## initial value 508.222231
## iter 10 value 453.870837
## final value 453.870773
## converged
## # weights: 151
## initial value 722.909507
## iter 10 value 453.882399
## iter 20 value 433.130110
## iter 30 value 384.557989
## iter 40 value 324.116355
## iter 50 value 314.508591
## iter 60 value 297.265258
## iter 70 value 289.918424
## iter 80 value 288.703117
## iter 90 value 288.459763
## iter 100 value 287.975849
## final value 287.975849
## stopped after 100 iterations
## # weights: 251
## initial value 584.446032
## iter 10 value 454.148348
## iter 20 value 453.696383
## iter 30 value 447.589999
## iter 40 value 444.550476
## iter 50 value 443.110576
## iter 60 value 432.643425
## iter 70 value 415.158611
## iter 80 value 385.322521
## iter 90 value 350.427648
## iter 100 value 298.926597
## final value 298.926597
## stopped after 100 iterations
## # weights: 51
## initial value 553.442096
## final value 453.832700
## converged
## # weights: 151
## initial value 457.950524
## final value 453.834463
## converged
## # weights: 251
## initial value 749.879671
## final value 453.837390
## converged
## # weights: 51
## initial value 490.357926
## final value 423.391291
## converged
## # weights: 151
## initial value 452.853034
## final value 423.391291
## converged
## # weights: 251
## initial value 748.911189
## final value 423.391292
## converged
## # weights: 51
## initial value 438.542291
## iter 10 value 423.508736
## iter 20 value 414.998938
## iter 30 value 394.783787
## iter 40 value 354.107282
## iter 50 value 336.031520
## iter 60 value 332.560532
## iter 70 value 330.775338
## iter 80 value 330.563830
## iter 90 value 329.352501
## iter 100 value 329.216965
## final value 329.216965
## stopped after 100 iterations
## # weights: 151
## initial value 794.513177
## iter 10 value 423.470795
## iter 20 value 421.489935
## iter 30 value 392.987969
## iter 40 value 381.314914
## iter 50 value 335.137504
## iter 60 value 329.136933
## iter 70 value 328.476096
## iter 80 value 323.997945
## iter 90 value 322.959680
## iter 100 value 321.179774
## final value 321.179774
## stopped after 100 iterations
## # weights: 251
## initial value 525.365971
## iter 10 value 423.200587
## iter 20 value 410.304040
## iter 30 value 362.618196
## iter 40 value 333.481920
## iter 50 value 323.964284
## iter 60 value 322.966498
## iter 70 value 322.660182
## iter 80 value 322.337216
## iter 90 value 320.059612
## iter 100 value 319.455877
## final value 319.455877
## stopped after 100 iterations
## # weights: 51
## initial value 437.873581
## final value 423.392257
## converged
## # weights: 151
## initial value 479.999960
## final value 423.393765
## converged
## # weights: 251
## initial value 820.598821
## final value 423.395353
## converged
## # weights: 51
## initial value 498.056065
## final value 455.577688
## converged
## # weights: 151
## initial value 657.505880
## final value 455.577688
## converged
## # weights: 251
## initial value 466.027174
## final value 455.577690
## converged
## # weights: 51
## initial value 510.365943
## iter 10 value 455.615448
## final value 455.615310
## converged
## # weights: 151
## initial value 459.608761
## iter 10 value 455.237881
## iter 20 value 453.034025
## iter 30 value 422.538874
## iter 40 value 380.615106
## iter 50 value 346.717295
## iter 60 value 324.260974
## iter 70 value 319.545671
## iter 80 value 317.383322
## iter 90 value 315.583420
## iter 100 value 315.082132
## final value 315.082132
## stopped after 100 iterations
## # weights: 251
## initial value 484.991592
## iter 10 value 455.973921
## iter 20 value 448.360212
## iter 30 value 445.307414
## iter 40 value 419.971678
## iter 50 value 359.566885
## iter 60 value 321.371325
## iter 70 value 295.078607
## iter 80 value 281.758315
## iter 90 value 277.531011
## iter 100 value 256.122629
## final value 256.122629
## stopped after 100 iterations
## # weights: 51
## initial value 567.493147
## final value 455.578635
## converged
## # weights: 151
## initial value 660.973565
## final value 455.580203
## converged
## # weights: 251
## initial value 479.169676
## final value 455.581718
## converged
## # weights: 51
## initial value 522.762460
## final value 434.936379
## converged
## # weights: 151
## initial value 522.938491
## final value 434.936379
## converged
## # weights: 251
## initial value 509.874672
## final value 434.936374
## converged
## # weights: 51
## initial value 637.458354
## iter 10 value 434.043432
## iter 20 value 415.097850
## iter 30 value 333.587916
## iter 40 value 306.603436
## iter 50 value 295.271336
## iter 60 value 292.057135
## iter 70 value 291.563434
## iter 80 value 291.417808
## final value 291.412068
## converged
## # weights: 151
## initial value 446.673092
## iter 10 value 435.042967
## iter 20 value 422.375286
## iter 30 value 420.875024
## iter 40 value 389.360684
## iter 50 value 325.742265
## iter 60 value 301.018458
## iter 70 value 299.278137
## iter 80 value 298.009387
## iter 90 value 295.875896
## iter 100 value 295.598544
## final value 295.598544
## stopped after 100 iterations
## # weights: 251
## initial value 516.696583
## iter 10 value 434.966590
## iter 20 value 423.276253
## iter 30 value 369.896372
## iter 40 value 323.698203
## iter 50 value 299.385283
## iter 60 value 295.894656
## iter 70 value 283.755904
## iter 80 value 276.242870
## iter 90 value 259.387671
## iter 100 value 240.993141
## final value 240.993141
## stopped after 100 iterations
## # weights: 51
## initial value 518.458897
## final value 434.937210
## converged
## # weights: 151
## initial value 664.307225
## final value 434.938878
## converged
## # weights: 251
## initial value 536.764023
## final value 434.940302
## converged
## # weights: 51
## initial value 459.366116
## final value 438.928465
## converged
## # weights: 151
## initial value 503.764300
## final value 438.928465
## converged
## # weights: 251
## initial value 501.374916
## final value 438.928465
## converged
## # weights: 51
## initial value 448.736541
## iter 10 value 439.023691
## iter 20 value 439.005116
## final value 438.977003
## converged
## # weights: 151
## initial value 449.536468
## iter 10 value 439.019461
## iter 20 value 438.952983
## iter 20 value 438.952980
## iter 20 value 438.952980
## final value 438.952980
## converged
## # weights: 251
## initial value 635.918159
## iter 10 value 438.948161
## iter 20 value 438.945185
## final value 438.945070
## converged
## # weights: 51
## initial value 628.626219
## final value 438.929375
## converged
## # weights: 151
## initial value 1011.168947
## final value 438.931032
## converged
## # weights: 251
## initial value 655.155804
## final value 438.932626
## converged
## # weights: 51
## initial value 491.026138
## final value 467.863143
## converged
## # weights: 151
## initial value 580.780889
## final value 467.863143
## converged
## # weights: 251
## initial value 482.689955
## final value 467.863143
## converged
## # weights: 51
## initial value 474.575487
## iter 10 value 467.893117
## final value 467.893050
## converged
## # weights: 151
## initial value 650.060584
## iter 10 value 467.892308
## iter 20 value 447.433345
## iter 30 value 416.948255
## iter 40 value 366.608640
## iter 50 value 344.765925
## iter 60 value 305.231182
## iter 70 value 293.779206
## iter 80 value 292.551272
## iter 90 value 286.974906
## iter 100 value 283.353873
## final value 283.353873
## stopped after 100 iterations
## # weights: 251
## initial value 603.971756
## iter 10 value 467.894284
## iter 20 value 462.624294
## iter 30 value 453.159328
## iter 40 value 451.206112
## iter 50 value 448.399122
## iter 60 value 447.864461
## iter 70 value 443.627370
## iter 80 value 429.132637
## iter 90 value 395.672251
## iter 100 value 383.036084
## final value 383.036084
## stopped after 100 iterations
## # weights: 51
## initial value 481.342490
## final value 467.864082
## converged
## # weights: 151
## initial value 511.453045
## final value 467.865689
## converged
## # weights: 251
## initial value 517.230838
## final value 467.867029
## converged
## # weights: 251
## initial value 505.866223
## iter 10 value 451.244156
## iter 20 value 438.579180
## iter 30 value 417.807116
## iter 40 value 376.373469
## iter 50 value 350.190538
## iter 60 value 343.468869
## iter 70 value 341.925235
## iter 80 value 339.059044
## iter 90 value 337.065140
## iter 100 value 336.404754
## final value 336.404754
## stopped after 100 iterationsSummary Neural Net Model
print(germandata.nnet)## Neural Network
##
## 750 samples
## 21 predictor
## 2 classes: '0', '1'
##
## No pre-processing
## Resampling: Bootstrapped (25 reps)
## Summary of sample sizes: 750, 750, 750, 750, 750, 750, ...
## Resampling results across tuning parameters:
##
## size decay Accuracy Kappa
## 1 0e+00 0.7062524 0.00000000
## 1 1e-04 0.7062524 0.00000000
## 1 1e-01 0.7053417 0.16570090
## 3 0e+00 0.7062524 0.00000000
## 3 1e-04 0.7084664 0.01696616
## 3 1e-01 0.7147138 0.26950472
## 5 0e+00 0.7050672 0.01029778
## 5 1e-04 0.7062524 0.00000000
## 5 1e-01 0.7121887 0.29552199
##
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were size = 3 and decay = 0.1.Plotting Neural Net Model
plot(germandata.nnet)
plotnet(germandata.nnet$finalModel, y_names = "response")
title("Graphical Representation of our Neural Network")
Train and Test predictions
pred.glm.gtrain.nn <- predict(germandata.nnet, type = "prob")[,2]
pred.glm.gtest.nn <- predict(germandata.nnet, newdata=germandata.test,type = "prob")[,2]
pred.train <- as.numeric(pred.glm.gtrain.nn > optimal.pcut)
pred.test <- as.numeric(pred.glm.gtest.nn > optimal.pcut)
confusion_matrix_train <- table(germandata.train$response, pred.train)
confusion_matrix_test <- table(germandata.test$response, pred.test)
misclassification_rate_train <- round((confusion_matrix_train[2]+confusion_matrix_train[3])/sum(confusion_matrix_train), 2)
misclassification_rate_test <- round((confusion_matrix_test[2]+confusion_matrix_test[3])/sum(confusion_matrix_test), 2)
cat("train misclassfication rate:", misclassification_rate_train, "| test misclassfication rate:", misclassification_rate_test)## train misclassfication rate: 0.28 | test misclassfication rate: 0.31Train/Test Confusion Matrix
confusion_matrix_test## pred.test
## 0 1
## 0 104 63
## 1 14 69confusion_matrix_train## pred.train
## 0 1
## 0 350 183
## 1 28 189ROC/AUC Curve-Train
par(mfrow=c(1,1))
roc.logit <- roc.plot(x=(germandata.train$response == "1"), pred =pred.glm.gtrain.nn)
roc.logit$roc.vol[2]## Area
## 1 0.8378451ROC/AUC Curve-Test
par(mfrow=c(1,1))
roc.logit.test <- roc.plot(x=(germandata.test$response == "1"), pred =pred.glm.gtest.nn)
roc.logit.test$roc.vol[2]## Area
## 1 0.8045596Train and Test Asymmetric Misclassfication Rate or Asymmetric Misclassification Cost
class.pred.train.nn <- (pred.glm.gtrain.nn>pcut)*1
cost.train <- round(cost(r = germandata.train$response, pi = class.pred.train.nn),2)
class.pred.test.nn<- (pred.glm.gtest.nn>pcut)*1
cost.test <- round(cost(r = germandata.test$response, pi = class.pred.test.nn),2)
cat("total train cost:", cost.train, "| total test cost:", cost.test)## total train cost: 0.43 | total test cost: 0.53