Multi- Class Classifer Performance Evaluation
Charls Joseph
May 3, 2020
Here, we will see how a multi level classifer can be compared with other classifer in terms of it performance. In binary classifers, we
library("e1071")
data(iris)
attach(iris)
head(iris)## Sepal.Length Sepal.Width Petal.Length Petal.Width Species
## 1 5.1 3.5 1.4 0.2 setosa
## 2 4.9 3.0 1.4 0.2 setosa
## 3 4.7 3.2 1.3 0.2 setosa
## 4 4.6 3.1 1.5 0.2 setosa
## 5 5.0 3.6 1.4 0.2 setosa
## 6 5.4 3.9 1.7 0.4 setosamodel <- svm(Species ~ ., data = iris)
# alternatively the traditional interface:
x <- subset(iris, select = -Species)
y <- Species
model <- svm(x, y)
print(model)##
## Call:
## svm.default(x = x, y = y)
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: radial
## cost: 1
##
## Number of Support Vectors: 51pred <- predict(model, x)
# Check accuracy:
table(pred, y)## y
## pred setosa versicolor virginica
## setosa 50 0 0
## versicolor 0 48 2
## virginica 0 2 48df_glass <- read.csv("C:\\Users\\Charls\\Documents\\CunyMSDS\\Data621\\blog3\\glass.csv")
df_glass$Type <- sapply(df_glass$Type, function(x)
{switch(as.character(x), "1" = "bld_wnd_flt", "2" = "bld_wnd_non_flt", "3" = "veh_wnd_flt", "5" = "cntner" , "6" = "tblwre", "7"= "hdlmp")
})
df_glass$Type <- as.factor(df_glass$Type)
head(df_glass)## RI Na Mg Al Si K Ca Ba Fe Type
## 1 1.52101 13.64 4.49 1.10 71.78 0.06 8.75 0 0.00 bld_wnd_flt
## 2 1.51761 13.89 3.60 1.36 72.73 0.48 7.83 0 0.00 bld_wnd_flt
## 3 1.51618 13.53 3.55 1.54 72.99 0.39 7.78 0 0.00 bld_wnd_flt
## 4 1.51766 13.21 3.69 1.29 72.61 0.57 8.22 0 0.00 bld_wnd_flt
## 5 1.51742 13.27 3.62 1.24 73.08 0.55 8.07 0 0.00 bld_wnd_flt
## 6 1.51596 12.79 3.61 1.62 72.97 0.64 8.07 0 0.26 bld_wnd_fltLets build two multi class classifers.
1st Model is SVM(Support Vector) which is non-parametric Model 2nd Model is Naive Bayes which is parameteric Model
The goal of this excercise is to compare the performance of these two multi level classifers in terms of performance metrices.
Distribution of Dataset
The dataset is imbalanced. Like binary classification, we cant rely completly on accuracy metrics. We have to use precision and recall metrics in order to evalulate the performance of the classifer output.
Since this is multi class labels, each label has its own precision, recall and f1 score. There are 3 ways to determine the precision, recall and f1 score for overall results.
- Macro Precision, Recall and F1 Score: This is determined by simply taking average of precision of all lables
- Macro Weighted Precision, Recall and F1 Score. This is calculated by the wighted average of precision of all lables
- Micro precision, recall and F1 score This is calculated by taking all positive prediction(diagonal values) divided by sum of all negative prediction. Here Precision = Recall = F1 Score.
bp <- barplot(table(df_glass$Type), beside = TRUE, main = "Total observations",
col = c("#ae1f85", "#262ad9", "#891fb0", "#fff3e1", "#2dda31", "#de5246"),
xlab = "Label class", names = c("bld_wnd_flt", "bld_n_flt", "veh_wnd_flt", "cntner", "tblwre", "hdlmp"),
ylab = "# of Obsevations", legend = c("bld_wnd_flt", "bld_n_flt", "veh_wnd_flt", "tblwre", "cntner", "hdlmp"),
args.legend = list(title = "Legend", x = "topright", cex = .7), ylim = c(0, 300))
text(bp, 0, round(table(df_glass$Type), 1),cex=1,pos=3) 
Split training and test dataset
library(caTools)
set.seed(1288)
split = sample.split(df_glass$Type, SplitRatio = 0.6)
df_train = subset(df_glass, split == TRUE)
df_test= subset(df_glass, split == FALSE)
x <- subset(df_test, select = -Type)
y <- df_test$TypeModel1 : SVM - Support Vector Model
svm_model <- svm(Type ~ ., data = df_train)
# predict with test data
pred <- predict(svm_model, x)
cm_svm = as.matrix(table(Predicted = pred, Actual = y)) # create the confusion matrix
cm_svm## Actual
## Predicted bld_wnd_flt bld_wnd_non_flt cntner hdlmp tblwre veh_wnd_flt
## bld_wnd_flt 24 9 0 0 0 6
## bld_wnd_non_flt 4 20 2 1 1 1
## cntner 0 1 3 0 1 0
## hdlmp 0 0 0 11 0 0
## tblwre 0 0 0 0 2 0
## veh_wnd_flt 0 0 0 0 0 0Model2 : Naive Bayes - Naive Bayes Model
nb_model=naiveBayes(as.factor(df_train$Type) ~ ., data=df_train)
# predict with test data
pred <- predict(nb_model, x)
cm_nb = as.matrix(table(Predicted = pred, Actual = y)) # create the confusion matrix
cm_nb## Actual
## Predicted bld_wnd_flt bld_wnd_non_flt cntner hdlmp tblwre veh_wnd_flt
## bld_wnd_flt 2 1 0 0 0 1
## bld_wnd_non_flt 1 2 1 0 0 0
## cntner 0 3 3 1 0 0
## hdlmp 0 0 1 11 0 0
## tblwre 1 2 0 0 4 0
## veh_wnd_flt 24 22 0 0 0 6Naive Bayes Model performance Metrics
n = sum(cm_nb) # number of instances
nc = nrow(cm_nb) # number of classes
diag = diag(cm_nb) # number of correctly classified instances per class
colsums = apply(cm_nb, 1, sum) # number of instances per class
rowsums = apply(cm_nb, 2, sum) # number of predictions per class
p = colsums / n # distribution of instances over the actual classes
q = rowsums / n # distribution of instances over the predicted classes
accuracy = sum(diag) / n
precision = diag / rowsums
recall = diag / colsums
f1 = 2 * precision * recall / (precision + recall)
accuracy## [1] 0.3255814data.frame(precision, recall, f1) ## precision recall f1
## bld_wnd_flt 0.07142857 0.5000000 0.1250000
## bld_wnd_non_flt 0.06666667 0.5000000 0.1176471
## cntner 0.60000000 0.4285714 0.5000000
## hdlmp 0.91666667 0.9166667 0.9166667
## tblwre 1.00000000 0.5714286 0.7272727
## veh_wnd_flt 0.85714286 0.1153846 0.2033898Macro Averged Precision, Recall and F1 Score
macroPrecision = mean(precision)
macroRecall = mean(recall)
macroF1 = mean(f1)
data.frame(macroPrecision, macroRecall, macroF1)## macroPrecision macroRecall macroF1
## 1 0.5853175 0.5053419 0.4316627Macro Weighted Average Precision, Recall and F1 Score
wtmacroPrecision = weighted.mean(precision,w = colsums)
wtmacroRecall = weighted.mean(recall,w = colsums)
wtmacroF1 = weighted.mean(f1,w = colsums)
data.frame(wtmacroPrecision, wtmacroRecall, wtmacroF1)## wtmacroPrecision wtmacroRecall wtmacroF1
## 1 0.782835 0.3255814 0.3620671Micro Precision, recall and f1 score
microprecision <- sum(diag)/n
microrecall <- sum(diag)/n
f1 <- 2*microprecision*microrecall/(microprecision + microrecall)
data.frame(microprecision, microrecall, f1)## microprecision microrecall f1
## 1 0.3255814 0.3255814 0.3255814SVM model Metrics
Lets do the same for model 1
n = sum(cm_svm) # number of instances
nc = nrow(cm_svm) # number of classes
diag = diag(cm_svm) # number of correctly classified instances per class
colsums = apply(cm_svm, 1, sum) # number of instances per class
rowsums = apply(cm_svm, 2, sum) # number of predictions per class
p = colsums / n # distribution of instances over the actual classes
q = rowsums / n # distribution of instances over the predicted classes
accuracy = sum(diag) / n
precision = diag / rowsums
precision<-ifelse(is.nan(precision),0,precision)
recall = diag / colsums
recall<-ifelse(is.nan(recall),0,recall)
f1 = 2 * precision * recall / (precision + recall)
f1<-ifelse(is.nan(f1),0,f1)
accuracy## [1] 0.6976744data.frame(precision, recall, f1) ## precision recall f1
## bld_wnd_flt 0.8571429 0.6153846 0.7164179
## bld_wnd_non_flt 0.6666667 0.6896552 0.6779661
## cntner 0.6000000 0.6000000 0.6000000
## hdlmp 0.9166667 1.0000000 0.9565217
## tblwre 0.5000000 1.0000000 0.6666667
## veh_wnd_flt 0.0000000 0.0000000 0.0000000Macro Averged Precision, Recall and F1 Score
macroPrecision = mean(precision)
macroRecall = mean(recall)
macroF1 = mean(f1)
data.frame(macroPrecision, macroRecall, macroF1)## macroPrecision macroRecall macroF1
## 1 0.5900794 0.65084 0.6029287Micro Precision, recall and f1 score
microprecision <- sum(diag)/n
microrecall <- sum(diag)/n
f1 <- 2*microprecision*microrecall/(microprecision + microrecall)
data.frame(microprecision, microrecall, f1)## microprecision microrecall f1
## 1 0.6976744 0.6976744 0.6976744From the above metrics, The SVM model has good overall f1 score, precision and recall.