1. Introduction

This is an R Markdown notebook to evaluate how your classification results reflect the real-world. Using a toy dataset, we explain how to compute common metrics that are often used in accuracy assessment.

Most code has been taken from a blog written by Said Bleik, Shaheen Gauher, Data Scientists at Microsoft, which you may visit here

2. Creation of classified and reference toy rasters

library(raster)
## Loading required package: sp

Create a vector object with the classification values:

(clas_val <- c(10,10,10,50,50,20,20,20, 30,30,
                           10,10,10,50,50,20,20,20, 30,30,
                           40,40,40,50,50,50,50,30, 30,30,
                           40,40,40,10,10,50,50,20, 20,30,
                           40,40,40,10,10,20,20,20, 30,30,
                           60,60,60,10,10,50,50,50, 50,50,
                           60,60,20,40,50,50,50,50, 50,50,
                           20,20,20,40,60,60,60,60, 60,30,
                           20,20,40,40,40,40,60,60, 50,30,
                           20,20,40,40,40,40,60,60, 50,30))
##   [1] 10 10 10 50 50 20 20 20 30 30 10 10 10 50 50 20 20 20 30 30 40 40 40 50 50
##  [26] 50 50 30 30 30 40 40 40 10 10 50 50 20 20 30 40 40 40 10 10 20 20 20 30 30
##  [51] 60 60 60 10 10 50 50 50 50 50 60 60 20 40 50 50 50 50 50 50 20 20 20 40 60
##  [76] 60 60 60 60 30 20 20 40 40 40 40 60 60 50 30 20 20 40 40 40 40 60 60 50 30

Create a raster object:

clas <- raster(ncol=10,nrow=10)

Set the raster values:

values(clas) <- clas_val

Define the raster extent:

extent(clas) <- c(0,10, 0,10)

Plot the classsified raster:

plot(clas)
text(clas)

Define the reference values (some people refer to these values as the ground truth):

(ref_val <- c(10,10,50,50,50,20,20,30, 30,30,
              10,10,10,50,50,20,20,30, 30,30,
              40,40,40,50,50,50,50,20, 30,30,
              40,40,40,10,10,50,50,20, 30,30,
              40,40,40,10,10,20,20,20, 20,20,
              60,60,60,40,10,50,50,50, 50,50,
              60,60,60,40,10,50,50,50, 50,50,
              20,20,60,60,60,60,60,60, 50,30,
              20,20,40,40,40,60,60,60, 30,30,
              20,20,40,40,40,40,50,50, 50,50))
##   [1] 10 10 50 50 50 20 20 30 30 30 10 10 10 50 50 20 20 30 30 30 40 40 40 50 50
##  [26] 50 50 20 30 30 40 40 40 10 10 50 50 20 30 30 40 40 40 10 10 20 20 20 20 20
##  [51] 60 60 60 40 10 50 50 50 50 50 60 60 60 40 10 50 50 50 50 50 20 20 60 60 60
##  [76] 60 60 60 50 30 20 20 40 40 40 60 60 60 30 30 20 20 40 40 40 40 50 50 50 50

Copy the raster:

ref <- clas

Set raster reference values:

values(ref) <- ref_val

Plot the reference raster:

plot(ref)
text(ref)

3. Creating the Confusion Matrix

(accmat <- table("pred" = clas_val, "ref" = ref_val))
##     ref
## pred 10 20 30 40 50 60
##   10 10  0  0  1  1  0
##   20  0 14  3  0  0  2
##   30  0  3  9  0  1  0
##   40  0  0  0 17  0  2
##   50  1  0  1  0 21  0
##   60  0  0  0  0  3 11

Overall accuracy is:

(OA <- sum(diag(accmat)) / sum(accmat) * 100)
## [1] 82

Next we will define some basic variables that will be needed to compute the evaluation metrics:

n = sum(accmat) # number of instances
 nc = nrow(accmat) # number of classes
 diag = diag(accmat) # number of correctly classified instances per class 
 rowsums = apply(accmat, 1, sum) # number of instances per class
 colsums = apply(accmat, 2, sum) # number of predictions per class
 p = rowsums / n # distribution of instances over the actual classes
 q = colsums / n # distribution of instances over the predicted classes
(accuracy = sum(diag) / n) 
## [1] 0.82

4. Computing per-class precision, recall, and F-1

 precision = diag / colsums 
 recall = diag / rowsums 
 f1 = 2 * precision * recall / (precision + recall) 
 data.frame(precision, recall, f1)

5. Computing macro-averaged metrics

The per-class metrics can be averaged over all the classes resulting in macro-averaged precision, recall and F-1.

(macroPrecision = mean(precision))
## [1] 0.8183997
(macroRecall = mean(recall))
## [1] 0.8093296
(macroF1 = mean(f1))
## [1] 0.8123889