Random Forest CI

library(caret)
library(doMC)
library(ranger)
library(dplyr)
library(ggplot2)
library(readr)

Dataset for botnet detection

dataset_botnet<-read_csv("https://www.dropbox.com/s/0ziry2zi6gsgpi6/ctu19_filtered_fv.csv?dl=1")
dataset_botnet$label<-dataset_botnet$label %>% as.factor()
test<-dataset_botnet %>% sample_frac(0.2)
train<-setdiff(dataset_botnet,test)

Let’s try 5 models using different numbers of trees using Crossvalidation 3x10

registerDoMC(cores = 4)
ctrl_fast <- trainControl(
  method = "repeatedcv",
  repeats = 3,
  number = 10,
  returnResamp = 'final',
  savePredictions = 'final',
  verboseIter = F,
  classProbs = TRUE,
  allowParallel = T
)
bot_formula<-formula(label ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl)

tgrid <- expand.grid(
  mtry = 3,
  splitrule = "gini",
  min.node.size = 10
)

rfFit10 <- caret::train(bot_formula,
  data = train,
  method = "ranger",
  tuneGrid = tgrid,
  verbose = 2,
  trControl = ctrl_fast,
  num.trees = 10
)

rfFit50 <- caret::train(bot_formula,
  data = train,
  method = "ranger",
  tuneGrid = tgrid,
  verbose = 2,
  trControl = ctrl_fast,
  num.trees = 50
)

rfFit100 <- caret::train(bot_formula,
  data = train,
  method = "ranger",
  tuneGrid = tgrid,
  verbose = 2,
  trControl = ctrl_fast,
  num.trees = 100
)

rfFit500 <- caret::train(bot_formula,
  data = train,
  method = "ranger",
  tuneGrid = tgrid,
  verbose = 2,
  trControl = ctrl_fast,
  num.trees = 500
)


rfFit1000 <- caret::train(bot_formula,
  data = train,
  method = "ranger",
  tuneGrid = tgrid,
  verbose = 2,
  trControl = ctrl_fast,
  num.trees = 1000
)

results<-rbind(rfFit10$results,
               rfFit50$results,
               rfFit100$results,
               rfFit500$results,
               rfFit1000$results)
results<-results %>% tibble::add_column(num.trees=c(10,50,100,500,1000))

results

Plot accuracy and standard deviation

results %>%
  ggplot(aes(x = num.trees %>% as.factor(), y = Accuracy)) +
  geom_point(color = 'red') +
  geom_errorbar(
    aes(ymin = Accuracy - AccuracySD, ymax = Accuracy + AccuracySD),
    width = .02,
    color = 'yellow'
  ) +
  ggdark::dark_theme_bw() +
  xlab("num.trees")+
  labs(title="RF: Mean and Standard deviation after hyper-parameter num.trees tuning")+
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

NA
NA

A function for using ranger se and some other calculations

ranger_rf<-function(n=10,train,test){
  rf <- ranger(label ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl, 
               train, 
               num.trees = n, 
               keep.inbag = TRUE, 
               probability = TRUE)
  pred <- predict(rf, test, type = "se")
  predictions<-pred$predictions %>% 
    as.data.frame() %>% mutate(prob=Botnet)
  predictions$class<-names(pred$predictions %>% 
                             as.data.frame())[max.col(pred$predictions)] 
  predictions$obs<-test$label
  predictions<-predictions %>% 
    select(prob,class,obs) %>% 
    cbind(pred$se %>% as.data.frame()) 
  predictions$se<-predictions %>% 
    apply(MARGIN=1,function(x) x[x['class']])
  predictions<-predictions %>% 
    select(prob,class,se,obs)
  predictions<-predictions %>% 
    mutate(error=ifelse(class!=obs,"uncorrect","correct"))
  accuracy<-predictions %>% 
    summarise(acccuracy=sum(error=='correct')/n()) 
  sensitivity<-predictions %>% 
    summarise( sensitivity=sum( class== 'Botnet' & obs=='Botnet')/sum(obs=='Botnet'))
  specificity<-predictions %>% 
    summarise( sensitivity=sum( class== 'Normal' & obs=='Normal')/sum(obs=='Normal'))
  list(acc = accuracy,
       sens = sensitivity,
       spec = specificity,
       preds = predictions)
}

Now, let’s calculate the SE using Wager’s approach.

rf_10   <- ranger_rf(10,train,test)
rf_50   <- ranger_rf(50,train,test)
rf_100  <- ranger_rf(100,train,test)
rf_500  <- ranger_rf(500,train,test)
rf_1000 <- ranger_rf(1000,train,test)

rf_n10_plot<-ggplot(data=rf_10$preds,aes(x=prob,y=as.numeric(se)))   +
  geom_point(aes(color=error),size=1,alpha=0.5)+
  geom_smooth(se = F,color='red',method = 'loess')+
  labs(caption = paste("Sensitivity:",rf_10$sens,"\nSpecificty:",rf_10$spec,"\nAccuracy:",rf_10$acc))+
  xlim(0,1)+
  ylim(0,3)+
  scale_color_manual(values=c('skyblue','orange'))+
  ggdark::dark_theme_bw()



rf_n50_plot<-ggplot(data=rf_50$preds,aes(x=prob,y=as.numeric(se)))   +
  geom_point(aes(color=error),size=1,alpha=0.5)+
  geom_smooth(se = F,color='red',method = 'loess')+
  labs(caption = paste("Sensitivity:",rf_10$sens,"\nSpecificty:",rf_10$spec,"\nAccuracy:",rf_10$acc))+
  xlim(0,1)+
  ylim(0,1)+
  scale_color_manual(values=c('skyblue','orange'))+
  ggdark::dark_theme_bw()


rf_n100_plot<-ggplot(data=rf_100$preds,aes(x=prob,y=as.numeric(se)))   +
  geom_point(aes(color=error),size=1,alpha=0.5)+
  geom_smooth(se = F,color='red',method = 'loess')+
  labs(caption = paste("Sensitivity:",rf_10$sens,"\nSpecificty:",rf_10$spec,"\nAccuracy:",rf_10$acc))+
  xlim(0,1)+
  ylim(0,1)+
  scale_color_manual(values=c('skyblue','orange'))+
  ggdark::dark_theme_bw()

rf_n500_plot<-ggplot(data=rf_500$preds,aes(x=prob,y=as.numeric(se)))   +
  geom_point(aes(color=error),size=1,alpha=0.5)+
  geom_smooth(se = F,color='red',method = 'loess')+
  labs(caption = paste("Sensitivity:",rf_10$sens,"\nSpecificty:",rf_10$spec,"\nAccuracy:",rf_10$acc))+
  xlim(0,1)+
  ylim(0,1)+
  scale_color_manual(values=c('skyblue','orange'))+
  ggdark::dark_theme_bw()

rf_n1000_plot<-ggplot(data=rf_1000$preds,aes(x=prob,y=as.numeric(se)))   +
  geom_point(aes(color=error),size=1,alpha=0.5)+
  geom_smooth(se = F,color='red',method = 'loess')+
  labs(caption = paste("Sensitivity:",rf_10$sens,"\nSpecificty:",rf_10$spec,"\nAccuracy:",rf_10$acc))+
  xlim(0,1)+
  ylim(0,1)+
  scale_color_manual(values=c('skyblue','orange'))+
  ggdark::dark_theme_bw()

gridExtra::grid.arrange(rf_n50_plot+theme(legend.position = "none")+labs(title="RF ntree=50")+ylab("se"),
                        rf_n100_plot+theme(legend.position = "none")+labs(title="RF ntree=100")+ylab("se"),
                        rf_n500_plot+theme(legend.position = "none")+labs(title="RF ntree=500")+ylab("se"),
                        rf_n1000_plot+theme(legend.position = "none")+labs(title="RF ntree=1000")+ylab("se"),
                        ncol=4)