JISA 2018

Selecting CNN best_model

Executing Rscript evaluate_dga_classifier.R --modelid=1 --tune --experimenttag=cnn_tune_sanson

Listing available datasets

ls -1 datasets/JISA2018*.csv.gz

datasets/JISA2018.csv.gz
datasets/JISA2018-cz-dataset.csv.gz
datasets/JISA2018-dganormal-dataset.csv.gz
datasets/JISA2018-dganormal-dataset-full.csv.gz
datasets/JISA2018-std-dataset.csv.gz

The standard-dataset refers to the dataset used in the argencon paper (OSINT+ALEXA)

The dataset do not contain neither .cz domains nor normaldga

Creating the model using standard datasets (i.e. OSINT and ALEXA)

Preparing dataset

source("preprocess.R")
datasets<-build_train_test(datasetfile = "datasets/JISA2018-std-dataset.csv.gz",maxlen = 45)

selected_parameters=list(
  nb_filter = 128,
  kernel_size = 8,
  embedingdim = 100,
  hidden_size = 1024
)

apply 5-folds Cross validation

result_cv$result 

result_cv$result %>%  filter(metric=="Sensitivity" | metric == "Specificity" | metric=="F1" | metric=="Balanced Accuracy") %>%
  ggplot()+
  geom_boxplot(aes(x=metric,y=value,fill=metric))+
  theme_bw()+
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

Applying a 2-fold evaluation (train/test)

Required for saving the model and use it in further experiments

summary(std_dataset_model)

____________________________________________________________________________________________________________________________________________________________________________
Layer (type)                                                                 Output Shape                                                         Param #                   
============================================================================================================================================================================
input_1 (InputLayer)                                                         (None, 45)                                                           0                         
____________________________________________________________________________________________________________________________________________________________________________
embedding_1 (Embedding)                                                      (None, 45, 100)                                                      4000                      
____________________________________________________________________________________________________________________________________________________________________________
conv1d_1 (Conv1D)                                                            (None, 38, 128)                                                      102528                    
____________________________________________________________________________________________________________________________________________________________________________
flatten_1 (Flatten)                                                          (None, 4864)                                                         0                         
____________________________________________________________________________________________________________________________________________________________________________
dense_1 (Dense)                                                              (None, 1024)                                                         4981760                   
____________________________________________________________________________________________________________________________________________________________________________
dense_2 (Dense)                                                              (None, 1)                                                            1025                      
============================================================================================================================================================================
Total params: 5,089,313
Trainable params: 5,089,313
Non-trainable params: 0
____________________________________________________________________________________________________________________________________________________________________________

Testing on normal cz domains (i.e. all domains are normal)

cz_dataset<-read_csv("./datasets/JISA2018-cz-dataset.csv.gz")
cz_dataset %>% group_by(label) %>% summarise(n=n())
cz_dataset$domain1<-str_split(cz_dataset$domain,"\\.",simplify = T)[,1]
cz_dataset_tokenized<-build_dataset(as.matrix(cz_dataset),maxlen=45)
cz_dataset_x<-cz_dataset_tokenized$encode
cz_dataset_y<-ifelse(grepl("normal",cz_dataset_tokenized$label) ,0,1)

Calcualting Recall per class (0,1)

preds<-get_predictions(std_dataset_model,cz_dataset_x,threshold = 0.9 )
table(preds) %>% data.frame() %>% mutate(total=sum(Freq),recall=Freq/total) %>% select(-total,-Freq) %>% reshape2::melt() %>%
  
  
  ggplot()+
  geom_col(aes(x=variable,y=value,fill=preds))+
  theme_bw()

Using preds as id variables

Printing Domains Not correctly detected as normal

data.frame(predicted_class=preds,class=cz_dataset_y,domain=cz_dataset_tokenized$domain) %>% filter (predicted_class==1) %>% select(domain)

Testing on normal dga domains

dga_normal_dataset<-read_csv("./datasets/JISA2018-dganormal-dataset-full.csv.gz")
dga_normal_dataset %>% group_by(label) %>% summarise(n=n())
dga_normal_dataset$domain1<-str_split(dga_normal_dataset$domain,"\\.",simplify = T)[,1]
dga_normal_dataset_tokenized<-build_dataset(as.matrix(dga_normal_dataset),maxlen=45)
dga_normal_dataset_x<-dga_normal_dataset_tokenized$encode
dga_normal_dataset_y<-ifelse(grepl("normal",dga_normal_dataset_tokenized$label) ,0,1)

Calculating (ploting) Recall per class (0,1)

preds<-get_predictions(std_dataset_model,dga_normal_dataset_x,threshold = 0.9 )
table(preds) %>% data.frame() %>% mutate(total=sum(Freq),recall=Freq/total) %>% select(-total,-Freq) %>% reshape2::melt() %>%
  ggplot()+
  geom_col(aes(x=variable,y=value,fill=preds))+
  theme_bw()

Using preds as id variables

Printing Domains Not correctly detected as normal

Plotin recall by normalDga type

calculate_recall(data.frame(label=dga_normal_dataset_tokenized$label, class=dga_normal_dataset_y,predicted_class=preds))%>%
  ggplot()+
  geom_col(aes(x=label,y=recall,fill=label))+
  theme_bw()+ylab("Sensitivity (AKA Recall)")+
  theme(axis.text.x = element_text(angle = 45, hjust = 1))+
  guides(color=FALSE)