Carregando os dados

train       <- read.csv("./data/train.csv", encoding="UTF-8")
test        <- read.csv("./data/test.csv", encoding="UTF-8")
submission  <- read.csv("./data/sample_submission.csv")
# Pre-processamento dos dados

## Retirando variáveis que identificamos como irrelevantes
train <- train %>% select(-cargo, -nome, -ocupacao, -sexo, -total_despesa, -total_receita, -sequencial_candidato)
test <- test %>% select(-cargo, -nome, -ocupacao, total_despesa, -total_receita)

## Mudando os valores de NA para zero
train[is.na(train)] <- 0
test[is.na(test)] <- 0
## Há desbalanceamento das classes (isto é, uma classe tem muito mais instâncias que outra)? Em que proporção? Quais efeitos colaterais o desbalanceamento de classes pode causar no classificador? Como você poderia tratar isso?
r train %>% group_by(situacao) %>% summarise(nao_eleitos=(n()/nrow(train)) * 100) %>% ggplot(aes(x=situacao, y=nao_eleitos, fill=situacao)) + geom_col() + labs(x = "Situação", y = "Percentual")
Existe um grande desbalanceamento entre as classes, como mostra o gráfico acima.Isso interfere na acuracia do modelo, uma vez que o clasificador vai tender para nao_eleito.
Para tratar isso, é necessário utilizar alguma técnica de rebalanceamento. Mas, dados gerados pelo excesso de amostragem prevêem a quantidade de observações repetidas. Os dados sintéticos gerados pelo ROSE para balencear são melhores pois fornecem uma melhor estimativa dos dados oiginais.
r balanced_data <- ROSE(situacao ~ ., data = train, seed = 1)$data table(balanced_data$situacao)
## ## nao_eleito eleito ## 3842 3780 Agora, vemos que os dados estão equilibrados.

Treine: um modelo de KNN, regressão logística, uma árvore de decisão e um modelo de adaboost. Tune esses modelos usando validação cruzada e controle overfitting se necessário, considerando as particularidades de cada modelo.

Modelo KNN

k_grid <- expand.grid(k = seq(1,25, length=25))

knn_control <- trainControl(method = "repeatedcv", 
                           number = 10, 
                           repeats = 5)  

model.knn <- caret::train(situacao ~ ., 
                     data = balanced_data, 
                     method = "knn", 
                     tuneGrid = k_grid,
                     preProcess = c("center", "scale"),
                     trControl = knn_control)
model.knn
## k-Nearest Neighbors 
## 
## 7622 samples
##   16 predictor
##    2 classes: 'nao_eleito', 'eleito' 
## 
## Pre-processing: centered (78), scaled (78) 
## Resampling: Cross-Validated (10 fold, repeated 5 times) 
## Summary of sample sizes: 6860, 6859, 6860, 6860, 6860, 6860, ... 
## Resampling results across tuning parameters:
## 
##   k   Accuracy   Kappa    
##    1  0.8823414  0.7644461
##    2  0.8572842  0.7142833
##    3  0.8559198  0.7116014
##    4  0.8423799  0.6845241
##    5  0.8359501  0.6716880
##    6  0.8248767  0.6495275
##    7  0.8215431  0.6428542
##    8  0.8123602  0.6244672
##    9  0.8110738  0.6218920
##   10  0.8054058  0.6105694
##   11  0.8012337  0.6022266
##   12  0.7955397  0.5908258
##   13  0.7934923  0.5867223
##   14  0.7904751  0.5806921
##   15  0.7866179  0.5729440
##   16  0.7851222  0.5699551
##   17  0.7841780  0.5680752
##   18  0.7831015  0.5659248
##   19  0.7823402  0.5644056
##   20  0.7801102  0.5599304
##   21  0.7795592  0.5588293
##   22  0.7790605  0.5578300
##   23  0.7784574  0.5566189
##   24  0.7764891  0.5526837
##   25  0.7751510  0.5499983
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was k = 1.

Modelo de regressão logística

modelo.reg <- glm(formula = situacao ~., 
                  data=balanced_data, 
                  family="binomial")

plot(modelo.reg)

Árvore de decisão

trctrl <- trainControl(method = "repeatedcv", number = 10, repeats = 5)
set.seed(3333)

model.tree <- caret::train(situacao ~., data = balanced_data, method = "rpart",
                   trControl = trctrl,
                   tuneLength = 10)
model.tree
## CART 
## 
## 7622 samples
##   16 predictor
##    2 classes: 'nao_eleito', 'eleito' 
## 
## No pre-processing
## Resampling: Cross-Validated (10 fold, repeated 5 times) 
## Summary of sample sizes: 6860, 6859, 6860, 6860, 6860, 6859, ... 
## Resampling results across tuning parameters:
## 
##   cp           Accuracy   Kappa    
##   0.003174603  0.9490155  0.8980694
##   0.003227513  0.9490155  0.8980694
##   0.004232804  0.9458931  0.8918316
##   0.005202822  0.9405403  0.8811420
##   0.010052910  0.9304370  0.8609620
##   0.016666667  0.9259241  0.8519262
##   0.044973545  0.9078974  0.8158212
##   0.057671958  0.8843601  0.7686401
##   0.151058201  0.8334227  0.6662590
##   0.593915344  0.6455407  0.2864130
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was cp = 0.003227513.

Adaboost

ada_grid    <- expand.grid(nIter = seq(1, 2, length = 2),
                           method = c("Adaboost.M1", "Real adaboost"))

ada_control <- trainControl(method = "repeatedcv",
                            sampling = "smote",
                            number = 3,
                            repeats = 3,
                            verboseIter = TRUE,
                            classProbs = TRUE)

modelo.ada <- caret::train(situacao ~.,
                            data=balanced_data,
                            method = "adaboost",
                            preProcess = c("scale", "center", "nzv"),
                            trControl = ada_control,
                            tuneGrid = ada_grid)
## + Fold1.Rep1: nIter=1, method=Adaboost.M1 
## - Fold1.Rep1: nIter=1, method=Adaboost.M1 
## + Fold1.Rep1: nIter=2, method=Adaboost.M1 
## - Fold1.Rep1: nIter=2, method=Adaboost.M1 
## + Fold1.Rep1: nIter=1, method=Real adaboost 
## - Fold1.Rep1: nIter=1, method=Real adaboost 
## + Fold1.Rep1: nIter=2, method=Real adaboost 
## - Fold1.Rep1: nIter=2, method=Real adaboost 
## + Fold2.Rep1: nIter=1, method=Adaboost.M1 
## - Fold2.Rep1: nIter=1, method=Adaboost.M1 
## + Fold2.Rep1: nIter=2, method=Adaboost.M1 
## - Fold2.Rep1: nIter=2, method=Adaboost.M1 
## + Fold2.Rep1: nIter=1, method=Real adaboost 
## - Fold2.Rep1: nIter=1, method=Real adaboost 
## + Fold2.Rep1: nIter=2, method=Real adaboost 
## - Fold2.Rep1: nIter=2, method=Real adaboost 
## + Fold3.Rep1: nIter=1, method=Adaboost.M1 
## - Fold3.Rep1: nIter=1, method=Adaboost.M1 
## + Fold3.Rep1: nIter=2, method=Adaboost.M1 
## - Fold3.Rep1: nIter=2, method=Adaboost.M1 
## + Fold3.Rep1: nIter=1, method=Real adaboost 
## - Fold3.Rep1: nIter=1, method=Real adaboost 
## + Fold3.Rep1: nIter=2, method=Real adaboost 
## - Fold3.Rep1: nIter=2, method=Real adaboost 
## + Fold1.Rep2: nIter=1, method=Adaboost.M1 
## - Fold1.Rep2: nIter=1, method=Adaboost.M1 
## + Fold1.Rep2: nIter=2, method=Adaboost.M1 
## - Fold1.Rep2: nIter=2, method=Adaboost.M1 
## + Fold1.Rep2: nIter=1, method=Real adaboost 
## - Fold1.Rep2: nIter=1, method=Real adaboost 
## + Fold1.Rep2: nIter=2, method=Real adaboost 
## - Fold1.Rep2: nIter=2, method=Real adaboost 
## + Fold2.Rep2: nIter=1, method=Adaboost.M1 
## - Fold2.Rep2: nIter=1, method=Adaboost.M1 
## + Fold2.Rep2: nIter=2, method=Adaboost.M1 
## - Fold2.Rep2: nIter=2, method=Adaboost.M1 
## + Fold2.Rep2: nIter=1, method=Real adaboost 
## - Fold2.Rep2: nIter=1, method=Real adaboost 
## + Fold2.Rep2: nIter=2, method=Real adaboost 
## - Fold2.Rep2: nIter=2, method=Real adaboost 
## + Fold3.Rep2: nIter=1, method=Adaboost.M1 
## - Fold3.Rep2: nIter=1, method=Adaboost.M1 
## + Fold3.Rep2: nIter=2, method=Adaboost.M1 
## - Fold3.Rep2: nIter=2, method=Adaboost.M1 
## + Fold3.Rep2: nIter=1, method=Real adaboost 
## - Fold3.Rep2: nIter=1, method=Real adaboost 
## + Fold3.Rep2: nIter=2, method=Real adaboost 
## - Fold3.Rep2: nIter=2, method=Real adaboost 
## + Fold1.Rep3: nIter=1, method=Adaboost.M1 
## - Fold1.Rep3: nIter=1, method=Adaboost.M1 
## + Fold1.Rep3: nIter=2, method=Adaboost.M1 
## - Fold1.Rep3: nIter=2, method=Adaboost.M1 
## + Fold1.Rep3: nIter=1, method=Real adaboost 
## - Fold1.Rep3: nIter=1, method=Real adaboost 
## + Fold1.Rep3: nIter=2, method=Real adaboost 
## - Fold1.Rep3: nIter=2, method=Real adaboost 
## + Fold2.Rep3: nIter=1, method=Adaboost.M1 
## - Fold2.Rep3: nIter=1, method=Adaboost.M1 
## + Fold2.Rep3: nIter=2, method=Adaboost.M1 
## - Fold2.Rep3: nIter=2, method=Adaboost.M1 
## + Fold2.Rep3: nIter=1, method=Real adaboost 
## - Fold2.Rep3: nIter=1, method=Real adaboost 
## + Fold2.Rep3: nIter=2, method=Real adaboost 
## - Fold2.Rep3: nIter=2, method=Real adaboost 
## + Fold3.Rep3: nIter=1, method=Adaboost.M1 
## - Fold3.Rep3: nIter=1, method=Adaboost.M1 
## + Fold3.Rep3: nIter=2, method=Adaboost.M1 
## - Fold3.Rep3: nIter=2, method=Adaboost.M1 
## + Fold3.Rep3: nIter=1, method=Real adaboost 
## - Fold3.Rep3: nIter=1, method=Real adaboost 
## + Fold3.Rep3: nIter=2, method=Real adaboost 
## - Fold3.Rep3: nIter=2, method=Real adaboost 
## Aggregating results
## Selecting tuning parameters
## Fitting nIter = 2, method = Real adaboost on full training set
modelo.ada
## AdaBoost Classification Trees 
## 
## 7622 samples
##   16 predictor
##    2 classes: 'nao_eleito', 'eleito' 
## 
## Pre-processing: scaled (25), centered (25), remove (53) 
## Resampling: Cross-Validated (3 fold, repeated 3 times) 
## Summary of sample sizes: 5081, 5081, 5082, 5081, 5082, 5081, ... 
## Addtional sampling using SMOTE prior to pre-processing
## 
## Resampling results across tuning parameters:
## 
##   nIter  method         Accuracy   Kappa    
##   1      Adaboost.M1    0.9532940  0.9065894
##   1      Real adaboost  0.9504514  0.9008927
##   2      Adaboost.M1    0.9566607  0.9133670
##   2      Real adaboost  0.9592410  0.9185068
## 
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were nIter = 2 and method =
##  Real adaboost.
## Reporte precision, recall e f-measure no treino e validação. Há uma grande diferença de desempenho no treino/validação? Como você avalia os resultados? Justifique sua resposta.
Podemos definir a precision como TP / (TP + FP), sendo TP: True Positives e FP: False negatives.
r precision <- function(TP, FP) { result <- TP / (TP + FP) return(result) }
Podemos definir recall como TP / (TP + FN), sendo FN: False Negatives.
r recall <- function(TP,FN) { result <- TP / (TP + FN) return(result) }
Podemos definir f-measure como a média ponderada entre precision e recall, da forma 2 * (precision * recall) / (precision + recall).
r fmeasure <- function(precision, recall) { result <- 2 * (precision * recall) / (precision + recall) }
Agora, vamos analisar:
#### KNN
r matriz.knn <- confusionMatrix(model.knn) knn.precision <- precision(matriz.knn$table[1], matriz.knn$table[3]) knn.recall <- recall(matriz.knn$table[1], matriz.knn$table[2]) knn.fmeasure <- fmeasure(knn.precision, knn.recall)
– Precision
r knn.precision
## [1] 0.8442585
– Recall
r knn.recall
## [1] 0.9399792
– F-measure
r knn.fmeasure
## [1] 0.8895512
#### Árvore de decisão
r matriz.tree <- confusionMatrix(model.tree) tree.precision <- precision(matriz.tree$table[1], matriz.tree$table[3]) tree.recall <- recall(matriz.tree$table[1], matriz.tree$table[2]) tree.fmeasure <- fmeasure(tree.precision, tree.recall)
– Precision
r tree.precision
## [1] 0.9739775
– Recall
r tree.recall
## [1] 0.9235294
– F-measure
r tree.fmeasure
## [1] 0.9480828
#### Adaboost
r matriz.ada <- confusionMatrix(modelo.ada) ada.precision <- precision(matriz.ada$table[1], matriz.ada$table[3]) ada.recall <- recall(matriz.ada$table[1], matriz.ada$table[2]) ada.fmeasure <- fmeasure(ada.precision, ada.recall)
– Precision
r ada.precision
## [1] 0.9804099
– Recall
r ada.recall
## [1] 0.9378796
– F-measure
r ada.fmeasure
## [1] 0.9586733

Interprete as saídas dos modelos. Quais atributos parecem ser mais importantes de acordo com cada modelo?

KNN

ggplot(varImp(model.knn))

Regressão Linear

ggplot(varImp(modelo.reg))

Árvore de Decisão

ggplot(varImp(model.tree))

Adaboost

ggplot(varImp(modelo.ada))

Envie seus melhores modelos à competição do Kaggle. Faça pelo menos uma submissão.

best_grid <- expand.grid(modelo.ada$bestTune)
model.best <- caret::train(situacao ~ .,
                    balanced_data,
                    method = "adaboost",
                    trControl = trainControl(verboseIter = TRUE),
                    tuneGrid = best_grid)
## + Resample01: nIter=2, method=Real adaboost 
## - Resample01: nIter=2, method=Real adaboost 
## + Resample02: nIter=2, method=Real adaboost 
## - Resample02: nIter=2, method=Real adaboost 
## + Resample03: nIter=2, method=Real adaboost 
## - Resample03: nIter=2, method=Real adaboost 
## + Resample04: nIter=2, method=Real adaboost 
## - Resample04: nIter=2, method=Real adaboost 
## + Resample05: nIter=2, method=Real adaboost 
## - Resample05: nIter=2, method=Real adaboost 
## + Resample06: nIter=2, method=Real adaboost 
## - Resample06: nIter=2, method=Real adaboost 
## + Resample07: nIter=2, method=Real adaboost 
## - Resample07: nIter=2, method=Real adaboost 
## + Resample08: nIter=2, method=Real adaboost 
## - Resample08: nIter=2, method=Real adaboost 
## + Resample09: nIter=2, method=Real adaboost 
## - Resample09: nIter=2, method=Real adaboost 
## + Resample10: nIter=2, method=Real adaboost 
## - Resample10: nIter=2, method=Real adaboost 
## + Resample11: nIter=2, method=Real adaboost 
## - Resample11: nIter=2, method=Real adaboost 
## + Resample12: nIter=2, method=Real adaboost 
## - Resample12: nIter=2, method=Real adaboost 
## + Resample13: nIter=2, method=Real adaboost 
## - Resample13: nIter=2, method=Real adaboost 
## + Resample14: nIter=2, method=Real adaboost 
## - Resample14: nIter=2, method=Real adaboost 
## + Resample15: nIter=2, method=Real adaboost 
## - Resample15: nIter=2, method=Real adaboost 
## + Resample16: nIter=2, method=Real adaboost 
## - Resample16: nIter=2, method=Real adaboost 
## + Resample17: nIter=2, method=Real adaboost 
## - Resample17: nIter=2, method=Real adaboost 
## + Resample18: nIter=2, method=Real adaboost 
## - Resample18: nIter=2, method=Real adaboost 
## + Resample19: nIter=2, method=Real adaboost 
## - Resample19: nIter=2, method=Real adaboost 
## + Resample20: nIter=2, method=Real adaboost 
## - Resample20: nIter=2, method=Real adaboost 
## + Resample21: nIter=2, method=Real adaboost 
## - Resample21: nIter=2, method=Real adaboost 
## + Resample22: nIter=2, method=Real adaboost 
## - Resample22: nIter=2, method=Real adaboost 
## + Resample23: nIter=2, method=Real adaboost 
## - Resample23: nIter=2, method=Real adaboost 
## + Resample24: nIter=2, method=Real adaboost 
## - Resample24: nIter=2, method=Real adaboost 
## + Resample25: nIter=2, method=Real adaboost 
## - Resample25: nIter=2, method=Real adaboost 
## Aggregating results
## Fitting final model on full training set
output <- test %>% select(sequencial_candidato)

predictions <- predict(model.best, test)

output$situacao <- predictions
output <- output %>% 
  select(Id = sequencial_candidato,
         Predicted = situacao)

write.csv(x = output,
          file = "./data/sample_submission.csv",
          row.names = FALSE)