Package
# ============================================================
# 1. INSTALL & LOAD PACKAGES
# ============================================================
# Jalankan install sekali jika belum ada:
# install.packages(c("readxl","caret","pROC","dplyr","e1071","rpart","rpart.plot",
# "randomForest","gbm","adabag","ipred","smotefamily",
# "ggplot2","reshape2","ROSE","class","tidyr","corrplot",
# "nnet","kernlab"))
library(readxl)
library(caret)
## Warning: package 'caret' was built under R version 4.5.3
## Loading required package: ggplot2
## Warning: package 'ggplot2' was built under R version 4.5.2
## Loading required package: lattice
library(pROC)
## Warning: package 'pROC' was built under R version 4.5.3
## Type 'citation("pROC")' for a citation.
##
## Attaching package: 'pROC'
## The following objects are masked from 'package:stats':
##
## cov, smooth, var
library(dplyr)
## Warning: package 'dplyr' was built under R version 4.5.3
##
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
##
## filter, lag
## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union
library(e1071)
##
## Attaching package: 'e1071'
## The following object is masked from 'package:ggplot2':
##
## element
library(rpart)
library(rpart.plot)
## Warning: package 'rpart.plot' was built under R version 4.5.3
library(randomForest)
## randomForest 4.7-1.2
## Type rfNews() to see new features/changes/bug fixes.
##
## Attaching package: 'randomForest'
## The following object is masked from 'package:dplyr':
##
## combine
## The following object is masked from 'package:ggplot2':
##
## margin
library(gbm)
## Warning: package 'gbm' was built under R version 4.5.3
## Loaded gbm 2.2.3
## This version of gbm is no longer under development. Consider transitioning to gbm3, https://github.com/gbm-developers/gbm3
library(adabag)
## Warning: package 'adabag' was built under R version 4.5.3
## Loading required package: foreach
## Loading required package: doParallel
## Loading required package: iterators
## Loading required package: parallel
library(ipred)
## Warning: package 'ipred' was built under R version 4.5.3
##
## Attaching package: 'ipred'
## The following object is masked from 'package:adabag':
##
## bagging
library(smotefamily)
## Warning: package 'smotefamily' was built under R version 4.5.3
library(ggplot2)
library(reshape2)
library(class)
library(tidyr)
##
## Attaching package: 'tidyr'
## The following object is masked from 'package:reshape2':
##
## smiths
library(corrplot)
## Warning: package 'corrplot' was built under R version 4.5.3
## corrplot 0.95 loaded
library(nnet)
library(kernlab)
## Warning: package 'kernlab' was built under R version 4.5.2
##
## Attaching package: 'kernlab'
## The following object is masked from 'package:ggplot2':
##
## alpha
Prepocessing Data
# ============================================================
# 3. PREPROCESSING DATA
# ============================================================
# --- 3.1 Hapus kolom ID (tidak relevan untuk pemodelan) ---
data <- data_raw[, -1] # hapus kolom 'id'
cat("\nKolom setelah hapus ID:", ncol(data), "\n")
##
## Kolom setelah hapus ID: 31
# --- 3.2 Cek Missing Value ---
cat("\n=== CEK MISSING VALUE ===\n")
##
## === CEK MISSING VALUE ===
mv <- colSums(is.na(data))
cat("Total missing value per kolom:\n")
## Total missing value per kolom:
print(mv)
## diagnosis radius_mean texture_mean
## 0 0 0
## perimeter_mean area_mean smoothness_mean
## 0 0 0
## compactness_mean concavity_mean concave_points_mean
## 0 0 0
## symmetry_mean fractal_dimension_mean radius_se
## 0 0 0
## texture_se perimeter_se area_se
## 0 0 0
## smoothness_se compactness_se concavity_se
## 0 0 0
## concave_points_se symmetry_se fractal_dimension_se
## 0 0 0
## radius_worst texture_worst perimeter_worst
## 0 0 0
## area_worst smoothness_worst compactness_worst
## 0 0 0
## concavity_worst concave_points_worst symmetry_worst
## 0 0 0
## fractal_dimension_worst
## 0
cat("\nTotal missing value keseluruhan:", sum(mv), "\n")
##
## Total missing value keseluruhan: 0
# --- 3.3 Encoding Variabel Target (diagnosis) ---
# M (Malignant/Ganas) = "yes" (kelas positif / minoritas)
# B (Benign/Jinak) = "no" (kelas negatif / mayoritas)
data$diagnosis <- ifelse(data$diagnosis == "M", "yes", "no")
data$diagnosis <- as.factor(data$diagnosis)
cat("\n=== STRUKTUR DATA SETELAH PREPROCESSING ===\n")
##
## === STRUKTUR DATA SETELAH PREPROCESSING ===
str(data)
## tibble [569 × 31] (S3: tbl_df/tbl/data.frame)
## $ diagnosis : Factor w/ 2 levels "no","yes": 2 2 2 2 2 2 2 2 2 2 ...
## $ radius_mean : num [1:569] 18 20.6 19.7 11.4 20.3 ...
## $ texture_mean : num [1:569] 10.4 17.8 21.2 20.4 14.3 ...
## $ perimeter_mean : num [1:569] 122.8 132.9 130 77.6 135.1 ...
## $ area_mean : num [1:569] 1001 1326 1203 386 1297 ...
## $ smoothness_mean : num [1:569] 0.1184 0.0847 0.1096 0.1425 0.1003 ...
## $ compactness_mean : num [1:569] 0.2776 0.0786 0.1599 0.2839 0.1328 ...
## $ concavity_mean : num [1:569] 0.3001 0.0869 0.1974 0.2414 0.198 ...
## $ concave_points_mean : num [1:569] 0.1471 0.0702 0.1279 0.1052 0.1043 ...
## $ symmetry_mean : num [1:569] 0.242 0.181 0.207 0.26 0.181 ...
## $ fractal_dimension_mean : num [1:569] 0.0787 0.0567 0.06 0.0974 0.0588 ...
## $ radius_se : num [1:569] 1.095 0.543 0.746 0.496 0.757 ...
## $ texture_se : num [1:569] 0.905 0.734 0.787 1.156 0.781 ...
## $ perimeter_se : num [1:569] 8.59 3.4 4.58 3.44 5.44 ...
## $ area_se : num [1:569] 153.4 74.1 94 27.2 94.4 ...
## $ smoothness_se : num [1:569] 0.0064 0.00522 0.00615 0.00911 0.01149 ...
## $ compactness_se : num [1:569] 0.049 0.0131 0.0401 0.0746 0.0246 ...
## $ concavity_se : num [1:569] 0.0537 0.0186 0.0383 0.0566 0.0569 ...
## $ concave_points_se : num [1:569] 0.0159 0.0134 0.0206 0.0187 0.0188 ...
## $ symmetry_se : num [1:569] 0.03 0.0139 0.0225 0.0596 0.0176 ...
## $ fractal_dimension_se : num [1:569] 0.00619 0.00353 0.00457 0.00921 0.00511 ...
## $ radius_worst : num [1:569] 25.4 25 23.6 14.9 22.5 ...
## $ texture_worst : num [1:569] 17.3 23.4 25.5 26.5 16.7 ...
## $ perimeter_worst : num [1:569] 184.6 158.8 152.5 98.9 152.2 ...
## $ area_worst : num [1:569] 2019 1956 1709 568 1575 ...
## $ smoothness_worst : num [1:569] 0.162 0.124 0.144 0.21 0.137 ...
## $ compactness_worst : num [1:569] 0.666 0.187 0.424 0.866 0.205 ...
## $ concavity_worst : num [1:569] 0.712 0.242 0.45 0.687 0.4 ...
## $ concave_points_worst : num [1:569] 0.265 0.186 0.243 0.258 0.163 ...
## $ symmetry_worst : num [1:569] 0.46 0.275 0.361 0.664 0.236 ...
## $ fractal_dimension_worst: num [1:569] 0.1189 0.089 0.0876 0.173 0.0768 ...
# --- 3.4 Analisis Distribusi Kelas ---
cat("\n=== DISTRIBUSI KELAS (VARIABEL diagnosis) ===\n")
##
## === DISTRIBUSI KELAS (VARIABEL diagnosis) ===
tbl_y <- table(data$diagnosis)
print(tbl_y)
##
## no yes
## 357 212
print(prop.table(tbl_y))
##
## no yes
## 0.6274165 0.3725835
# Visualisasi distribusi kelas
barplot(tbl_y,
main = "Distribusi Kelas Diagnosis",
col = c("steelblue", "tomato"),
xlab = "Kelas (no=Benign, yes=Malignant)",
ylab = "Frekuensi",
ylim = c(0, 400))
text(x = c(0.7, 1.9),
y = tbl_y + 10,
labels = paste0(round(prop.table(tbl_y)*100, 1), "%"),
col = "black", cex = 1.1, font = 2)
# --- 3.5 Feature Selection (Pearson Correlation) ---
# Karena ada 30 fitur numerik, lakukan seleksi fitur
# berdasarkan korelasi dengan variabel target
cat("\n=== FEATURE SELECTION ===\n")
##
## === FEATURE SELECTION ===
# Buat versi numerik target untuk korelasi
y_numeric <- ifelse(data$diagnosis == "yes", 1, 0)
X_all <- data[, -1] # semua fitur numerik
# Hitung korelasi absolut dengan target
cor_vals <- sapply(X_all, function(x) abs(cor(x, y_numeric, method = "pearson")))
cor_df <- data.frame(Fitur = names(cor_vals), Korelasi = cor_vals)
cor_df <- cor_df[order(-cor_df$Korelasi), ]
cat("Top 20 fitur berdasarkan korelasi dengan target:\n")
## Top 20 fitur berdasarkan korelasi dengan target:
print(head(cor_df, 20))
## Fitur Korelasi
## concave_points_worst concave_points_worst 0.7935660
## perimeter_worst perimeter_worst 0.7829141
## concave_points_mean concave_points_mean 0.7766138
## radius_worst radius_worst 0.7764538
## perimeter_mean perimeter_mean 0.7426355
## area_worst area_worst 0.7338250
## radius_mean radius_mean 0.7300285
## area_mean area_mean 0.7089838
## concavity_mean concavity_mean 0.6963597
## concavity_worst concavity_worst 0.6596102
## compactness_mean compactness_mean 0.5965337
## compactness_worst compactness_worst 0.5909982
## radius_se radius_se 0.5671338
## perimeter_se perimeter_se 0.5561407
## area_se area_se 0.5482359
## texture_worst texture_worst 0.4569028
## smoothness_worst smoothness_worst 0.4214649
## symmetry_worst symmetry_worst 0.4162943
## texture_mean texture_mean 0.4151853
## concave_points_se concave_points_se 0.4080423
# Pilih 15 fitur teratas
selected_features <- as.character(cor_df$Fitur[1:15])
cat("\nFitur yang dipilih (top 15):\n")
##
## Fitur yang dipilih (top 15):
print(selected_features)
## [1] "concave_points_worst" "perimeter_worst" "concave_points_mean"
## [4] "radius_worst" "perimeter_mean" "area_worst"
## [7] "radius_mean" "area_mean" "concavity_mean"
## [10] "concavity_worst" "compactness_mean" "compactness_worst"
## [13] "radius_se" "perimeter_se" "area_se"
# Buat dataset final dengan fitur terpilih + target
data_model <- data[, c("diagnosis", selected_features)]
# Visualisasi korelasi fitur terpilih
cor_matrix <- cor(data_model[, -1])
corrplot(cor_matrix, method = "color", type = "upper",
tl.cex = 0.7, title = "Matriks Korelasi Fitur Terpilih",
mar = c(0, 0, 2, 0))
# --- 3.6 Train-Test Split (80:20) ---
cat("\n=== TRAIN-TEST SPLIT (80:20) ===\n")
##
## === TRAIN-TEST SPLIT (80:20) ===
set.seed(123)
idx_train <- createDataPartition(data_model$diagnosis, p = 0.8, list = FALSE)
train <- data_model[ idx_train, ]
test <- data_model[-idx_train, ]
cat("Dimensi Training :", dim(train), "\n")
## Dimensi Training : 456 16
cat("Dimensi Testing :", dim(test), "\n")
## Dimensi Testing : 113 16
cat("\nDistribusi Training:\n")
##
## Distribusi Training:
print(table(train$diagnosis))
##
## no yes
## 286 170
print(prop.table(table(train$diagnosis)))
##
## no yes
## 0.627193 0.372807
cat("\nDistribusi Testing:\n")
##
## Distribusi Testing:
print(table(test$diagnosis))
##
## no yes
## 71 42
print(prop.table(table(test$diagnosis)))
##
## no yes
## 0.6283186 0.3716814
# --- 3.7 Scaling Data (untuk LR, SVM, KNN, Neural Network) ---
num_cols <- selected_features
# Simpan scaler dari training (CEGAH DATA LEAKAGE)
preproc <- preProcess(train[, num_cols], method = c("center", "scale"))
train_scaled <- train
test_scaled <- test
train_scaled[, num_cols] <- predict(preproc, train[, num_cols])
test_scaled[, num_cols] <- predict(preproc, test[, num_cols])
cat("\nData berhasil di-scale untuk LR, SVM, KNN, Neural Network.\n")
##
## Data berhasil di-scale untuk LR, SVM, KNN, Neural Network.
Fungsi Evaluasi Model
# ============================================================
# 4. FUNGSI EVALUASI MODEL
# ============================================================
evaluasi_model <- function(actual, pred_class, pred_prob = NULL,
model_name = "Model", data_type = "Testing") {
cm <- confusionMatrix(pred_class, actual, positive = "yes")
acc <- cm$overall["Accuracy"]
prec <- cm$byClass["Pos Pred Value"]
rec <- cm$byClass["Sensitivity"]
spec <- cm$byClass["Specificity"]
f1 <- cm$byClass["F1"]
auc_val <- NA
if (!is.null(pred_prob)) {
roc_obj <- roc(actual, pred_prob, levels = c("no", "yes"),
direction = "<", quiet = TRUE)
auc_val <- auc(roc_obj)
plot(roc_obj, main = paste("ROC Curve -", model_name, "-", data_type))
}
cat(sprintf("\n[%s - %s]\n", model_name, data_type))
print(cm$table)
cat(sprintf(" Accuracy : %.4f\n", acc))
cat(sprintf(" Precision : %.4f\n", prec))
cat(sprintf(" Sensitivity : %.4f\n", rec))
cat(sprintf(" Specificity : %.4f\n", spec))
cat(sprintf(" F1-Score : %.4f\n", f1))
if (!is.na(auc_val)) cat(sprintf(" AUC : %.4f\n", auc_val))
return(data.frame(
Model = model_name,
Data = data_type,
Accuracy = round(as.numeric(acc), 4),
Precision = round(as.numeric(prec), 4),
Sensitivity = round(as.numeric(rec), 4),
Specificity = round(as.numeric(spec), 4),
F1_Score = round(as.numeric(f1), 4),
AUC = round(as.numeric(auc_val), 4)
))
}
# Data frame penampung semua hasil
hasil_semua <- data.frame()
Skenario 1 - Tanpa Tuning dan Tanpa SMOTE
# ============================================================
# 5. SKENARIO 1 - TANPA SMOTE, TANPA TUNING
# ============================================================
cat("\n\n")
cat("============================================================\n")
## ============================================================
cat(" SKENARIO 1 - TANPA SMOTE, TANPA TUNING\n")
## SKENARIO 1 - TANPA SMOTE, TANPA TUNING
cat("============================================================\n")
## ============================================================
# ----- 5.1 LOGISTIC REGRESSION -----
cat("\n--- LOGISTIC REGRESSION ---\n")
##
## --- LOGISTIC REGRESSION ---
set.seed(123)
model_lr1 <- glm(diagnosis ~ ., data = train_scaled, family = "binomial")
## Warning: glm.fit: fitted probabilities numerically 0 or 1 occurred
summary(model_lr1)
##
## Call:
## glm(formula = diagnosis ~ ., family = "binomial", data = train_scaled)
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 1.7792 1.1157 1.595 0.1108
## concave_points_worst 1.5502 1.5614 0.993 0.3208
## perimeter_worst -5.6658 8.3462 -0.679 0.4972
## concave_points_mean 4.3471 1.9172 2.267 0.0234 *
## radius_worst 11.6710 11.5480 1.011 0.3122
## perimeter_mean 52.8134 25.9948 2.032 0.0422 *
## area_worst 1.0474 14.2739 0.073 0.9415
## radius_mean -59.3305 28.0905 -2.112 0.0347 *
## area_mean 5.2607 13.6743 0.385 0.7004
## concavity_mean -2.8402 1.8821 -1.509 0.1313
## concavity_worst 3.2252 1.8341 1.758 0.0787 .
## compactness_mean -5.5518 2.2799 -2.435 0.0149 *
## compactness_worst 1.1908 1.5219 0.782 0.4340
## radius_se 0.3185 4.7278 0.067 0.9463
## perimeter_se -2.8692 3.4014 -0.844 0.3989
## area_se 9.2281 7.6037 1.214 0.2249
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 602.31 on 455 degrees of freedom
## Residual deviance: 73.17 on 440 degrees of freedom
## AIC: 105.17
##
## Number of Fisher Scoring iterations: 10
prob_lr1_train <- predict(model_lr1, train_scaled, type = "response")
pred_lr1_train <- factor(ifelse(prob_lr1_train > 0.5, "yes", "no"), levels = c("no", "yes"))
res_lr1_train <- evaluasi_model(train_scaled$diagnosis, pred_lr1_train, prob_lr1_train,
"LR - S1", "Training")
##
## [LR - S1 - Training]
## Reference
## Prediction no yes
## no 279 10
## yes 7 160
## Accuracy : 0.9627
## Precision : 0.9581
## Sensitivity : 0.9412
## Specificity : 0.9755
## F1-Score : 0.9496
## AUC : 0.9958
prob_lr1_test <- predict(model_lr1, test_scaled, type = "response")
pred_lr1_test <- factor(ifelse(prob_lr1_test > 0.5, "yes", "no"), levels = c("no", "yes"))
res_lr1_test <- evaluasi_model(test_scaled$diagnosis, pred_lr1_test, prob_lr1_test,
"LR - S1", "Testing")
##
## [LR - S1 - Testing]
## Reference
## Prediction no yes
## no 64 1
## yes 7 41
## Accuracy : 0.9292
## Precision : 0.8542
## Sensitivity : 0.9762
## Specificity : 0.9014
## F1-Score : 0.9111
## AUC : 0.9940
hasil_semua <- rbind(hasil_semua, res_lr1_train, res_lr1_test)
# ----- 5.2 DECISION TREE -----
cat("\n--- DECISION TREE ---\n")
##
## --- DECISION TREE ---
set.seed(123)
model_dt1 <- rpart(diagnosis ~ ., data = train, method = "class",
control = rpart.control(cp = 0.01, minsplit = 20, maxdepth = 10))
rpart.plot(model_dt1, extra = 104, fallen.leaves = TRUE, cex = 0.7,
main = "Decision Tree - Skenario 1")
prob_dt1_train <- predict(model_dt1, train, type = "prob")[, "yes"]
pred_dt1_train <- predict(model_dt1, train, type = "class")
res_dt1_train <- evaluasi_model(train$diagnosis, pred_dt1_train, prob_dt1_train,
"DT - S1", "Training")
##
## [DT - S1 - Training]
## Reference
## Prediction no yes
## no 272 6
## yes 14 164
## Accuracy : 0.9561
## Precision : 0.9213
## Sensitivity : 0.9647
## Specificity : 0.9510
## F1-Score : 0.9425
## AUC : 0.9784
prob_dt1_test <- predict(model_dt1, test, type = "prob")[, "yes"]
pred_dt1_test <- predict(model_dt1, test, type = "class")
res_dt1_test <- evaluasi_model(test$diagnosis, pred_dt1_test, prob_dt1_test,
"DT - S1", "Testing")
##
## [DT - S1 - Testing]
## Reference
## Prediction no yes
## no 62 2
## yes 9 40
## Accuracy : 0.9027
## Precision : 0.8163
## Sensitivity : 0.9524
## Specificity : 0.8732
## F1-Score : 0.8791
## AUC : 0.9802
hasil_semua <- rbind(hasil_semua, res_dt1_train, res_dt1_test)
# ----- 5.3 NAIVE BAYES -----
cat("\n--- NAIVE BAYES ---\n")
##
## --- NAIVE BAYES ---
set.seed(123)
model_nb1 <- naiveBayes(diagnosis ~ ., data = train, laplace = 1)
prob_nb1_train <- predict(model_nb1, train, type = "raw")[, "yes"]
pred_nb1_train <- factor(predict(model_nb1, train, type = "class"), levels = c("no", "yes"))
res_nb1_train <- evaluasi_model(train$diagnosis, pred_nb1_train, prob_nb1_train,
"NB - S1", "Training")
##
## [NB - S1 - Training]
## Reference
## Prediction no yes
## no 273 18
## yes 13 152
## Accuracy : 0.9320
## Precision : 0.9212
## Sensitivity : 0.8941
## Specificity : 0.9545
## F1-Score : 0.9075
## AUC : 0.9838
prob_nb1_test <- predict(model_nb1, test, type = "raw")[, "yes"]
pred_nb1_test <- factor(predict(model_nb1, test, type = "class"), levels = c("no", "yes"))
res_nb1_test <- evaluasi_model(test$diagnosis, pred_nb1_test, prob_nb1_test,
"NB - S1", "Testing")
##
## [NB - S1 - Testing]
## Reference
## Prediction no yes
## no 67 2
## yes 4 40
## Accuracy : 0.9469
## Precision : 0.9091
## Sensitivity : 0.9524
## Specificity : 0.9437
## F1-Score : 0.9302
## AUC : 0.9893
hasil_semua <- rbind(hasil_semua, res_nb1_train, res_nb1_test)
# ----- 5.4 K-NEAREST NEIGHBOR (KNN) -----
cat("\n--- K-NEAREST NEIGHBOR (KNN) ---\n")
##
## --- K-NEAREST NEIGHBOR (KNN) ---
x_train_knn <- model.matrix(diagnosis ~ . - 1, data = train_scaled)
x_test_knn <- model.matrix(diagnosis ~ . - 1, data = test_scaled)
y_train_knn <- train_scaled$diagnosis
y_test_knn <- test_scaled$diagnosis
# Pilih k = 5 sebagai default (tanpa tuning di S1)
set.seed(123)
knn1_raw_train <- knn(x_train_knn, x_train_knn, y_train_knn, k = 5, prob = TRUE)
prob_knn1_train <- ifelse(knn1_raw_train == "yes",
attr(knn1_raw_train, "prob"), 1 - attr(knn1_raw_train, "prob"))
pred_knn1_train <- factor(as.character(knn1_raw_train), levels = c("no", "yes"))
res_knn1_train <- evaluasi_model(y_train_knn, pred_knn1_train, prob_knn1_train,
"KNN - S1", "Training")
##
## [KNN - S1 - Training]
## Reference
## Prediction no yes
## no 276 11
## yes 10 159
## Accuracy : 0.9539
## Precision : 0.9408
## Sensitivity : 0.9353
## Specificity : 0.9650
## F1-Score : 0.9381
## AUC : 0.9928
knn1_raw_test <- knn(x_train_knn, x_test_knn, y_train_knn, k = 5, prob = TRUE)
prob_knn1_test <- ifelse(knn1_raw_test == "yes",
attr(knn1_raw_test, "prob"), 1 - attr(knn1_raw_test, "prob"))
pred_knn1_test <- factor(as.character(knn1_raw_test), levels = c("no", "yes"))
res_knn1_test <- evaluasi_model(y_test_knn, pred_knn1_test, prob_knn1_test,
"KNN - S1", "Testing")
##
## [KNN - S1 - Testing]
## Reference
## Prediction no yes
## no 67 2
## yes 4 40
## Accuracy : 0.9469
## Precision : 0.9091
## Sensitivity : 0.9524
## Specificity : 0.9437
## F1-Score : 0.9302
## AUC : 0.9935
hasil_semua <- rbind(hasil_semua, res_knn1_train, res_knn1_test)
# ----- 5.5 SUPPORT VECTOR MACHINE (SVM) -----
cat("\n--- SUPPORT VECTOR MACHINE (SVM) ---\n")
##
## --- SUPPORT VECTOR MACHINE (SVM) ---
set.seed(123)
model_svm1 <- ksvm(diagnosis ~ ., data = train_scaled, kernel = "rbfdot",
prob.model = TRUE, C = 1)
prob_svm1_train <- predict(model_svm1, train_scaled, type = "probabilities")[, "yes"]
pred_svm1_train <- factor(predict(model_svm1, train_scaled, type = "response"),
levels = c("no", "yes"))
res_svm1_train <- evaluasi_model(train_scaled$diagnosis, pred_svm1_train, prob_svm1_train,
"SVM - S1", "Training")
##
## [SVM - S1 - Training]
## Reference
## Prediction no yes
## no 278 7
## yes 8 163
## Accuracy : 0.9671
## Precision : 0.9532
## Sensitivity : 0.9588
## Specificity : 0.9720
## F1-Score : 0.9560
## AUC : 0.9937
prob_svm1_test <- predict(model_svm1, test_scaled, type = "probabilities")[, "yes"]
pred_svm1_test <- factor(predict(model_svm1, test_scaled, type = "response"),
levels = c("no", "yes"))
res_svm1_test <- evaluasi_model(test_scaled$diagnosis, pred_svm1_test, prob_svm1_test,
"SVM - S1", "Testing")
##
## [SVM - S1 - Testing]
## Reference
## Prediction no yes
## no 68 1
## yes 3 41
## Accuracy : 0.9646
## Precision : 0.9318
## Sensitivity : 0.9762
## Specificity : 0.9577
## F1-Score : 0.9535
## AUC : 0.9960
hasil_semua <- rbind(hasil_semua, res_svm1_train, res_svm1_test)
# ----- 5.6 NEURAL NETWORK -----
cat("\n--- NEURAL NETWORK ---\n")
##
## --- NEURAL NETWORK ---
set.seed(123)
model_nn1 <- nnet(diagnosis ~ ., data = train_scaled, size = 5,
maxit = 200, decay = 0.01, trace = FALSE)
prob_nn1_train <- predict(model_nn1, train_scaled, type = "raw")[, 1]
pred_nn1_train <- factor(ifelse(prob_nn1_train > 0.5, "yes", "no"), levels = c("no", "yes"))
res_nn1_train <- evaluasi_model(train_scaled$diagnosis, pred_nn1_train, prob_nn1_train,
"NN - S1", "Training")
##
## [NN - S1 - Training]
## Reference
## Prediction no yes
## no 286 0
## yes 0 170
## Accuracy : 1.0000
## Precision : 1.0000
## Sensitivity : 1.0000
## Specificity : 1.0000
## F1-Score : 1.0000
## AUC : 1.0000
prob_nn1_test <- predict(model_nn1, test_scaled, type = "raw")[, 1]
pred_nn1_test <- factor(ifelse(prob_nn1_test > 0.5, "yes", "no"), levels = c("no", "yes"))
res_nn1_test <- evaluasi_model(test_scaled$diagnosis, pred_nn1_test, prob_nn1_test,
"NN - S1", "Testing")
##
## [NN - S1 - Testing]
## Reference
## Prediction no yes
## no 68 2
## yes 3 40
## Accuracy : 0.9558
## Precision : 0.9302
## Sensitivity : 0.9524
## Specificity : 0.9577
## F1-Score : 0.9412
## AUC : 0.9906
hasil_semua <- rbind(hasil_semua, res_nn1_train, res_nn1_test)
# ----- 5.7 RANDOM FOREST -----
cat("\n--- RANDOM FOREST ---\n")
##
## --- RANDOM FOREST ---
set.seed(123)
model_rf1 <- randomForest(diagnosis ~ ., data = train, ntree = 100,
mtry = 3, importance = TRUE)
print(model_rf1)
##
## Call:
## randomForest(formula = diagnosis ~ ., data = train, ntree = 100, mtry = 3, importance = TRUE)
## Type of random forest: classification
## Number of trees: 100
## No. of variables tried at each split: 3
##
## OOB estimate of error rate: 6.58%
## Confusion matrix:
## no yes class.error
## no 270 16 0.05594406
## yes 14 156 0.08235294
varImpPlot(model_rf1, main = "Variable Importance - Random Forest S1")
prob_rf1_train <- model_rf1$votes[, "yes"]
pred_rf1_train <- factor(model_rf1$predicted, levels = c("no", "yes"))
res_rf1_train <- evaluasi_model(train$diagnosis, pred_rf1_train, prob_rf1_train,
"RF - S1", "Training")
##
## [RF - S1 - Training]
## Reference
## Prediction no yes
## no 270 14
## yes 16 156
## Accuracy : 0.9342
## Precision : 0.9070
## Sensitivity : 0.9176
## Specificity : 0.9441
## F1-Score : 0.9123
## AUC : 0.9800
prob_rf1_test <- predict(model_rf1, test, type = "prob")[, "yes"]
pred_rf1_test <- predict(model_rf1, test, type = "class")
res_rf1_test <- evaluasi_model(test$diagnosis, pred_rf1_test, prob_rf1_test,
"RF - S1", "Testing")
##
## [RF - S1 - Testing]
## Reference
## Prediction no yes
## no 68 1
## yes 3 41
## Accuracy : 0.9646
## Precision : 0.9318
## Sensitivity : 0.9762
## Specificity : 0.9577
## F1-Score : 0.9535
## AUC : 0.9956
hasil_semua <- rbind(hasil_semua, res_rf1_train, res_rf1_test)
# ----- 5.8 GRADIENT BOOSTING (GBM) -----
cat("\n--- GRADIENT BOOSTING (GBM) ---\n")
##
## --- GRADIENT BOOSTING (GBM) ---
train_gbm1 <- train
test_gbm1 <- test
train_gbm1$diagnosis <- ifelse(train_gbm1$diagnosis == "yes", 1, 0)
test_gbm1$diagnosis <- ifelse(test_gbm1$diagnosis == "yes", 1, 0)
set.seed(123)
model_gbm1 <- gbm(diagnosis ~ ., data = train_gbm1, distribution = "bernoulli",
n.trees = 200, interaction.depth = 3,
shrinkage = 0.05, n.minobsinnode = 5, verbose = FALSE)
summary(model_gbm1)
## var rel.inf
## perimeter_worst perimeter_worst 29.5703057
## concave_points_worst concave_points_worst 29.0810515
## area_worst area_worst 11.8074420
## radius_worst radius_worst 8.7078076
## concavity_worst concavity_worst 3.4366313
## area_se area_se 3.2378808
## concave_points_mean concave_points_mean 2.6231541
## compactness_mean compactness_mean 2.1418541
## radius_se radius_se 2.0047602
## concavity_mean concavity_mean 1.6341558
## perimeter_mean perimeter_mean 1.5038956
## area_mean area_mean 1.3974693
## perimeter_se perimeter_se 1.0298473
## radius_mean radius_mean 1.0244455
## compactness_worst compactness_worst 0.7992992
prob_gbm1_train <- predict(model_gbm1, train_gbm1, n.trees = 200, type = "response")
pred_gbm1_train <- factor(ifelse(prob_gbm1_train > 0.5, "yes", "no"), levels = c("no", "yes"))
res_gbm1_train <- evaluasi_model(train$diagnosis, pred_gbm1_train, prob_gbm1_train,
"GBM - S1", "Training")
##
## [GBM - S1 - Training]
## Reference
## Prediction no yes
## no 286 1
## yes 0 169
## Accuracy : 0.9978
## Precision : 1.0000
## Sensitivity : 0.9941
## Specificity : 1.0000
## F1-Score : 0.9971
## AUC : 1.0000
prob_gbm1_test <- predict(model_gbm1, test_gbm1, n.trees = 200, type = "response")
pred_gbm1_test <- factor(ifelse(prob_gbm1_test > 0.5, "yes", "no"), levels = c("no", "yes"))
res_gbm1_test <- evaluasi_model(test$diagnosis, pred_gbm1_test, prob_gbm1_test,
"GBM - S1", "Testing")
##
## [GBM - S1 - Testing]
## Reference
## Prediction no yes
## no 66 2
## yes 5 40
## Accuracy : 0.9381
## Precision : 0.8889
## Sensitivity : 0.9524
## Specificity : 0.9296
## F1-Score : 0.9195
## AUC : 0.9943
hasil_semua <- rbind(hasil_semua, res_gbm1_train, res_gbm1_test)
# --- Ringkasan Skenario 1 ---
cat("\n=== RINGKASAN SKENARIO 1 ===\n")
##
## === RINGKASAN SKENARIO 1 ===
print(hasil_semua[grepl("S1", hasil_semua$Model), ])
## Model Data Accuracy Precision Sensitivity Specificity F1_Score AUC
## 1 LR - S1 Training 0.9627 0.9581 0.9412 0.9755 0.9496 0.9958
## 2 LR - S1 Testing 0.9292 0.8542 0.9762 0.9014 0.9111 0.9940
## 3 DT - S1 Training 0.9561 0.9213 0.9647 0.9510 0.9425 0.9784
## 4 DT - S1 Testing 0.9027 0.8163 0.9524 0.8732 0.8791 0.9802
## 5 NB - S1 Training 0.9320 0.9212 0.8941 0.9545 0.9075 0.9838
## 6 NB - S1 Testing 0.9469 0.9091 0.9524 0.9437 0.9302 0.9893
## 7 KNN - S1 Training 0.9539 0.9408 0.9353 0.9650 0.9381 0.9928
## 8 KNN - S1 Testing 0.9469 0.9091 0.9524 0.9437 0.9302 0.9935
## 9 SVM - S1 Training 0.9671 0.9532 0.9588 0.9720 0.9560 0.9937
## 10 SVM - S1 Testing 0.9646 0.9318 0.9762 0.9577 0.9535 0.9960
## 11 NN - S1 Training 1.0000 1.0000 1.0000 1.0000 1.0000 1.0000
## 12 NN - S1 Testing 0.9558 0.9302 0.9524 0.9577 0.9412 0.9906
## 13 RF - S1 Training 0.9342 0.9070 0.9176 0.9441 0.9123 0.9800
## 14 RF - S1 Testing 0.9646 0.9318 0.9762 0.9577 0.9535 0.9956
## 15 GBM - S1 Training 0.9978 1.0000 0.9941 1.0000 0.9971 1.0000
## 16 GBM - S1 Testing 0.9381 0.8889 0.9524 0.9296 0.9195 0.9943
Skenario 2 - Tuning dan Tanpa SMOTE
# ============================================================
# 6. SKENARIO 2 - TANPA SMOTE, DENGAN TUNING
# ============================================================
cat("\n\n")
cat("============================================================\n")
## ============================================================
cat(" SKENARIO 2 - TANPA SMOTE, DENGAN TUNING\n")
## SKENARIO 2 - TANPA SMOTE, DENGAN TUNING
cat("============================================================\n")
## ============================================================
# ----- 6.1 LOGISTIC REGRESSION - Tuning Threshold -----
cat("\n--- LOGISTIC REGRESSION (Tuning Threshold) ---\n")
##
## --- LOGISTIC REGRESSION (Tuning Threshold) ---
thresholds <- seq(0.3, 0.7, by = 0.05)
f1_thresh <- sapply(thresholds, function(t) {
pr_t <- factor(ifelse(prob_lr1_train > t, "yes", "no"), levels = c("no", "yes"))
cm_t <- confusionMatrix(pr_t, train_scaled$diagnosis, positive = "yes")
as.numeric(cm_t$byClass["F1"])
})
best_thresh_lr <- thresholds[which.max(f1_thresh)]
cat("Threshold optimal LR:", best_thresh_lr, "\n")
## Threshold optimal LR: 0.55
pred_lr2_train <- factor(ifelse(prob_lr1_train > best_thresh_lr, "yes", "no"),
levels = c("no", "yes"))
res_lr2_train <- evaluasi_model(train_scaled$diagnosis, pred_lr2_train, prob_lr1_train,
"LR - S2", "Training")
##
## [LR - S2 - Training]
## Reference
## Prediction no yes
## no 281 10
## yes 5 160
## Accuracy : 0.9671
## Precision : 0.9697
## Sensitivity : 0.9412
## Specificity : 0.9825
## F1-Score : 0.9552
## AUC : 0.9958
pred_lr2_test <- factor(ifelse(prob_lr1_test > best_thresh_lr, "yes", "no"),
levels = c("no", "yes"))
res_lr2_test <- evaluasi_model(test_scaled$diagnosis, pred_lr2_test, prob_lr1_test,
"LR - S2", "Testing")
##
## [LR - S2 - Testing]
## Reference
## Prediction no yes
## no 66 1
## yes 5 41
## Accuracy : 0.9469
## Precision : 0.8913
## Sensitivity : 0.9762
## Specificity : 0.9296
## F1-Score : 0.9318
## AUC : 0.9940
hasil_semua <- rbind(hasil_semua, res_lr2_train, res_lr2_test)
# ----- 6.2 DECISION TREE - Grid Search (cp & maxdepth) -----
cat("\n--- DECISION TREE (Grid Search) ---\n")
##
## --- DECISION TREE (Grid Search) ---
cp_grid <- seq(0.001, 0.05, by = 0.005)
depth_grid <- c(3, 5, 7, 10, 15)
grid_dt2 <- expand.grid(cp = cp_grid, maxdepth = depth_grid)
grid_dt2$F1 <- NA
for (i in 1:nrow(grid_dt2)) {
dt_tmp <- rpart(diagnosis ~ ., data = train, method = "class",
control = rpart.control(cp = grid_dt2$cp[i],
minsplit = 20, maxdepth = grid_dt2$maxdepth[i]))
pr_tmp <- factor(predict(dt_tmp, test, type = "class"), levels = c("no", "yes"))
cm_tmp <- confusionMatrix(pr_tmp, test$diagnosis, positive = "yes")
grid_dt2$F1[i] <- as.numeric(cm_tmp$byClass["F1"])
}
best_dt2 <- grid_dt2[which.max(grid_dt2$F1), ]
cat("Best DT params - cp:", best_dt2$cp, "| maxdepth:", best_dt2$maxdepth, "\n")
## Best DT params - cp: 0.001 | maxdepth: 3
set.seed(123)
model_dt2 <- rpart(diagnosis ~ ., data = train, method = "class",
control = rpart.control(cp = best_dt2$cp, minsplit = 20,
maxdepth = best_dt2$maxdepth))
cp_tbl2 <- model_dt2$cptable
best_cp2 <- cp_tbl2[which.min(cp_tbl2[, "xerror"]), "CP"]
model_dt2 <- prune(model_dt2, cp = best_cp2)
rpart.plot(model_dt2, extra = 104, cex = 0.7, main = "Decision Tree Tuned - Skenario 2")
prob_dt2_train <- predict(model_dt2, train, type = "prob")[, "yes"]
pred_dt2_train <- predict(model_dt2, train, type = "class")
res_dt2_train <- evaluasi_model(train$diagnosis, pred_dt2_train, prob_dt2_train,
"DT - S2", "Training")
##
## [DT - S2 - Training]
## Reference
## Prediction no yes
## no 274 11
## yes 12 159
## Accuracy : 0.9496
## Precision : 0.9298
## Sensitivity : 0.9353
## Specificity : 0.9580
## F1-Score : 0.9326
## AUC : 0.9774
prob_dt2_test <- predict(model_dt2, test, type = "prob")[, "yes"]
pred_dt2_test <- predict(model_dt2, test, type = "class")
res_dt2_test <- evaluasi_model(test$diagnosis, pred_dt2_test, prob_dt2_test,
"DT - S2", "Testing")
##
## [DT - S2 - Testing]
## Reference
## Prediction no yes
## no 65 2
## yes 6 40
## Accuracy : 0.9292
## Precision : 0.8696
## Sensitivity : 0.9524
## Specificity : 0.9155
## F1-Score : 0.9091
## AUC : 0.9832
hasil_semua <- rbind(hasil_semua, res_dt2_train, res_dt2_test)
# ----- 6.3 NAIVE BAYES - Tuning Laplace -----
cat("\n--- NAIVE BAYES (Tuning Laplace) ---\n")
##
## --- NAIVE BAYES (Tuning Laplace) ---
laplace_vals <- 0:5
f1_nb2 <- sapply(laplace_vals, function(lap) {
m_tmp <- naiveBayes(diagnosis ~ ., data = train, laplace = lap)
pr_tmp <- factor(predict(m_tmp, test, type = "class"), levels = c("no", "yes"))
cm_tmp <- confusionMatrix(pr_tmp, test$diagnosis, positive = "yes")
as.numeric(cm_tmp$byClass["F1"])
})
best_lap2 <- laplace_vals[which.max(f1_nb2)]
cat("Laplace optimal NB:", best_lap2, "\n")
## Laplace optimal NB: 0
model_nb2 <- naiveBayes(diagnosis ~ ., data = train, laplace = best_lap2)
prob_nb2_train <- predict(model_nb2, train, type = "raw")[, "yes"]
pred_nb2_train <- factor(predict(model_nb2, train, type = "class"), levels = c("no", "yes"))
res_nb2_train <- evaluasi_model(train$diagnosis, pred_nb2_train, prob_nb2_train,
"NB - S2", "Training")
##
## [NB - S2 - Training]
## Reference
## Prediction no yes
## no 273 18
## yes 13 152
## Accuracy : 0.9320
## Precision : 0.9212
## Sensitivity : 0.8941
## Specificity : 0.9545
## F1-Score : 0.9075
## AUC : 0.9838
prob_nb2_test <- predict(model_nb2, test, type = "raw")[, "yes"]
pred_nb2_test <- factor(predict(model_nb2, test, type = "class"), levels = c("no", "yes"))
res_nb2_test <- evaluasi_model(test$diagnosis, pred_nb2_test, prob_nb2_test,
"NB - S2", "Testing")
##
## [NB - S2 - Testing]
## Reference
## Prediction no yes
## no 67 2
## yes 4 40
## Accuracy : 0.9469
## Precision : 0.9091
## Sensitivity : 0.9524
## Specificity : 0.9437
## F1-Score : 0.9302
## AUC : 0.9893
hasil_semua <- rbind(hasil_semua, res_nb2_train, res_nb2_test)
# ----- 6.4 KNN - Tuning k -----
cat("\n--- KNN (Tuning k) ---\n")
##
## --- KNN (Tuning k) ---
k_range2 <- seq(1, 25, by = 2)
f1_knn2 <- sapply(k_range2, function(k) {
pr_tmp <- knn(x_train_knn, x_test_knn, y_train_knn, k = k)
pr_tmp <- factor(pr_tmp, levels = c("no", "yes"))
cm_tmp <- confusionMatrix(pr_tmp, y_test_knn, positive = "yes")
as.numeric(cm_tmp$byClass["F1"])
})
best_k2 <- k_range2[which.max(f1_knn2)]
cat("k optimal KNN:", best_k2, "\n")
## k optimal KNN: 9
knn2_raw_train <- knn(x_train_knn, x_train_knn, y_train_knn, k = best_k2, prob = TRUE)
prob_knn2_train <- ifelse(knn2_raw_train == "yes",
attr(knn2_raw_train, "prob"), 1 - attr(knn2_raw_train, "prob"))
pred_knn2_train <- factor(as.character(knn2_raw_train), levels = c("no", "yes"))
res_knn2_train <- evaluasi_model(y_train_knn, pred_knn2_train, prob_knn2_train,
"KNN - S2", "Training")
##
## [KNN - S2 - Training]
## Reference
## Prediction no yes
## no 273 13
## yes 13 157
## Accuracy : 0.9430
## Precision : 0.9235
## Sensitivity : 0.9235
## Specificity : 0.9545
## F1-Score : 0.9235
## AUC : 0.9913
knn2_raw_test <- knn(x_train_knn, x_test_knn, y_train_knn, k = best_k2, prob = TRUE)
prob_knn2_test <- ifelse(knn2_raw_test == "yes",
attr(knn2_raw_test, "prob"), 1 - attr(knn2_raw_test, "prob"))
pred_knn2_test <- factor(as.character(knn2_raw_test), levels = c("no", "yes"))
res_knn2_test <- evaluasi_model(y_test_knn, pred_knn2_test, prob_knn2_test,
"KNN - S2", "Testing")
##
## [KNN - S2 - Testing]
## Reference
## Prediction no yes
## no 69 2
## yes 2 40
## Accuracy : 0.9646
## Precision : 0.9524
## Sensitivity : 0.9524
## Specificity : 0.9718
## F1-Score : 0.9524
## AUC : 0.9946
hasil_semua <- rbind(hasil_semua, res_knn2_train, res_knn2_test)
# ----- 6.5 SVM - Tuning C (Cost) -----
cat("\n--- SVM (Tuning C) ---\n")
##
## --- SVM (Tuning C) ---
C_vals <- c(0.1, 0.5, 1, 5, 10)
f1_svm2 <- sapply(C_vals, function(c_val) {
set.seed(123)
m_tmp <- ksvm(diagnosis ~ ., data = train_scaled, kernel = "rbfdot",
prob.model = TRUE, C = c_val)
pr_tmp <- factor(predict(m_tmp, test_scaled, type = "response"),
levels = c("no", "yes"))
cm_tmp <- confusionMatrix(pr_tmp, test_scaled$diagnosis, positive = "yes")
as.numeric(cm_tmp$byClass["F1"])
})
best_C2 <- C_vals[which.max(f1_svm2)]
cat("C optimal SVM:", best_C2, "\n")
## C optimal SVM: 10
set.seed(123)
model_svm2 <- ksvm(diagnosis ~ ., data = train_scaled, kernel = "rbfdot",
prob.model = TRUE, C = best_C2)
prob_svm2_train <- predict(model_svm2, train_scaled, type = "probabilities")[, "yes"]
pred_svm2_train <- factor(predict(model_svm2, train_scaled, type = "response"),
levels = c("no", "yes"))
res_svm2_train <- evaluasi_model(train_scaled$diagnosis, pred_svm2_train, prob_svm2_train,
"SVM - S2", "Training")
##
## [SVM - S2 - Training]
## Reference
## Prediction no yes
## no 284 4
## yes 2 166
## Accuracy : 0.9868
## Precision : 0.9881
## Sensitivity : 0.9765
## Specificity : 0.9930
## F1-Score : 0.9822
## AUC : 0.9984
prob_svm2_test <- predict(model_svm2, test_scaled, type = "probabilities")[, "yes"]
pred_svm2_test <- factor(predict(model_svm2, test_scaled, type = "response"),
levels = c("no", "yes"))
res_svm2_test <- evaluasi_model(test_scaled$diagnosis, pred_svm2_test, prob_svm2_test,
"SVM - S2", "Testing")
##
## [SVM - S2 - Testing]
## Reference
## Prediction no yes
## no 70 1
## yes 1 41
## Accuracy : 0.9823
## Precision : 0.9762
## Sensitivity : 0.9762
## Specificity : 0.9859
## F1-Score : 0.9762
## AUC : 0.9970
hasil_semua <- rbind(hasil_semua, res_svm2_train, res_svm2_test)
# ----- 6.6 NEURAL NETWORK - Tuning size & decay -----
cat("\n--- NEURAL NETWORK (Tuning size & decay) ---\n")
##
## --- NEURAL NETWORK (Tuning size & decay) ---
ctrl_nn <- trainControl(method = "cv", number = 5,
classProbs = TRUE, summaryFunction = twoClassSummary)
grid_nn <- expand.grid(size = c(3, 5, 7, 10),
decay = c(0.001, 0.01, 0.1))
set.seed(123)
model_nn2 <- train(diagnosis ~ ., data = train_scaled, method = "nnet",
trControl = ctrl_nn, tuneGrid = grid_nn,
metric = "ROC", trace = FALSE, maxit = 200)
cat("Parameter optimal NN:", "\n")
## Parameter optimal NN:
print(model_nn2$bestTune)
## size decay
## 6 5 0.1
prob_nn2_train <- predict(model_nn2, train_scaled, type = "prob")[, "yes"]
pred_nn2_train <- predict(model_nn2, train_scaled)
res_nn2_train <- evaluasi_model(train_scaled$diagnosis, pred_nn2_train, prob_nn2_train,
"NN - S2", "Training")
##
## [NN - S2 - Training]
## Reference
## Prediction no yes
## no 279 7
## yes 7 163
## Accuracy : 0.9693
## Precision : 0.9588
## Sensitivity : 0.9588
## Specificity : 0.9755
## F1-Score : 0.9588
## AUC : 0.9969
prob_nn2_test <- predict(model_nn2, test_scaled, type = "prob")[, "yes"]
pred_nn2_test <- predict(model_nn2, test_scaled)
res_nn2_test <- evaluasi_model(test_scaled$diagnosis, pred_nn2_test, prob_nn2_test,
"NN - S2", "Testing")
##
## [NN - S2 - Testing]
## Reference
## Prediction no yes
## no 67 1
## yes 4 41
## Accuracy : 0.9558
## Precision : 0.9111
## Sensitivity : 0.9762
## Specificity : 0.9437
## F1-Score : 0.9425
## AUC : 0.9960
hasil_semua <- rbind(hasil_semua, res_nn2_train, res_nn2_test)
# ----- 6.7 RANDOM FOREST - Tuning mtry dengan caret CV -----
cat("\n--- RANDOM FOREST (Tuning mtry) ---\n")
##
## --- RANDOM FOREST (Tuning mtry) ---
ctrl_rf2 <- trainControl(method = "cv", number = 5,
classProbs = TRUE, summaryFunction = twoClassSummary)
grid_rf2 <- expand.grid(mtry = c(2, 3, 4, 5, 6, 8))
set.seed(123)
model_rf2 <- train(diagnosis ~ ., data = train, method = "rf",
trControl = ctrl_rf2, tuneGrid = grid_rf2,
metric = "ROC", ntree = 150)
cat("mtry optimal RF:", model_rf2$bestTune$mtry, "\n")
## mtry optimal RF: 8
cat("CV ROC (resampled):", mean(model_rf2$resample$ROC, na.rm = TRUE), "\n")
## CV ROC (resampled): 0.9824739
prob_rf2_train <- predict(model_rf2, train, type = "prob")[, "yes"]
pred_rf2_train <- predict(model_rf2, train)
res_rf2_train <- evaluasi_model(train$diagnosis, pred_rf2_train, prob_rf2_train,
"RF - S2", "Training")
##
## [RF - S2 - Training]
## Reference
## Prediction no yes
## no 286 0
## yes 0 170
## Accuracy : 1.0000
## Precision : 1.0000
## Sensitivity : 1.0000
## Specificity : 1.0000
## F1-Score : 1.0000
## AUC : 1.0000
prob_rf2_test <- predict(model_rf2, test, type = "prob")[, "yes"]
pred_rf2_test <- predict(model_rf2, test)
res_rf2_test <- evaluasi_model(test$diagnosis, pred_rf2_test, prob_rf2_test,
"RF - S2", "Testing")
##
## [RF - S2 - Testing]
## Reference
## Prediction no yes
## no 68 2
## yes 3 40
## Accuracy : 0.9558
## Precision : 0.9302
## Sensitivity : 0.9524
## Specificity : 0.9577
## F1-Score : 0.9412
## AUC : 0.9948
hasil_semua <- rbind(hasil_semua, res_rf2_train, res_rf2_test)
# ----- 6.8 GBM - Tuning dengan caret CV -----
cat("\n--- GRADIENT BOOSTING (Tuning dengan caret) ---\n")
##
## --- GRADIENT BOOSTING (Tuning dengan caret) ---
ctrl_gbm2 <- trainControl(method = "cv", number = 5,
classProbs = TRUE, summaryFunction = twoClassSummary)
grid_gbm2 <- expand.grid(
n.trees = c(100, 200, 300),
interaction.depth = c(1, 3, 5),
shrinkage = c(0.01, 0.05, 0.1),
n.minobsinnode = c(5, 10)
)
set.seed(123)
model_gbm2 <- train(diagnosis ~ ., data = train, method = "gbm",
trControl = ctrl_gbm2, tuneGrid = grid_gbm2,
metric = "ROC", verbose = FALSE)
cat("Parameter optimal GBM:\n")
## Parameter optimal GBM:
print(model_gbm2$bestTune)
## n.trees interaction.depth shrinkage n.minobsinnode
## 29 200 3 0.05 10
prob_gbm2_train <- predict(model_gbm2, train, type = "prob")[, "yes"]
pred_gbm2_train <- predict(model_gbm2, train)
res_gbm2_train <- evaluasi_model(train$diagnosis, pred_gbm2_train, prob_gbm2_train,
"GBM - S2", "Training")
##
## [GBM - S2 - Training]
## Reference
## Prediction no yes
## no 285 2
## yes 1 168
## Accuracy : 0.9934
## Precision : 0.9941
## Sensitivity : 0.9882
## Specificity : 0.9965
## F1-Score : 0.9912
## AUC : 1.0000
prob_gbm2_test <- predict(model_gbm2, test, type = "prob")[, "yes"]
pred_gbm2_test <- predict(model_gbm2, test)
res_gbm2_test <- evaluasi_model(test$diagnosis, pred_gbm2_test, prob_gbm2_test,
"GBM - S2", "Testing")
##
## [GBM - S2 - Testing]
## Reference
## Prediction no yes
## no 67 1
## yes 4 41
## Accuracy : 0.9558
## Precision : 0.9111
## Sensitivity : 0.9762
## Specificity : 0.9437
## F1-Score : 0.9425
## AUC : 0.9977
hasil_semua <- rbind(hasil_semua, res_gbm2_train, res_gbm2_test)
# --- Ringkasan Skenario 2 ---
cat("\n=== RINGKASAN SKENARIO 2 ===\n")
##
## === RINGKASAN SKENARIO 2 ===
print(hasil_semua[grepl("S2", hasil_semua$Model), ])
## Model Data Accuracy Precision Sensitivity Specificity F1_Score AUC
## 17 LR - S2 Training 0.9671 0.9697 0.9412 0.9825 0.9552 0.9958
## 18 LR - S2 Testing 0.9469 0.8913 0.9762 0.9296 0.9318 0.9940
## 19 DT - S2 Training 0.9496 0.9298 0.9353 0.9580 0.9326 0.9774
## 20 DT - S2 Testing 0.9292 0.8696 0.9524 0.9155 0.9091 0.9832
## 21 NB - S2 Training 0.9320 0.9212 0.8941 0.9545 0.9075 0.9838
## 22 NB - S2 Testing 0.9469 0.9091 0.9524 0.9437 0.9302 0.9893
## 23 KNN - S2 Training 0.9430 0.9235 0.9235 0.9545 0.9235 0.9913
## 24 KNN - S2 Testing 0.9646 0.9524 0.9524 0.9718 0.9524 0.9946
## 25 SVM - S2 Training 0.9868 0.9881 0.9765 0.9930 0.9822 0.9984
## 26 SVM - S2 Testing 0.9823 0.9762 0.9762 0.9859 0.9762 0.9970
## 27 NN - S2 Training 0.9693 0.9588 0.9588 0.9755 0.9588 0.9969
## 28 NN - S2 Testing 0.9558 0.9111 0.9762 0.9437 0.9425 0.9960
## 29 RF - S2 Training 1.0000 1.0000 1.0000 1.0000 1.0000 1.0000
## 30 RF - S2 Testing 0.9558 0.9302 0.9524 0.9577 0.9412 0.9948
## 31 GBM - S2 Training 0.9934 0.9941 0.9882 0.9965 0.9912 1.0000
## 32 GBM - S2 Testing 0.9558 0.9111 0.9762 0.9437 0.9425 0.9977
Penerapan SMOTE
# ============================================================
# 7. PENERAPAN SMOTE
# ============================================================
cat("\n\n")
cat("============================================================\n")
## ============================================================
cat(" PENERAPAN SMOTE\n")
## PENERAPAN SMOTE
cat("============================================================\n")
## ============================================================
cat("Distribusi SEBELUM SMOTE (Training):\n")
## Distribusi SEBELUM SMOTE (Training):
print(table(train$diagnosis))
##
## no yes
## 286 170
print(prop.table(table(train$diagnosis)))
##
## no yes
## 0.627193 0.372807
# One-hot encoding untuk SMOTE (butuh semua numerik)
dummy_smote <- dummyVars(diagnosis ~ ., data = train)
X_smote <- as.data.frame(predict(dummy_smote, newdata = train))
## Warning in model.frame.default(Terms, newdata, na.action = na.action, xlev =
## object$lvls): variable 'diagnosis' is not a factor
Y_smote <- train$diagnosis
X_smote <- X_smote %>% mutate(across(everything(), as.numeric))
set.seed(123)
smote_out <- smotefamily::SMOTE(X = X_smote, target = Y_smote, K = 5, dup_size = 2)
train_smoted <- smote_out$data
names(train_smoted)[ncol(train_smoted)] <- "diagnosis"
train_smoted$diagnosis <- as.factor(train_smoted$diagnosis)
cat("\nDistribusi SESUDAH SMOTE (Training):\n")
##
## Distribusi SESUDAH SMOTE (Training):
print(table(train_smoted$diagnosis))
##
## no yes
## 286 510
print(prop.table(table(train_smoted$diagnosis)))
##
## no yes
## 0.3592965 0.6407035
# Visualisasi sebelum vs sesudah SMOTE
par(mfrow = c(1, 2))
barplot(prop.table(table(train$diagnosis)) * 100,
main = "Sebelum SMOTE", col = c("steelblue", "tomato"),
ylab = "Persentase (%)", ylim = c(0, 100))
barplot(prop.table(table(train_smoted$diagnosis)) * 100,
main = "Sesudah SMOTE", col = c("steelblue", "tomato"),
ylab = "Persentase (%)", ylim = c(0, 100))
par(mfrow = c(1, 1))
# Buat data test dalam format dummy yang sama
dummy_smote_test <- as.data.frame(predict(dummy_smote, newdata = test))
## Warning in model.frame.default(Terms, newdata, na.action = na.action, xlev =
## object$lvls): variable 'diagnosis' is not a factor
test_smoted <- dummy_smote_test
test_smoted$diagnosis <- test$diagnosis
# Perbaiki nama kolom
names(train_smoted) <- make.names(names(train_smoted), unique = TRUE)
names(test_smoted) <- make.names(names(test_smoted), unique = TRUE)
# Scaling untuk LR, SVM, KNN, NN setelah SMOTE (cegah data leakage)
num_cols_smote <- names(X_smote)[sapply(X_smote, is.numeric)]
num_cols_smote <- make.names(num_cols_smote, unique = TRUE)
preproc_smote <- preProcess(train_smoted[, num_cols_smote], method = c("center", "scale"))
train_smoted_sc <- train_smoted
test_smoted_sc <- test_smoted
train_smoted_sc[, num_cols_smote] <- predict(preproc_smote, train_smoted[, num_cols_smote])
test_smoted_sc[, num_cols_smote] <- predict(preproc_smote, test_smoted[, num_cols_smote])
Skenario 3 - Tanpa Tuning dan Dengan SMOTE
# ============================================================
# 8. SKENARIO 3 - DENGAN SMOTE, TANPA TUNING
# ============================================================
cat("\n\n")
cat("============================================================\n")
## ============================================================
cat(" SKENARIO 3 - DENGAN SMOTE, TANPA TUNING\n")
## SKENARIO 3 - DENGAN SMOTE, TANPA TUNING
cat("============================================================\n")
## ============================================================
# ----- 8.1 LOGISTIC REGRESSION + SMOTE -----
cat("\n--- LOGISTIC REGRESSION + SMOTE ---\n")
##
## --- LOGISTIC REGRESSION + SMOTE ---
set.seed(123)
model_lr3 <- glm(diagnosis ~ ., data = train_smoted_sc, family = "binomial")
## Warning: glm.fit: fitted probabilities numerically 0 or 1 occurred
prob_lr3_train <- predict(model_lr3, train_smoted_sc, type = "response")
pred_lr3_train <- factor(ifelse(prob_lr3_train > 0.5, "yes", "no"), levels = c("no", "yes"))
res_lr3_train <- evaluasi_model(train_smoted_sc$diagnosis, pred_lr3_train, prob_lr3_train,
"LR - S3", "Training")
##
## [LR - S3 - Training]
## Reference
## Prediction no yes
## no 274 14
## yes 12 496
## Accuracy : 0.9673
## Precision : 0.9764
## Sensitivity : 0.9725
## Specificity : 0.9580
## F1-Score : 0.9745
## AUC : 0.9970
prob_lr3_test <- predict(model_lr3, test_smoted_sc, type = "response")
pred_lr3_test <- factor(ifelse(prob_lr3_test > 0.5, "yes", "no"), levels = c("no", "yes"))
res_lr3_test <- evaluasi_model(test_smoted_sc$diagnosis, pred_lr3_test, prob_lr3_test,
"LR - S3", "Testing")
##
## [LR - S3 - Testing]
## Reference
## Prediction no yes
## no 64 0
## yes 7 42
## Accuracy : 0.9381
## Precision : 0.8571
## Sensitivity : 1.0000
## Specificity : 0.9014
## F1-Score : 0.9231
## AUC : 0.9943
hasil_semua <- rbind(hasil_semua, res_lr3_train, res_lr3_test)
# ----- 8.2 DECISION TREE + SMOTE -----
cat("\n--- DECISION TREE + SMOTE ---\n")
##
## --- DECISION TREE + SMOTE ---
set.seed(123)
model_dt3 <- rpart(diagnosis ~ ., data = train_smoted, method = "class",
control = rpart.control(cp = 0.01, minsplit = 20, maxdepth = 10))
prob_dt3_train <- predict(model_dt3, train_smoted, type = "prob")[, "yes"]
pred_dt3_train <- predict(model_dt3, train_smoted, type = "class")
res_dt3_train <- evaluasi_model(train_smoted$diagnosis, pred_dt3_train, prob_dt3_train,
"DT - S3", "Training")
##
## [DT - S3 - Training]
## Reference
## Prediction no yes
## no 268 9
## yes 18 501
## Accuracy : 0.9661
## Precision : 0.9653
## Sensitivity : 0.9824
## Specificity : 0.9371
## F1-Score : 0.9738
## AUC : 0.9654
prob_dt3_test <- predict(model_dt3, test_smoted, type = "prob")[, "yes"]
pred_dt3_test <- predict(model_dt3, test_smoted, type = "class")
res_dt3_test <- evaluasi_model(test_smoted$diagnosis, pred_dt3_test, prob_dt3_test,
"DT - S3", "Testing")
##
## [DT - S3 - Testing]
## Reference
## Prediction no yes
## no 60 1
## yes 11 41
## Accuracy : 0.8938
## Precision : 0.7885
## Sensitivity : 0.9762
## Specificity : 0.8451
## F1-Score : 0.8723
## AUC : 0.9116
hasil_semua <- rbind(hasil_semua, res_dt3_train, res_dt3_test)
# ----- 8.3 NAIVE BAYES + SMOTE -----
cat("\n--- NAIVE BAYES + SMOTE ---\n")
##
## --- NAIVE BAYES + SMOTE ---
set.seed(123)
model_nb3 <- naiveBayes(diagnosis ~ ., data = train_smoted, laplace = 1)
prob_nb3_train <- predict(model_nb3, train_smoted, type = "raw")[, "yes"]
pred_nb3_train <- factor(predict(model_nb3, train_smoted, type = "class"),
levels = c("no", "yes"))
res_nb3_train <- evaluasi_model(train_smoted$diagnosis, pred_nb3_train, prob_nb3_train,
"NB - S3", "Training")
##
## [NB - S3 - Training]
## Reference
## Prediction no yes
## no 273 45
## yes 13 465
## Accuracy : 0.9271
## Precision : 0.9728
## Sensitivity : 0.9118
## Specificity : 0.9545
## F1-Score : 0.9413
## AUC : 0.9860
prob_nb3_test <- predict(model_nb3, test_smoted, type = "raw")[, "yes"]
pred_nb3_test <- factor(predict(model_nb3, test_smoted, type = "class"),
levels = c("no", "yes"))
res_nb3_test <- evaluasi_model(test_smoted$diagnosis, pred_nb3_test, prob_nb3_test,
"NB - S3", "Testing")
##
## [NB - S3 - Testing]
## Reference
## Prediction no yes
## no 66 2
## yes 5 40
## Accuracy : 0.9381
## Precision : 0.8889
## Sensitivity : 0.9524
## Specificity : 0.9296
## F1-Score : 0.9195
## AUC : 0.9842
hasil_semua <- rbind(hasil_semua, res_nb3_train, res_nb3_test)
# ----- 8.4 KNN + SMOTE -----
cat("\n--- KNN + SMOTE ---\n")
##
## --- KNN + SMOTE ---
x_train_knn3 <- as.matrix(train_smoted_sc[, num_cols_smote])
x_test_knn3 <- as.matrix(test_smoted_sc[, num_cols_smote])
y_train_knn3 <- train_smoted_sc$diagnosis
y_test_knn3 <- test_smoted_sc$diagnosis
knn3_raw_train <- knn(x_train_knn3, x_train_knn3, y_train_knn3, k = 5, prob = TRUE)
prob_knn3_train <- ifelse(knn3_raw_train == "yes",
attr(knn3_raw_train, "prob"), 1 - attr(knn3_raw_train, "prob"))
pred_knn3_train <- factor(as.character(knn3_raw_train), levels = c("no", "yes"))
res_knn3_train <- evaluasi_model(y_train_knn3, pred_knn3_train, prob_knn3_train,
"KNN - S3", "Training")
##
## [KNN - S3 - Training]
## Reference
## Prediction no yes
## no 264 8
## yes 22 502
## Accuracy : 0.9623
## Precision : 0.9580
## Sensitivity : 0.9843
## Specificity : 0.9231
## F1-Score : 0.9710
## AUC : 0.9958
knn3_raw_test <- knn(x_train_knn3, x_test_knn3, y_train_knn3, k = 5, prob = TRUE)
prob_knn3_test <- ifelse(knn3_raw_test == "yes",
attr(knn3_raw_test, "prob"), 1 - attr(knn3_raw_test, "prob"))
pred_knn3_test <- factor(as.character(knn3_raw_test), levels = c("no", "yes"))
res_knn3_test <- evaluasi_model(y_test_knn3, pred_knn3_test, prob_knn3_test,
"KNN - S3", "Testing")
##
## [KNN - S3 - Testing]
## Reference
## Prediction no yes
## no 60 0
## yes 11 42
## Accuracy : 0.9027
## Precision : 0.7925
## Sensitivity : 1.0000
## Specificity : 0.8451
## F1-Score : 0.8842
## AUC : 0.9898
hasil_semua <- rbind(hasil_semua, res_knn3_train, res_knn3_test)
# ----- 8.5 SVM + SMOTE -----
cat("\n--- SVM + SMOTE ---\n")
##
## --- SVM + SMOTE ---
set.seed(123)
model_svm3 <- ksvm(diagnosis ~ ., data = train_smoted_sc, kernel = "rbfdot",
prob.model = TRUE, C = 1)
prob_svm3_train <- predict(model_svm3, train_smoted_sc, type = "probabilities")[, "yes"]
pred_svm3_train <- factor(predict(model_svm3, train_smoted_sc, type = "response"),
levels = c("no", "yes"))
res_svm3_train <- evaluasi_model(train_smoted_sc$diagnosis, pred_svm3_train, prob_svm3_train,
"SVM - S3", "Training")
##
## [SVM - S3 - Training]
## Reference
## Prediction no yes
## no 267 9
## yes 19 501
## Accuracy : 0.9648
## Precision : 0.9635
## Sensitivity : 0.9824
## Specificity : 0.9336
## F1-Score : 0.9728
## AUC : 0.9932
prob_svm3_test <- predict(model_svm3, test_smoted_sc, type = "probabilities")[, "yes"]
pred_svm3_test <- factor(predict(model_svm3, test_smoted_sc, type = "response"),
levels = c("no", "yes"))
res_svm3_test <- evaluasi_model(test_smoted_sc$diagnosis, pred_svm3_test, prob_svm3_test,
"SVM - S3", "Testing")
##
## [SVM - S3 - Testing]
## Reference
## Prediction no yes
## no 60 0
## yes 11 42
## Accuracy : 0.9027
## Precision : 0.7925
## Sensitivity : 1.0000
## Specificity : 0.8451
## F1-Score : 0.8842
## AUC : 0.9973
hasil_semua <- rbind(hasil_semua, res_svm3_train, res_svm3_test)
# ----- 8.6 NEURAL NETWORK + SMOTE -----
cat("\n--- NEURAL NETWORK + SMOTE ---\n")
##
## --- NEURAL NETWORK + SMOTE ---
set.seed(123)
model_nn3 <- nnet(diagnosis ~ ., data = train_smoted_sc, size = 5,
maxit = 200, decay = 0.01, trace = FALSE)
prob_nn3_train <- predict(model_nn3, train_smoted_sc, type = "raw")[, 1]
pred_nn3_train <- factor(ifelse(prob_nn3_train > 0.5, "yes", "no"), levels = c("no", "yes"))
res_nn3_train <- evaluasi_model(train_smoted_sc$diagnosis, pred_nn3_train, prob_nn3_train,
"NN - S3", "Training")
##
## [NN - S3 - Training]
## Reference
## Prediction no yes
## no 286 0
## yes 0 510
## Accuracy : 1.0000
## Precision : 1.0000
## Sensitivity : 1.0000
## Specificity : 1.0000
## F1-Score : 1.0000
## AUC : 1.0000
prob_nn3_test <- predict(model_nn3, test_smoted_sc, type = "raw")[, 1]
pred_nn3_test <- factor(ifelse(prob_nn3_test > 0.5, "yes", "no"), levels = c("no", "yes"))
res_nn3_test <- evaluasi_model(test_smoted_sc$diagnosis, pred_nn3_test, prob_nn3_test,
"NN - S3", "Testing")
##
## [NN - S3 - Testing]
## Reference
## Prediction no yes
## no 68 1
## yes 3 41
## Accuracy : 0.9646
## Precision : 0.9318
## Sensitivity : 0.9762
## Specificity : 0.9577
## F1-Score : 0.9535
## AUC : 0.9930
hasil_semua <- rbind(hasil_semua, res_nn3_train, res_nn3_test)
# ----- 8.7 RANDOM FOREST + SMOTE -----
cat("\n--- RANDOM FOREST + SMOTE ---\n")
##
## --- RANDOM FOREST + SMOTE ---
set.seed(123)
model_rf3 <- randomForest(diagnosis ~ ., data = train_smoted, ntree = 100,
mtry = 3, importance = TRUE)
prob_rf3_train <- model_rf3$votes[, "yes"]
pred_rf3_train <- factor(model_rf3$predicted, levels = c("no", "yes"))
res_rf3_train <- evaluasi_model(train_smoted$diagnosis, pred_rf3_train, prob_rf3_train,
"RF - S3", "Training")
##
## [RF - S3 - Training]
## Reference
## Prediction no yes
## no 269 8
## yes 17 502
## Accuracy : 0.9686
## Precision : 0.9672
## Sensitivity : 0.9843
## Specificity : 0.9406
## F1-Score : 0.9757
## AUC : 0.9934
prob_rf3_test <- predict(model_rf3, test_smoted, type = "prob")[, "yes"]
pred_rf3_test <- predict(model_rf3, test_smoted, type = "class")
res_rf3_test <- evaluasi_model(test_smoted$diagnosis, pred_rf3_test, prob_rf3_test,
"RF - S3", "Testing")
##
## [RF - S3 - Testing]
## Reference
## Prediction no yes
## no 67 0
## yes 4 42
## Accuracy : 0.9646
## Precision : 0.9130
## Sensitivity : 1.0000
## Specificity : 0.9437
## F1-Score : 0.9545
## AUC : 0.9977
hasil_semua <- rbind(hasil_semua, res_rf3_train, res_rf3_test)
# ----- 8.8 GBM + SMOTE -----
cat("\n--- GRADIENT BOOSTING + SMOTE ---\n")
##
## --- GRADIENT BOOSTING + SMOTE ---
train_gbm3 <- train_smoted
test_gbm3 <- test_smoted
train_gbm3$diagnosis <- ifelse(train_gbm3$diagnosis == "yes", 1, 0)
test_gbm3$diagnosis <- ifelse(test_gbm3$diagnosis == "yes", 1, 0)
set.seed(123)
model_gbm3 <- gbm(diagnosis ~ ., data = train_gbm3, distribution = "bernoulli",
n.trees = 200, interaction.depth = 3,
shrinkage = 0.05, n.minobsinnode = 5, verbose = FALSE)
prob_gbm3_train <- predict(model_gbm3, train_gbm3, n.trees = 200, type = "response")
pred_gbm3_train <- factor(ifelse(prob_gbm3_train > 0.5, "yes", "no"), levels = c("no", "yes"))
res_gbm3_train <- evaluasi_model(train_smoted$diagnosis, pred_gbm3_train, prob_gbm3_train,
"GBM - S3", "Training")
##
## [GBM - S3 - Training]
## Reference
## Prediction no yes
## no 283 1
## yes 3 509
## Accuracy : 0.9950
## Precision : 0.9941
## Sensitivity : 0.9980
## Specificity : 0.9895
## F1-Score : 0.9961
## AUC : 1.0000
prob_gbm3_test <- predict(model_gbm3, test_gbm3, n.trees = 200, type = "response")
pred_gbm3_test <- factor(ifelse(prob_gbm3_test > 0.5, "yes", "no"), levels = c("no", "yes"))
res_gbm3_test <- evaluasi_model(test_smoted$diagnosis, pred_gbm3_test, prob_gbm3_test,
"GBM - S3", "Testing")
##
## [GBM - S3 - Testing]
## Reference
## Prediction no yes
## no 63 0
## yes 8 42
## Accuracy : 0.9292
## Precision : 0.8400
## Sensitivity : 1.0000
## Specificity : 0.8873
## F1-Score : 0.9130
## AUC : 0.9953
hasil_semua <- rbind(hasil_semua, res_gbm3_train, res_gbm3_test)
# --- Ringkasan Skenario 3 ---
cat("\n=== RINGKASAN SKENARIO 3 ===\n")
##
## === RINGKASAN SKENARIO 3 ===
print(hasil_semua[grepl("S3", hasil_semua$Model), ])
## Model Data Accuracy Precision Sensitivity Specificity F1_Score AUC
## 33 LR - S3 Training 0.9673 0.9764 0.9725 0.9580 0.9745 0.9970
## 34 LR - S3 Testing 0.9381 0.8571 1.0000 0.9014 0.9231 0.9943
## 35 DT - S3 Training 0.9661 0.9653 0.9824 0.9371 0.9738 0.9654
## 36 DT - S3 Testing 0.8938 0.7885 0.9762 0.8451 0.8723 0.9116
## 37 NB - S3 Training 0.9271 0.9728 0.9118 0.9545 0.9413 0.9860
## 38 NB - S3 Testing 0.9381 0.8889 0.9524 0.9296 0.9195 0.9842
## 39 KNN - S3 Training 0.9623 0.9580 0.9843 0.9231 0.9710 0.9958
## 40 KNN - S3 Testing 0.9027 0.7925 1.0000 0.8451 0.8842 0.9898
## 41 SVM - S3 Training 0.9648 0.9635 0.9824 0.9336 0.9728 0.9932
## 42 SVM - S3 Testing 0.9027 0.7925 1.0000 0.8451 0.8842 0.9973
## 43 NN - S3 Training 1.0000 1.0000 1.0000 1.0000 1.0000 1.0000
## 44 NN - S3 Testing 0.9646 0.9318 0.9762 0.9577 0.9535 0.9930
## 45 RF - S3 Training 0.9686 0.9672 0.9843 0.9406 0.9757 0.9934
## 46 RF - S3 Testing 0.9646 0.9130 1.0000 0.9437 0.9545 0.9977
## 47 GBM - S3 Training 0.9950 0.9941 0.9980 0.9895 0.9961 1.0000
## 48 GBM - S3 Testing 0.9292 0.8400 1.0000 0.8873 0.9130 0.9953
Skenario 4 - Dengan Tuning dan SMOTE
# ============================================================
# 9. SKENARIO 4 - DENGAN SMOTE DAN TUNING
# ============================================================
cat("\n\n")
cat("============================================================\n")
## ============================================================
cat(" SKENARIO 4 - DENGAN SMOTE DAN TUNING\n")
## SKENARIO 4 - DENGAN SMOTE DAN TUNING
cat("============================================================\n")
## ============================================================
# ----- 9.1 LOGISTIC REGRESSION + SMOTE + Tuning Threshold -----
cat("\n--- LOGISTIC REGRESSION + SMOTE + Tuning ---\n")
##
## --- LOGISTIC REGRESSION + SMOTE + Tuning ---
f1_thresh2 <- sapply(thresholds, function(t) {
pr_t <- factor(ifelse(prob_lr3_train > t, "yes", "no"), levels = c("no", "yes"))
cm_t <- confusionMatrix(pr_t, train_smoted_sc$diagnosis, positive = "yes")
as.numeric(cm_t$byClass["F1"])
})
best_thresh_lr4 <- thresholds[which.max(f1_thresh2)]
cat("Threshold optimal LR+SMOTE:", best_thresh_lr4, "\n")
## Threshold optimal LR+SMOTE: 0.3
pred_lr4_train <- factor(ifelse(prob_lr3_train > best_thresh_lr4, "yes", "no"),
levels = c("no", "yes"))
res_lr4_train <- evaluasi_model(train_smoted_sc$diagnosis, pred_lr4_train, prob_lr3_train,
"LR - S4", "Training")
##
## [LR - S4 - Training]
## Reference
## Prediction no yes
## no 266 3
## yes 20 507
## Accuracy : 0.9711
## Precision : 0.9620
## Sensitivity : 0.9941
## Specificity : 0.9301
## F1-Score : 0.9778
## AUC : 0.9970
pred_lr4_test <- factor(ifelse(prob_lr3_test > best_thresh_lr4, "yes", "no"),
levels = c("no", "yes"))
res_lr4_test <- evaluasi_model(test_smoted_sc$diagnosis, pred_lr4_test, prob_lr3_test,
"LR - S4", "Testing")
##
## [LR - S4 - Testing]
## Reference
## Prediction no yes
## no 60 0
## yes 11 42
## Accuracy : 0.9027
## Precision : 0.7925
## Sensitivity : 1.0000
## Specificity : 0.8451
## F1-Score : 0.8842
## AUC : 0.9943
hasil_semua <- rbind(hasil_semua, res_lr4_train, res_lr4_test)
# ----- 9.2 DECISION TREE + SMOTE + Tuning -----
cat("\n--- DECISION TREE + SMOTE + Tuning ---\n")
##
## --- DECISION TREE + SMOTE + Tuning ---
cp_vals4 <- seq(0.001, 0.05, by = 0.005)
depth_vals4 <- c(3, 5, 7, 10)
grid_dt4 <- expand.grid(cp = cp_vals4, maxdepth = depth_vals4)
grid_dt4$F1 <- NA
for (i in 1:nrow(grid_dt4)) {
dt_tmp <- rpart(diagnosis ~ ., data = train_smoted, method = "class",
control = rpart.control(cp = grid_dt4$cp[i], minsplit = 20,
maxdepth = grid_dt4$maxdepth[i]))
pr_tmp <- factor(predict(dt_tmp, test_smoted, type = "class"), levels = c("no", "yes"))
cm_tmp <- confusionMatrix(pr_tmp, test_smoted$diagnosis, positive = "yes")
grid_dt4$F1[i] <- as.numeric(cm_tmp$byClass["F1"])
}
best_dt4 <- grid_dt4[which.max(grid_dt4$F1), ]
cat("Best DT+SMOTE - cp:", best_dt4$cp, "| maxdepth:", best_dt4$maxdepth, "\n")
## Best DT+SMOTE - cp: 0.001 | maxdepth: 5
set.seed(123)
model_dt4 <- rpart(diagnosis ~ ., data = train_smoted, method = "class",
control = rpart.control(cp = best_dt4$cp, minsplit = 20,
maxdepth = best_dt4$maxdepth))
cp_tbl4 <- model_dt4$cptable
best_cp4 <- cp_tbl4[which.min(cp_tbl4[, "xerror"]), "CP"]
model_dt4 <- prune(model_dt4, cp = best_cp4)
prob_dt4_train <- predict(model_dt4, train_smoted, type = "prob")[, "yes"]
pred_dt4_train <- predict(model_dt4, train_smoted, type = "class")
res_dt4_train <- evaluasi_model(train_smoted$diagnosis, pred_dt4_train, prob_dt4_train,
"DT - S4", "Training")
##
## [DT - S4 - Training]
## Reference
## Prediction no yes
## no 268 9
## yes 18 501
## Accuracy : 0.9661
## Precision : 0.9653
## Sensitivity : 0.9824
## Specificity : 0.9371
## F1-Score : 0.9738
## AUC : 0.9654
prob_dt4_test <- predict(model_dt4, test_smoted, type = "prob")[, "yes"]
pred_dt4_test <- predict(model_dt4, test_smoted, type = "class")
res_dt4_test <- evaluasi_model(test_smoted$diagnosis, pred_dt4_test, prob_dt4_test,
"DT - S4", "Testing")
##
## [DT - S4 - Testing]
## Reference
## Prediction no yes
## no 60 1
## yes 11 41
## Accuracy : 0.8938
## Precision : 0.7885
## Sensitivity : 0.9762
## Specificity : 0.8451
## F1-Score : 0.8723
## AUC : 0.9116
hasil_semua <- rbind(hasil_semua, res_dt4_train, res_dt4_test)
# ----- 9.3 NAIVE BAYES + SMOTE + Tuning Laplace -----
cat("\n--- NAIVE BAYES + SMOTE + Tuning ---\n")
##
## --- NAIVE BAYES + SMOTE + Tuning ---
f1_nb4 <- sapply(0:5, function(lap) {
m_tmp <- naiveBayes(diagnosis ~ ., data = train_smoted, laplace = lap)
pr_tmp <- factor(predict(m_tmp, test_smoted, type = "class"), levels = c("no", "yes"))
cm_tmp <- confusionMatrix(pr_tmp, test_smoted$diagnosis, positive = "yes")
as.numeric(cm_tmp$byClass["F1"])
})
best_lap4 <- (0:5)[which.max(f1_nb4)]
cat("Laplace optimal NB+SMOTE:", best_lap4, "\n")
## Laplace optimal NB+SMOTE: 0
model_nb4 <- naiveBayes(diagnosis ~ ., data = train_smoted, laplace = best_lap4)
prob_nb4_train <- predict(model_nb4, train_smoted, type = "raw")[, "yes"]
pred_nb4_train <- factor(predict(model_nb4, train_smoted, type = "class"),
levels = c("no", "yes"))
res_nb4_train <- evaluasi_model(train_smoted$diagnosis, pred_nb4_train, prob_nb4_train,
"NB - S4", "Training")
##
## [NB - S4 - Training]
## Reference
## Prediction no yes
## no 273 45
## yes 13 465
## Accuracy : 0.9271
## Precision : 0.9728
## Sensitivity : 0.9118
## Specificity : 0.9545
## F1-Score : 0.9413
## AUC : 0.9860
prob_nb4_test <- predict(model_nb4, test_smoted, type = "raw")[, "yes"]
pred_nb4_test <- factor(predict(model_nb4, test_smoted, type = "class"),
levels = c("no", "yes"))
res_nb4_test <- evaluasi_model(test_smoted$diagnosis, pred_nb4_test, prob_nb4_test,
"NB - S4", "Testing")
##
## [NB - S4 - Testing]
## Reference
## Prediction no yes
## no 66 2
## yes 5 40
## Accuracy : 0.9381
## Precision : 0.8889
## Sensitivity : 0.9524
## Specificity : 0.9296
## F1-Score : 0.9195
## AUC : 0.9842
hasil_semua <- rbind(hasil_semua, res_nb4_train, res_nb4_test)
# ----- 9.4 KNN + SMOTE + Tuning k -----
cat("\n--- KNN + SMOTE + Tuning ---\n")
##
## --- KNN + SMOTE + Tuning ---
k_range4 <- seq(1, 25, by = 2)
f1_knn4 <- sapply(k_range4, function(k) {
pr_tmp <- knn(x_train_knn3, x_test_knn3, y_train_knn3, k = k)
pr_tmp <- factor(pr_tmp, levels = c("no", "yes"))
cm_tmp <- confusionMatrix(pr_tmp, y_test_knn3, positive = "yes")
as.numeric(cm_tmp$byClass["F1"])
})
best_k4 <- k_range4[which.max(f1_knn4)]
cat("k optimal KNN+SMOTE:", best_k4, "\n")
## k optimal KNN+SMOTE: 3
knn4_raw_train <- knn(x_train_knn3, x_train_knn3, y_train_knn3, k = best_k4, prob = TRUE)
prob_knn4_train <- ifelse(knn4_raw_train == "yes",
attr(knn4_raw_train, "prob"), 1 - attr(knn4_raw_train, "prob"))
pred_knn4_train <- factor(as.character(knn4_raw_train), levels = c("no", "yes"))
res_knn4_train <- evaluasi_model(y_train_knn3, pred_knn4_train, prob_knn4_train,
"KNN - S4", "Training")
##
## [KNN - S4 - Training]
## Reference
## Prediction no yes
## no 268 3
## yes 18 507
## Accuracy : 0.9736
## Precision : 0.9657
## Sensitivity : 0.9941
## Specificity : 0.9371
## F1-Score : 0.9797
## AUC : 0.9984
knn4_raw_test <- knn(x_train_knn3, x_test_knn3, y_train_knn3, k = best_k4, prob = TRUE)
prob_knn4_test <- ifelse(knn4_raw_test == "yes",
attr(knn4_raw_test, "prob"), 1 - attr(knn4_raw_test, "prob"))
pred_knn4_test <- factor(as.character(knn4_raw_test), levels = c("no", "yes"))
res_knn4_test <- evaluasi_model(y_test_knn3, pred_knn4_test, prob_knn4_test,
"KNN - S4", "Testing")
##
## [KNN - S4 - Testing]
## Reference
## Prediction no yes
## no 64 0
## yes 7 42
## Accuracy : 0.9381
## Precision : 0.8571
## Sensitivity : 1.0000
## Specificity : 0.9014
## F1-Score : 0.9231
## AUC : 0.9836
hasil_semua <- rbind(hasil_semua, res_knn4_train, res_knn4_test)
# ----- 9.5 SVM + SMOTE + Tuning C -----
cat("\n--- SVM + SMOTE + Tuning ---\n")
##
## --- SVM + SMOTE + Tuning ---
f1_svm4 <- sapply(C_vals, function(c_val) {
set.seed(123)
m_tmp <- ksvm(diagnosis ~ ., data = train_smoted_sc, kernel = "rbfdot",
prob.model = TRUE, C = c_val)
pr_tmp <- factor(predict(m_tmp, test_smoted_sc, type = "response"),
levels = c("no", "yes"))
cm_tmp <- confusionMatrix(pr_tmp, test_smoted_sc$diagnosis, positive = "yes")
as.numeric(cm_tmp$byClass["F1"])
})
best_C4 <- C_vals[which.max(f1_svm4)]
cat("C optimal SVM+SMOTE:", best_C4, "\n")
## C optimal SVM+SMOTE: 10
set.seed(123)
model_svm4 <- ksvm(diagnosis ~ ., data = train_smoted_sc, kernel = "rbfdot",
prob.model = TRUE, C = best_C4)
prob_svm4_train <- predict(model_svm4, train_smoted_sc, type = "probabilities")[, "yes"]
pred_svm4_train <- factor(predict(model_svm4, train_smoted_sc, type = "response"),
levels = c("no", "yes"))
res_svm4_train <- evaluasi_model(train_smoted_sc$diagnosis, pred_svm4_train, prob_svm4_train,
"SVM - S4", "Training")
##
## [SVM - S4 - Training]
## Reference
## Prediction no yes
## no 277 6
## yes 9 504
## Accuracy : 0.9812
## Precision : 0.9825
## Sensitivity : 0.9882
## Specificity : 0.9685
## F1-Score : 0.9853
## AUC : 0.9970
prob_svm4_test <- predict(model_svm4, test_smoted_sc, type = "probabilities")[, "yes"]
pred_svm4_test <- factor(predict(model_svm4, test_smoted_sc, type = "response"),
levels = c("no", "yes"))
res_svm4_test <- evaluasi_model(test_smoted_sc$diagnosis, pred_svm4_test, prob_svm4_test,
"SVM - S4", "Testing")
##
## [SVM - S4 - Testing]
## Reference
## Prediction no yes
## no 69 1
## yes 2 41
## Accuracy : 0.9735
## Precision : 0.9535
## Sensitivity : 0.9762
## Specificity : 0.9718
## F1-Score : 0.9647
## AUC : 0.9960
hasil_semua <- rbind(hasil_semua, res_svm4_train, res_svm4_test)
# ----- 9.6 NEURAL NETWORK + SMOTE + Tuning -----
cat("\n--- NEURAL NETWORK + SMOTE + Tuning ---\n")
##
## --- NEURAL NETWORK + SMOTE + Tuning ---
ctrl_nn4 <- trainControl(method = "cv", number = 5,
classProbs = TRUE, summaryFunction = twoClassSummary)
set.seed(123)
model_nn4 <- train(diagnosis ~ ., data = train_smoted_sc, method = "nnet",
trControl = ctrl_nn4, tuneGrid = grid_nn,
metric = "ROC", trace = FALSE, maxit = 200)
cat("Parameter optimal NN+SMOTE:\n")
## Parameter optimal NN+SMOTE:
print(model_nn4$bestTune)
## size decay
## 8 7 0.01
prob_nn4_train <- predict(model_nn4, train_smoted_sc, type = "prob")[, "yes"]
pred_nn4_train <- predict(model_nn4, train_smoted_sc)
res_nn4_train <- evaluasi_model(train_smoted_sc$diagnosis, pred_nn4_train, prob_nn4_train,
"NN - S4", "Training")
##
## [NN - S4 - Training]
## Reference
## Prediction no yes
## no 286 0
## yes 0 510
## Accuracy : 1.0000
## Precision : 1.0000
## Sensitivity : 1.0000
## Specificity : 1.0000
## F1-Score : 1.0000
## AUC : 1.0000
prob_nn4_test <- predict(model_nn4, test_smoted_sc, type = "prob")[, "yes"]
pred_nn4_test <- predict(model_nn4, test_smoted_sc)
res_nn4_test <- evaluasi_model(test_smoted_sc$diagnosis, pred_nn4_test, prob_nn4_test,
"NN - S4", "Testing")
##
## [NN - S4 - Testing]
## Reference
## Prediction no yes
## no 68 2
## yes 3 40
## Accuracy : 0.9558
## Precision : 0.9302
## Sensitivity : 0.9524
## Specificity : 0.9577
## F1-Score : 0.9412
## AUC : 0.9936
hasil_semua <- rbind(hasil_semua, res_nn4_train, res_nn4_test)
# ----- 9.7 RANDOM FOREST + SMOTE + Tuning -----
cat("\n--- RANDOM FOREST + SMOTE + Tuning ---\n")
##
## --- RANDOM FOREST + SMOTE + Tuning ---
ctrl_rf4 <- trainControl(method = "cv", number = 5,
classProbs = TRUE, summaryFunction = twoClassSummary)
grid_rf4 <- expand.grid(mtry = c(2, 3, 4, 5, 6, 8))
set.seed(123)
model_rf4 <- train(diagnosis ~ ., data = train_smoted, method = "rf",
trControl = ctrl_rf4, tuneGrid = grid_rf4,
metric = "ROC", ntree = 150)
cat("mtry optimal RF+SMOTE:", model_rf4$bestTune$mtry, "\n")
## mtry optimal RF+SMOTE: 2
cat("CV ROC (resampled):", mean(model_rf4$resample$ROC, na.rm = TRUE), "\n")
## CV ROC (resampled): 0.9945589
prob_rf4_train <- predict(model_rf4, train_smoted, type = "prob")[, "yes"]
pred_rf4_train <- predict(model_rf4, train_smoted)
res_rf4_train <- evaluasi_model(train_smoted$diagnosis, pred_rf4_train, prob_rf4_train,
"RF - S4", "Training")
##
## [RF - S4 - Training]
## Reference
## Prediction no yes
## no 286 0
## yes 0 510
## Accuracy : 1.0000
## Precision : 1.0000
## Sensitivity : 1.0000
## Specificity : 1.0000
## F1-Score : 1.0000
## AUC : 1.0000
prob_rf4_test <- predict(model_rf4, test_smoted, type = "prob")[, "yes"]
pred_rf4_test <- predict(model_rf4, test_smoted)
res_rf4_test <- evaluasi_model(test_smoted$diagnosis, pred_rf4_test, prob_rf4_test,
"RF - S4", "Testing")
##
## [RF - S4 - Testing]
## Reference
## Prediction no yes
## no 67 0
## yes 4 42
## Accuracy : 0.9646
## Precision : 0.9130
## Sensitivity : 1.0000
## Specificity : 0.9437
## F1-Score : 0.9545
## AUC : 0.9975
hasil_semua <- rbind(hasil_semua, res_rf4_train, res_rf4_test)
# ----- 9.8 GBM + SMOTE + Tuning -----
cat("\n--- GRADIENT BOOSTING + SMOTE + Tuning ---\n")
##
## --- GRADIENT BOOSTING + SMOTE + Tuning ---
ctrl_gbm4 <- trainControl(method = "cv", number = 5,
classProbs = TRUE, summaryFunction = twoClassSummary)
grid_gbm4 <- expand.grid(
n.trees = c(100, 200),
interaction.depth = c(3, 5),
shrinkage = c(0.05, 0.1),
n.minobsinnode = c(5, 10)
)
set.seed(123)
model_gbm4 <- train(diagnosis ~ ., data = train_smoted, method = "gbm",
trControl = ctrl_gbm4, tuneGrid = grid_gbm4,
metric = "ROC", verbose = FALSE)
cat("Parameter optimal GBM+SMOTE:\n")
## Parameter optimal GBM+SMOTE:
print(model_gbm4$bestTune)
## n.trees interaction.depth shrinkage n.minobsinnode
## 11 100 3 0.1 10
prob_gbm4_train <- predict(model_gbm4, train_smoted, type = "prob")[, "yes"]
pred_gbm4_train <- predict(model_gbm4, train_smoted)
res_gbm4_train <- evaluasi_model(train_smoted$diagnosis, pred_gbm4_train, prob_gbm4_train,
"GBM - S4", "Training")
##
## [GBM - S4 - Training]
## Reference
## Prediction no yes
## no 282 2
## yes 4 508
## Accuracy : 0.9925
## Precision : 0.9922
## Sensitivity : 0.9961
## Specificity : 0.9860
## F1-Score : 0.9941
## AUC : 0.9998
prob_gbm4_test <- predict(model_gbm4, test_smoted, type = "prob")[, "yes"]
pred_gbm4_test <- predict(model_gbm4, test_smoted)
res_gbm4_test <- evaluasi_model(test_smoted$diagnosis, pred_gbm4_test, prob_gbm4_test,
"GBM - S4", "Testing")
##
## [GBM - S4 - Testing]
## Reference
## Prediction no yes
## no 66 0
## yes 5 42
## Accuracy : 0.9558
## Precision : 0.8936
## Sensitivity : 1.0000
## Specificity : 0.9296
## F1-Score : 0.9438
## AUC : 0.9970
hasil_semua <- rbind(hasil_semua, res_gbm4_train, res_gbm4_test)
# --- Ringkasan Skenario 4 ---
cat("\n=== RINGKASAN SKENARIO 4 ===\n")
##
## === RINGKASAN SKENARIO 4 ===
print(hasil_semua[grepl("S4", hasil_semua$Model), ])
## Model Data Accuracy Precision Sensitivity Specificity F1_Score AUC
## 49 LR - S4 Training 0.9711 0.9620 0.9941 0.9301 0.9778 0.9970
## 50 LR - S4 Testing 0.9027 0.7925 1.0000 0.8451 0.8842 0.9943
## 51 DT - S4 Training 0.9661 0.9653 0.9824 0.9371 0.9738 0.9654
## 52 DT - S4 Testing 0.8938 0.7885 0.9762 0.8451 0.8723 0.9116
## 53 NB - S4 Training 0.9271 0.9728 0.9118 0.9545 0.9413 0.9860
## 54 NB - S4 Testing 0.9381 0.8889 0.9524 0.9296 0.9195 0.9842
## 55 KNN - S4 Training 0.9736 0.9657 0.9941 0.9371 0.9797 0.9984
## 56 KNN - S4 Testing 0.9381 0.8571 1.0000 0.9014 0.9231 0.9836
## 57 SVM - S4 Training 0.9812 0.9825 0.9882 0.9685 0.9853 0.9970
## 58 SVM - S4 Testing 0.9735 0.9535 0.9762 0.9718 0.9647 0.9960
## 59 NN - S4 Training 1.0000 1.0000 1.0000 1.0000 1.0000 1.0000
## 60 NN - S4 Testing 0.9558 0.9302 0.9524 0.9577 0.9412 0.9936
## 61 RF - S4 Training 1.0000 1.0000 1.0000 1.0000 1.0000 1.0000
## 62 RF - S4 Testing 0.9646 0.9130 1.0000 0.9437 0.9545 0.9975
## 63 GBM - S4 Training 0.9925 0.9922 0.9961 0.9860 0.9941 0.9998
## 64 GBM - S4 Testing 0.9558 0.8936 1.0000 0.9296 0.9438 0.9970
Perbandingan Semua Model
# ============================================================
# 10. PERBANDINGAN SEMUA MODEL DAN PENENTUAN MODEL TERBAIK
# ============================================================
cat("\n\n")
cat("============================================================\n")
## ============================================================
cat(" PERBANDINGAN SELURUH MODEL - SEMUA SKENARIO\n")
## PERBANDINGAN SELURUH MODEL - SEMUA SKENARIO
cat("============================================================\n")
## ============================================================
# Filter hasil testing saja, urutkan berdasarkan AUC
hasil_testing <- hasil_semua[hasil_semua$Data == "Testing", ]
hasil_testing <- hasil_testing[order(-hasil_testing$AUC, -hasil_testing$F1_Score), ]
rownames(hasil_testing) <- NULL
cat("\n=== TABEL PERBANDINGAN SEMUA MODEL (Testing) ===\n")
##
## === TABEL PERBANDINGAN SEMUA MODEL (Testing) ===
print(hasil_testing)
## Model Data Accuracy Precision Sensitivity Specificity F1_Score AUC
## 1 RF - S3 Testing 0.9646 0.9130 1.0000 0.9437 0.9545 0.9977
## 2 GBM - S2 Testing 0.9558 0.9111 0.9762 0.9437 0.9425 0.9977
## 3 RF - S4 Testing 0.9646 0.9130 1.0000 0.9437 0.9545 0.9975
## 4 SVM - S3 Testing 0.9027 0.7925 1.0000 0.8451 0.8842 0.9973
## 5 SVM - S2 Testing 0.9823 0.9762 0.9762 0.9859 0.9762 0.9970
## 6 GBM - S4 Testing 0.9558 0.8936 1.0000 0.9296 0.9438 0.9970
## 7 SVM - S4 Testing 0.9735 0.9535 0.9762 0.9718 0.9647 0.9960
## 8 SVM - S1 Testing 0.9646 0.9318 0.9762 0.9577 0.9535 0.9960
## 9 NN - S2 Testing 0.9558 0.9111 0.9762 0.9437 0.9425 0.9960
## 10 RF - S1 Testing 0.9646 0.9318 0.9762 0.9577 0.9535 0.9956
## 11 GBM - S3 Testing 0.9292 0.8400 1.0000 0.8873 0.9130 0.9953
## 12 RF - S2 Testing 0.9558 0.9302 0.9524 0.9577 0.9412 0.9948
## 13 KNN - S2 Testing 0.9646 0.9524 0.9524 0.9718 0.9524 0.9946
## 14 LR - S3 Testing 0.9381 0.8571 1.0000 0.9014 0.9231 0.9943
## 15 GBM - S1 Testing 0.9381 0.8889 0.9524 0.9296 0.9195 0.9943
## 16 LR - S4 Testing 0.9027 0.7925 1.0000 0.8451 0.8842 0.9943
## 17 LR - S2 Testing 0.9469 0.8913 0.9762 0.9296 0.9318 0.9940
## 18 LR - S1 Testing 0.9292 0.8542 0.9762 0.9014 0.9111 0.9940
## 19 NN - S4 Testing 0.9558 0.9302 0.9524 0.9577 0.9412 0.9936
## 20 KNN - S1 Testing 0.9469 0.9091 0.9524 0.9437 0.9302 0.9935
## 21 NN - S3 Testing 0.9646 0.9318 0.9762 0.9577 0.9535 0.9930
## 22 NN - S1 Testing 0.9558 0.9302 0.9524 0.9577 0.9412 0.9906
## 23 KNN - S3 Testing 0.9027 0.7925 1.0000 0.8451 0.8842 0.9898
## 24 NB - S1 Testing 0.9469 0.9091 0.9524 0.9437 0.9302 0.9893
## 25 NB - S2 Testing 0.9469 0.9091 0.9524 0.9437 0.9302 0.9893
## 26 NB - S3 Testing 0.9381 0.8889 0.9524 0.9296 0.9195 0.9842
## 27 NB - S4 Testing 0.9381 0.8889 0.9524 0.9296 0.9195 0.9842
## 28 KNN - S4 Testing 0.9381 0.8571 1.0000 0.9014 0.9231 0.9836
## 29 DT - S2 Testing 0.9292 0.8696 0.9524 0.9155 0.9091 0.9832
## 30 DT - S1 Testing 0.9027 0.8163 0.9524 0.8732 0.8791 0.9802
## 31 DT - S3 Testing 0.8938 0.7885 0.9762 0.8451 0.8723 0.9116
## 32 DT - S4 Testing 0.8938 0.7885 0.9762 0.8451 0.8723 0.9116
# Model terbaik berdasarkan AUC testing
model_terbaik <- hasil_testing[1, ]
cat("\n=== MODEL TERBAIK ===\n")
##
## === MODEL TERBAIK ===
cat("Model :", model_terbaik$Model, "\n")
## Model : RF - S3
cat("Accuracy :", model_terbaik$Accuracy, "\n")
## Accuracy : 0.9646
cat("Precision :", model_terbaik$Precision, "\n")
## Precision : 0.913
cat("Sensitivity :", model_terbaik$Sensitivity, "\n")
## Sensitivity : 1
cat("Specificity :", model_terbaik$Specificity, "\n")
## Specificity : 0.9437
cat("F1-Score :", model_terbaik$F1_Score, "\n")
## F1-Score : 0.9545
cat("AUC :", model_terbaik$AUC, "\n")
## AUC : 0.9977
Visualisasi Perbandingan dan Model Terbaik
# ============================================================
# 11. VISUALISASI PERBANDINGAN
# ============================================================
# Tambahkan kolom Skenario dan Algoritma
hasil_testing$Skenario <- sub(".*(S[1-4]).*", "\\1", hasil_testing$Model)
hasil_testing$Algoritma <- sub(" - S[1-4]", "", hasil_testing$Model)
# --- 11.1 Plot F1-Score semua model (Testing) ---
ggplot(hasil_testing, aes(x = reorder(Model, F1_Score), y = F1_Score, fill = Skenario)) +
geom_col(show.legend = TRUE) +
coord_flip() +
labs(title = "Perbandingan F1-Score Semua Model (Testing)",
x = "Model", y = "F1-Score") +
theme_minimal() +
scale_fill_brewer(palette = "Set2")
# --- 11.2 Plot AUC semua model (Testing) ---
ggplot(hasil_testing[!is.na(hasil_testing$AUC), ],
aes(x = reorder(Model, AUC), y = AUC, fill = Skenario)) +
geom_col() +
coord_flip() +
geom_hline(yintercept = 0.9, linetype = "dashed", color = "red", linewidth = 0.8) +
labs(title = "Perbandingan AUC Semua Model (Testing)",
x = "Model", y = "AUC") +
theme_minimal() +
scale_fill_brewer(palette = "Set1")
# --- 11.3 Perbandingan semua metrik (heatmap) ---
hasil_testing_long <- hasil_testing %>%
select(Model, Accuracy, Precision, Sensitivity, Specificity, F1_Score) %>%
pivot_longer(cols = -Model, names_to = "Metrik", values_to = "Nilai")
ggplot(hasil_testing_long, aes(x = Metrik, y = Model, fill = Nilai)) +
geom_tile(color = "white") +
geom_text(aes(label = round(Nilai, 2)), size = 2.8) +
scale_fill_gradient2(low = "white", high = "steelblue", midpoint = 0.7) +
labs(title = "Heatmap Performa Semua Model (Testing)",
x = "Metrik Evaluasi", y = "Model") +
theme_minimal() +
theme(axis.text.y = element_text(size = 7))
# --- 11.4 Pengaruh SMOTE: S1 vs S3 (tanpa tuning) ---
smote_compare <- hasil_testing[grepl("S1|S3", hasil_testing$Skenario), ]
cat("\n=== PENGARUH SMOTE (S1 vs S3) ===\n")
##
## === PENGARUH SMOTE (S1 vs S3) ===
print(smote_compare[, c("Model", "Accuracy", "Sensitivity", "F1_Score", "AUC")])
## Model Accuracy Sensitivity F1_Score AUC
## 1 RF - S3 0.9646 1.0000 0.9545 0.9977
## 4 SVM - S3 0.9027 1.0000 0.8842 0.9973
## 8 SVM - S1 0.9646 0.9762 0.9535 0.9960
## 10 RF - S1 0.9646 0.9762 0.9535 0.9956
## 11 GBM - S3 0.9292 1.0000 0.9130 0.9953
## 14 LR - S3 0.9381 1.0000 0.9231 0.9943
## 15 GBM - S1 0.9381 0.9524 0.9195 0.9943
## 18 LR - S1 0.9292 0.9762 0.9111 0.9940
## 20 KNN - S1 0.9469 0.9524 0.9302 0.9935
## 21 NN - S3 0.9646 0.9762 0.9535 0.9930
## 22 NN - S1 0.9558 0.9524 0.9412 0.9906
## 23 KNN - S3 0.9027 1.0000 0.8842 0.9898
## 24 NB - S1 0.9469 0.9524 0.9302 0.9893
## 26 NB - S3 0.9381 0.9524 0.9195 0.9842
## 30 DT - S1 0.9027 0.9524 0.8791 0.9802
## 31 DT - S3 0.8938 0.9762 0.8723 0.9116
# --- 11.5 Pengaruh Tuning: S1 vs S2, S3 vs S4 ---
cat("\n=== PENGARUH TUNING ===\n")
##
## === PENGARUH TUNING ===
print(hasil_testing[, c("Model", "Accuracy", "F1_Score", "AUC")])
## Model Accuracy F1_Score AUC
## 1 RF - S3 0.9646 0.9545 0.9977
## 2 GBM - S2 0.9558 0.9425 0.9977
## 3 RF - S4 0.9646 0.9545 0.9975
## 4 SVM - S3 0.9027 0.8842 0.9973
## 5 SVM - S2 0.9823 0.9762 0.9970
## 6 GBM - S4 0.9558 0.9438 0.9970
## 7 SVM - S4 0.9735 0.9647 0.9960
## 8 SVM - S1 0.9646 0.9535 0.9960
## 9 NN - S2 0.9558 0.9425 0.9960
## 10 RF - S1 0.9646 0.9535 0.9956
## 11 GBM - S3 0.9292 0.9130 0.9953
## 12 RF - S2 0.9558 0.9412 0.9948
## 13 KNN - S2 0.9646 0.9524 0.9946
## 14 LR - S3 0.9381 0.9231 0.9943
## 15 GBM - S1 0.9381 0.9195 0.9943
## 16 LR - S4 0.9027 0.8842 0.9943
## 17 LR - S2 0.9469 0.9318 0.9940
## 18 LR - S1 0.9292 0.9111 0.9940
## 19 NN - S4 0.9558 0.9412 0.9936
## 20 KNN - S1 0.9469 0.9302 0.9935
## 21 NN - S3 0.9646 0.9535 0.9930
## 22 NN - S1 0.9558 0.9412 0.9906
## 23 KNN - S3 0.9027 0.8842 0.9898
## 24 NB - S1 0.9469 0.9302 0.9893
## 25 NB - S2 0.9469 0.9302 0.9893
## 26 NB - S3 0.9381 0.9195 0.9842
## 27 NB - S4 0.9381 0.9195 0.9842
## 28 KNN - S4 0.9381 0.9231 0.9836
## 29 DT - S2 0.9292 0.9091 0.9832
## 30 DT - S1 0.9027 0.8791 0.9802
## 31 DT - S3 0.8938 0.8723 0.9116
## 32 DT - S4 0.8938 0.8723 0.9116
# --- 11.6 Ensemble vs Non-Ensemble ---
hasil_testing$Kategori <- ifelse(
grepl("^(LR|DT|NB|KNN|SVM|NN)", hasil_testing$Algoritma),
"Non-Ensemble", "Ensemble"
)
cat("\n=== RATA-RATA PERFORMA: ENSEMBLE vs NON-ENSEMBLE ===\n")
##
## === RATA-RATA PERFORMA: ENSEMBLE vs NON-ENSEMBLE ===
performa_kat <- hasil_testing %>%
group_by(Kategori) %>%
summarise(
Avg_Accuracy = mean(Accuracy, na.rm = TRUE),
Avg_F1 = mean(F1_Score, na.rm = TRUE),
Avg_AUC = mean(AUC, na.rm = TRUE),
Avg_Sensitivity = mean(Sensitivity, na.rm = TRUE),
.groups = "drop"
)
print(performa_kat)
## # A tibble: 2 × 5
## Kategori Avg_Accuracy Avg_F1 Avg_AUC Avg_Sensitivity
## <chr> <dbl> <dbl> <dbl> <dbl>
## 1 Ensemble 0.954 0.940 0.996 0.982
## 2 Non-Ensemble 0.938 0.922 0.985 0.971
# --- 11.7 Cek Overfitting (Training vs Testing) ---
hasil_semua$Skenario <- sub(".*(S[1-4]).*", "\\1", hasil_semua$Model)
hasil_semua$Algoritma <- sub(" - S[1-4]", "", hasil_semua$Model)
overfitting_check <- hasil_semua %>%
select(Model, Data, Accuracy, F1_Score) %>%
pivot_wider(names_from = Data, values_from = c(Accuracy, F1_Score))
cat("\n=== CEK OVERFITTING (Training vs Testing) ===\n")
##
## === CEK OVERFITTING (Training vs Testing) ===
print(overfitting_check)
## # A tibble: 32 × 5
## Model Accuracy_Training Accuracy_Testing F1_Score_Training F1_Score_Testing
## <chr> <dbl> <dbl> <dbl> <dbl>
## 1 LR - S1 0.963 0.929 0.950 0.911
## 2 DT - S1 0.956 0.903 0.942 0.879
## 3 NB - S1 0.932 0.947 0.908 0.930
## 4 KNN - … 0.954 0.947 0.938 0.930
## 5 SVM - … 0.967 0.965 0.956 0.954
## 6 NN - S1 1 0.956 1 0.941
## 7 RF - S1 0.934 0.965 0.912 0.954
## 8 GBM - … 0.998 0.938 0.997 0.920
## 9 LR - S2 0.967 0.947 0.955 0.932
## 10 DT - S2 0.950 0.929 0.933 0.909
## # ℹ 22 more rows
# --- 11.8 Export hasil ke CSV ---
write.csv(hasil_semua, "hasil_semua_model.csv", row.names = FALSE)
write.csv(hasil_testing, "hasil_testing_model.csv", row.names = FALSE)
cat("\nHasil evaluasi disimpan ke: hasil_semua_model.csv & hasil_testing_model.csv\n")
##
## Hasil evaluasi disimpan ke: hasil_semua_model.csv & hasil_testing_model.csv
# ============================================================
# 12. INTERPRETASI MODEL TERBAIK
# ============================================================
cat("\n\n")
cat("============================================================\n")
## ============================================================
cat(" INTERPRETASI MODEL TERBAIK\n")
## INTERPRETASI MODEL TERBAIK
cat("============================================================\n")
## ============================================================
cat("Model terbaik:", model_terbaik$Model, "\n\n")
## Model terbaik: RF - S3
# Catatan: sesuaikan bagian ini setelah menjalankan syntax di atas
# dan melihat model mana yang menjadi terbaik.
# Contoh interpretasi jika model terbaik adalah GBM atau RF:
# Jika model terbaik adalah GBM (Skenario 2):
# print(summary(model_gbm2)) # relative influence tiap fitur
# Jika model terbaik adalah RF (Skenario 4 + SMOTE):
# varImpPlot(model_rf4$finalModel, main = "Variable Importance - Model Terbaik")
# importance(model_rf4$finalModel)
# Partial Dependence Plot untuk fitur terpenting (contoh):
# Ganti "model_gbm2" dan nama fitur sesuai model terbaik
# plot(model_gbm2, i.var = 1, n.trees = 200, main = "PDP - Fitur Terpenting")
# ============================================================
# SELESAI
# ============================================================
cat("\n=== SYNTAX SELESAI DIJALANKAN ===\n")
##
## === SYNTAX SELESAI DIJALANKAN ===
cat("Semua skenario (S1-S4) telah selesai.\n")
## Semua skenario (S1-S4) telah selesai.
cat("Model terbaik:", model_terbaik$Model, "\n")
## Model terbaik: RF - S3
cat("AUC Testing :", model_terbaik$AUC, "\n")
## AUC Testing : 0.9977
cat("F1-Score Test:", model_terbaik$F1_Score, "\n")
## F1-Score Test: 0.9545