Homework 8 - Classification Analysis
your name
2026-07-25
Set up
knitr::opts_chunk$set(echo = TRUE,
fig.align = "center")
# load the following packages: tidyverse, class, caret, rpart, rpart.plot
pacman::p_load(tidyverse, class, caret, skimr, rpart, rpart.plot)
# Vector of wanted types for the homework
wanted_types <- c('Fighting', 'Rock', 'Flying')
# Colors to use for graphs
poke_colors <- c('Fighting' = '#C22E28', 'Flying' = '#A98FF3', 'Rock' = '#B6A136')
# Reading in the data set
poke3 <-
read.csv('https://raw.githubusercontent.com/Shammalamala/DS-2870-Data-Sets/refs/heads/main/pokedex.csv') |>
# Keeping only types that are fighting, psychic, or dark
filter(
type1 %in% wanted_types |
type2 %in% wanted_types,
!(type1 %in% wanted_types & type2 %in% wanted_types)
) |>
# Matching type to the wanted type if they belong to type 1 or type 2
mutate(
type = case_when(
type1 == wanted_types[1] | type2 == wanted_types[1] ~ wanted_types[1],
type1 == wanted_types[2] | type2 == wanted_types[2] ~ wanted_types[2],
type1 == wanted_types[3] | type2 == wanted_types[3] ~ wanted_types[3],
) |> factor()
) |>
column_to_rownames(var = 'name') |>
# Only keeping relevant columns
dplyr::select(type, hp:speed) Data Description
## # A tibble: 173 × 7
## type hp attack defense sp_atk sp_def speed
## <fct> <int> <int> <int> <int> <int> <int>
## 1 Flying 78 84 78 109 85 100
## 2 Flying 60 45 50 90 80 70
## 3 Flying 40 45 40 35 35 56
## 4 Flying 63 60 55 50 50 71
## 5 Flying 83 80 75 70 70 101
## 6 Flying 40 60 30 31 31 70
## 7 Flying 65 90 65 61 61 100
## 8 Flying 40 45 35 30 40 55
## 9 Flying 75 80 70 65 75 90
## 10 Fighting 40 80 35 35 45 70
## # ℹ 163 more rowsThe poke3 data set has 7 variables for 173 pokemon:
Which of three types (Fighting/Flying/Ground) and six different combat
attributes (default values on a scale from 0 to 200). In this homework
assignment, you’ll attempt to predict which of the three types a pokemon
is based on the 6 different attributes
Question 1) Exporatory Data Analysis
Part 1a) Graph of attributes across types
Create the set of boxplots seen in Brightspace. Use the
poke_colors vector to match the color to the types
poke3 |>
pivot_longer(
cols = hp:speed,
names_to = 'attribute',
values_to = 'score'
) |>
ggplot(
mapping = aes(
x = score,
y = type,
fill = type
)
) +
geom_boxplot(show.legend = F) +
facet_wrap(
facets = vars(attribute),
nrow = 3
#scales = 'free_x'
) +
scale_fill_manual(values = poke_colors) +
theme_bw() +
labs(x = NULL,
y = NULL,
title = 'How to the attributes differ across pokemon types?')
Part 1b) How type differ across the attributes
Which attribute(s) appear to be useful when predicting which of the three types the pokemon is?
Which attribute(s) appear to not be useful when predicting which of the three types the pokemon is?
Question 2) kNN
For this question, you’ll be using all six attributes, regardless of your answer from part 1b.
Part 2a) Standardize and Normalize the data
Create a data set named poke_stan and
poke_norm for the standardized and normalized attributes,
respectively
# create normalization function
normalize <- function(x){return((x - min(x))/(x |> range() |> diff() |> abs()))}
# Standardized
poke_stan <-
poke3 |>
mutate(
across(.cols = where(is.numeric),
.fns = ~ (. - mean(.))/sd(.))
)
# Normalized
poke_norm <-
poke3 |>
mutate(
across(.cols = where(is.numeric),
.fns = ~ (. - min(.))/(max(.) - min(.)))
)Use the code chunk below to check that the attributes have been standardized correctly
# confirm that standardization worked
poke_stan |>
pivot_longer(
cols = -type
) |>
summarize(
.by = name,
att_avg = round(mean(value), 4),
att_sd = sd(value)
)## # A tibble: 6 × 3
## name att_avg att_sd
## <chr> <dbl> <dbl>
## 1 hp 0 1
## 2 attack 0 1
## 3 defense 0 1
## 4 sp_atk 0 1
## 5 sp_def 0 1
## 6 speed 0 1Use the code chunk below to check that the attributes have been standardized correctly
# confirm that normalization worked
poke_norm |>
pivot_longer(
cols = -type
) |>
summarize(
.by = name,
att_min = min(value),
att_max = max(value)
)## # A tibble: 6 × 3
## name att_min att_max
## <chr> <dbl> <dbl>
## 1 hp 0 1
## 2 attack 0 1
## 3 defense 0 1
## 4 sp_atk 0 1
## 5 sp_def 0 1
## 6 speed 0 1Part 2b) Data frame to save the results
Create a data frame name k_search that has three
columns:
k: A column of values 2 to 150 by increments of 1Standardize: a column of -1Normalize: a column of -1
k_search <-
data.frame(
k = 2:150,
Normalize = -1,
Standardize = -1
)
# Displaying the data frame
tibble(k_search)## # A tibble: 149 × 3
## k Normalize Standardize
## <int> <dbl> <dbl>
## 1 2 -1 -1
## 2 3 -1 -1
## 3 4 -1 -1
## 4 5 -1 -1
## 5 6 -1 -1
## 6 7 -1 -1
## 7 8 -1 -1
## 8 9 -1 -1
## 9 10 -1 -1
## 10 11 -1 -1
## # ℹ 139 more rowsPart 2c) Finding best kNN “model”
Find the best choice of k to use for the kNN algorithm for both the normalized and standardized data (separately). Only use one loop, not two!
# Keep this at the top of the codechunk
RNGversion("4.1.0");set.seed(2870)
for (i in 1:nrow(k_search)){
# Accuracy for normalized data
knn_norm <-
knn.cv(
train = poke_norm |> dplyr::select(-type),
cl = poke_norm$type,
k = k_search$k[i]
)
k_search[i, 'Normalize'] <- mean(poke_norm$type == knn_norm)
# Accuracy for standardized data
knn_stan <-
knn.cv(
train = poke_stan |> dplyr::select(-type),
cl = poke_stan$type,
k = k_search$k[i]
)
k_search[i, 'Standardize'] <- mean(poke_stan$type == knn_stan)
}Use the code chunk below to check your results
k_search |>
slice(seq(2, 150, length.out = 10) |> floor())## k Normalize Standardize
## 1 3 0.6878613 0.7225434
## 2 19 0.7167630 0.7167630
## 3 35 0.6763006 0.6763006
## 4 52 0.6531792 0.6647399
## 5 68 0.6300578 0.6358382
## 6 85 0.6127168 0.6300578
## 7 101 0.5895954 0.6011561
## 8 118 0.5317919 0.5260116
## 9 134 0.4739884 0.4739884Part 2d) Create a line graph of the grid search
Create the line graph seen in Brightspace.
k_search |>
pivot_longer(
cols = -k,
names_to = 'type',
values_to = 'accuracy'
) |>
ggplot(
mapping = aes(
x = k,
y = accuracy,
color = type
)
) +
geom_line() +
theme_bw() +
labs(
color = NULL,
x = 'k',
y = 'Accuracy'
)
Does either normalization or standardization appear to be a better method overall?
No, the lines for normalization and standardization appear to be pretty similar across the graph
Part 2e) Best choice of k
What is the best choice of k and method of rescaling?
k_search |>
pivot_longer(
cols = -k,
names_to = 'rescale',
values_to = 'accuracy'
) |>
slice_max(accuracy, n = 1)## # A tibble: 1 × 3
## k rescale accuracy
## <int> <chr> <dbl>
## 1 23 Standardize 0.734k: 23
rescale: Standardize
accuracy: About 73.4%
Part 2f) Confusion Matrix
Create the confusion matrix from
caret::confusionMatrix() using your best choice of k and
rescale method from the previous question.
confusionMatrix(
table(
'predicted' =
knn.cv(
train = poke_stan |> dplyr::select(-type),
cl = poke_stan$type,
k = 23
),
'actual' = poke3$type
)
)## Confusion Matrix and Statistics
##
## actual
## predicted Fighting Flying Rock
## Fighting 14 2 4
## Flying 19 76 11
## Rock 7 4 36
##
## Overall Statistics
##
## Accuracy : 0.7283
## 95% CI : (0.6556, 0.7931)
## No Information Rate : 0.474
## P-Value [Acc > NIR] : 9.688e-12
##
## Kappa : 0.5493
##
## Mcnemar's Test P-Value : 0.000473
##
## Statistics by Class:
##
## Class: Fighting Class: Flying Class: Rock
## Sensitivity 0.35000 0.9268 0.7059
## Specificity 0.95489 0.6703 0.9098
## Pos Pred Value 0.70000 0.7170 0.7660
## Neg Pred Value 0.83007 0.9104 0.8810
## Prevalence 0.23121 0.4740 0.2948
## Detection Rate 0.08092 0.4393 0.2081
## Detection Prevalence 0.11561 0.6127 0.2717
## Balanced Accuracy 0.65244 0.7986 0.8079Which type does kNN have the hardest time predicting? What type is it most commonly predicted to be?
Fighting is predicted correctly the least. It is more likely to be predicted as a Flying type than a Fighting type
Part 2g) Important variables
Using the kNN results, can you determine which variables are the most important when determining if a pokemon is a Fighting, Flying, or Rock type? If you can, which variables are the most important?
No, kNN can’t tell you how the variables are used and which variables are the most or least important since it is a lazy learner.
Question 3) Classification Tree
Instead of using kNN to predict which of the three types, you’ll be using a classification tree.
3a) Grow the full tree
Using all the other columns of the data, grow a full classification
tree to predict type. Display the complexity parameter
table of the full tree rounded to 4 decimal places
# Leave this at the top
RNGversion('4.1.0'); set.seed(1870)
poke_tree_full <-
rpart(
formula = type ~ .,
data = poke3,
parms = list(split = 'information'),
cp = -1,
minbucket = 1,
minsplit = 2
)
# Plot the full class tree:
round(poke_tree_full$cptable, 4)## CP nsplit rel error xerror xstd
## 1 0.3297 0 1.0000 1.0000 0.0722
## 2 0.0714 1 0.6703 0.6703 0.0691
## 3 0.0440 3 0.5275 0.7143 0.0700
## 4 0.0330 5 0.4396 0.6374 0.0682
## 5 0.0220 6 0.4066 0.5824 0.0666
## 6 0.0165 7 0.3846 0.6044 0.0673
## 7 0.0147 11 0.3187 0.6264 0.0679
## 8 0.0110 14 0.2747 0.6264 0.0679
## 9 0.0082 27 0.1319 0.6484 0.0685
## 10 0.0073 31 0.0989 0.7363 0.0704
## 11 0.0055 35 0.0659 0.7363 0.0704
## 12 0.0037 45 0.0110 0.7582 0.0708
## 13 -1.0000 48 0.0000 0.7582 0.0708How many leaf nodes (terminal nodes) does the full tree have?
Since there are 48 splits, there are 49 leaf nodes
Part 3b) Find the cut off to prune the tree
Find the xerror and cp value to prune the full tree.
xcutoff <-
poke_tree_full$cptable |>
data.frame() |>
slice_min(xerror) |>
mutate(xcutoff = xerror + xstd) |>
slice(1) |>
pull(xcutoff)
# Getting the cp value
cp_prune <-
poke_tree_full$cptable |>
data.frame() |>
filter(xerror < xcutoff) |>
slice(1) |>
pull(CP)
c('xerror cut off' = xcutoff, 'cp' = cp_prune) |> round(3)## xerror cut off cp
## 0.649 0.033The cutoff for the xerror is 0.513, which occurs on row 5 with 7 splits for a total of 8 leaf nodes.
The cp value to use for the cutoff is any value between 0.00706 and 0.011
Part 3c) Creating the pruned tree
Prune the full tree from part a) and plot it. Include
box.col = poke_colors[poke_tree$frame$yval] in
rpart.plot() to have the colors match the types, where
poke_tree represents the saved pruned tree. If you name
your tree a different name, just replace poke_tree with
whatever you named your pruned tree as.
poke_pruned <-
prune(
tree = poke_tree_full,
cp = cp_prune
)
# Graphing the tree
rpart.plot(
x = poke_pruned,
type = 5,
# Changing the color scheme to match the type
box.col = poke_colors[poke_pruned$frame$yval],
extra = 101
)
Part 2d) Intepreting two leaf nodes
Interpret the leaf node third from the right and the right most leaf node
Third from the right:
If the pokemon has a speed above 59 but an attack less than 95, we predict it to be a flying type
Right most:
If a pokemon has a speed below 59 (slow) and a defense above 88 (high defense), we expect it to be a rock type
Part 3e) Confusion Matrix
Create the confusion matrix for the pruned tree. What is the model’s accuracy?
confusionMatrix(
data = predict(poke_pruned, type = 'class'),
reference = poke3$type
)## Confusion Matrix and Statistics
##
## Reference
## Prediction Fighting Flying Rock
## Fighting 26 5 15
## Flying 12 76 5
## Rock 2 1 31
##
## Overall Statistics
##
## Accuracy : 0.7688
## 95% CI : (0.6987, 0.8294)
## No Information Rate : 0.474
## P-Value [Acc > NIR] : 2.16e-15
##
## Kappa : 0.6305
##
## Mcnemar's Test P-Value : 0.001442
##
## Statistics by Class:
##
## Class: Fighting Class: Flying Class: Rock
## Sensitivity 0.6500 0.9268 0.6078
## Specificity 0.8496 0.8132 0.9754
## Pos Pred Value 0.5652 0.8172 0.9118
## Neg Pred Value 0.8898 0.9250 0.8561
## Prevalence 0.2312 0.4740 0.2948
## Detection Rate 0.1503 0.4393 0.1792
## Detection Prevalence 0.2659 0.5376 0.1965
## Balanced Accuracy 0.7498 0.8700 0.7916Part 3f) Variable Importance
Which three attributes are the most important when predicting the type?
caret::varImp(poke_pruned) |>
arrange(desc(Overall))## Overall
## defense 59.216989
## speed 56.374216
## attack 48.750457
## hp 17.403008
## sp_atk 14.221279
## sp_def 8.950039Question 4) The return of kNN
For question 4, you’ll be redoing question 2 (kNN), but only using the three attributes specified in question 3f. Because kNN uses every variable, if you include useless ones, it can reduce the accuracy of the predictions. So you’ll be trying again, this time only using the important features!
Part 4a) Search for the best choice of k
You should be able to copy and paste your code from parts 2b and 2c in the code chunk below, but alter it so that it only uses the top three important variables (attributes).
# Leave this at the top
RNGversion("4.1.0")
set.seed(2870)
# Creating the data frame to save the results
k_search <-
data.frame(
k = 2:150,
Normalize = -1,
Standardize = -1
)
# Performing the search
for (i in 1:nrow(k_search)){
# Accuracy for normalized data
knn_norm <-
knn.cv(
train = poke_norm |> dplyr::select(defense, speed, attack),
cl = poke_norm$type,
k = k_search$k[i]
)
k_search[i, 'Normalize'] <- mean(poke_norm$type == knn_norm)
# Accuracy for standardized data
knn_stan <-
knn.cv(
train = poke_stan |> dplyr::select(defense, speed, attack),
cl = poke_stan$type,
k = k_search$k[i]
)
k_search[i, 'Standardize'] <- mean(poke_stan$type == knn_stan)
}
k_search## k Normalize Standardize
## 1 2 0.6416185 0.6358382
## 2 3 0.7167630 0.7283237
## 3 4 0.7456647 0.7341040
## 4 5 0.7167630 0.7052023
## 5 6 0.6994220 0.7398844
## 6 7 0.7341040 0.7283237
## 7 8 0.7514451 0.7398844
## 8 9 0.7456647 0.7398844
## 9 10 0.7398844 0.7514451
## 10 11 0.7572254 0.7456647
## 11 12 0.7630058 0.7514451
## 12 13 0.7456647 0.7572254
## 13 14 0.7225434 0.7630058
## 14 15 0.7283237 0.7514451
## 15 16 0.7341040 0.7456647
## 16 17 0.7167630 0.7283237
## 17 18 0.7052023 0.7109827
## 18 19 0.7052023 0.7052023
## 19 20 0.7109827 0.7225434
## 20 21 0.6936416 0.7109827
## 21 22 0.7109827 0.7283237
## 22 23 0.7283237 0.7109827
## 23 24 0.7109827 0.7167630
## 24 25 0.6994220 0.7052023
## 25 26 0.7052023 0.7052023
## 26 27 0.6994220 0.7109827
## 27 28 0.6936416 0.7167630
## 28 29 0.7109827 0.7109827
## 29 30 0.7052023 0.6994220
## 30 31 0.7109827 0.6994220
## 31 32 0.7109827 0.6878613
## 32 33 0.6936416 0.6878613
## 33 34 0.7167630 0.6878613
## 34 35 0.7052023 0.6820809
## 35 36 0.6994220 0.6763006
## 36 37 0.6936416 0.6878613
## 37 38 0.6820809 0.6705202
## 38 39 0.6647399 0.6647399
## 39 40 0.6647399 0.6994220
## 40 41 0.6705202 0.6994220
## 41 42 0.6820809 0.6936416
## 42 43 0.6994220 0.6878613
## 43 44 0.6878613 0.6994220
## 44 45 0.6994220 0.6936416
## 45 46 0.6994220 0.6936416
## 46 47 0.7052023 0.6936416
## 47 48 0.7167630 0.6705202
## 48 49 0.7283237 0.6763006
## 49 50 0.7052023 0.6763006
## 50 51 0.6705202 0.6878613
## 51 52 0.6820809 0.6936416
## 52 53 0.6820809 0.7052023
## 53 54 0.6878613 0.6994220
## 54 55 0.6878613 0.6936416
## 55 56 0.6820809 0.7052023
## 56 57 0.6936416 0.7052023
## 57 58 0.7109827 0.6936416
## 58 59 0.6936416 0.6820809
## 59 60 0.6820809 0.6763006
## 60 61 0.6763006 0.6820809
## 61 62 0.6647399 0.6820809
## 62 63 0.6647399 0.6763006
## 63 64 0.6705202 0.6763006
## 64 65 0.6820809 0.6763006
## 65 66 0.6589595 0.6705202
## 66 67 0.6647399 0.6705202
## 67 68 0.6763006 0.6763006
## 68 69 0.6705202 0.6878613
## 69 70 0.6705202 0.6878613
## 70 71 0.6763006 0.6763006
## 71 72 0.6763006 0.6878613
## 72 73 0.6763006 0.6878613
## 73 74 0.6705202 0.6878613
## 74 75 0.6705202 0.6878613
## 75 76 0.6705202 0.6763006
## 76 77 0.6705202 0.6820809
## 77 78 0.6705202 0.6763006
## 78 79 0.6763006 0.6763006
## 79 80 0.6705202 0.6936416
## 80 81 0.6705202 0.6763006
## 81 82 0.6705202 0.6705202
## 82 83 0.6589595 0.6647399
## 83 84 0.6705202 0.6647399
## 84 85 0.6705202 0.6705202
## 85 86 0.6589595 0.6705202
## 86 87 0.6589595 0.6647399
## 87 88 0.6589595 0.6589595
## 88 89 0.6589595 0.6647399
## 89 90 0.6647399 0.6705202
## 90 91 0.6473988 0.6647399
## 91 92 0.6473988 0.6589595
## 92 93 0.6358382 0.6589595
## 93 94 0.6358382 0.6531792
## 94 95 0.6300578 0.6473988
## 95 96 0.6242775 0.6416185
## 96 97 0.6300578 0.6416185
## 97 98 0.6242775 0.6358382
## 98 99 0.6300578 0.6416185
## 99 100 0.6242775 0.6358382
## 100 101 0.6184971 0.6358382
## 101 102 0.6184971 0.6300578
## 102 103 0.6184971 0.6358382
## 103 104 0.6184971 0.6300578
## 104 105 0.6127168 0.6300578
## 105 106 0.6127168 0.6300578
## 106 107 0.6069364 0.6300578
## 107 108 0.6011561 0.6300578
## 108 109 0.6011561 0.6300578
## 109 110 0.5895954 0.6300578
## 110 111 0.5895954 0.6300578
## 111 112 0.5953757 0.6242775
## 112 113 0.5953757 0.6184971
## 113 114 0.5895954 0.6184971
## 114 115 0.5838150 0.6184971
## 115 116 0.5838150 0.6184971
## 116 117 0.5780347 0.6069364
## 117 118 0.5780347 0.6011561
## 118 119 0.5780347 0.5838150
## 119 120 0.5838150 0.5895954
## 120 121 0.5780347 0.5895954
## 121 122 0.5722543 0.5838150
## 122 123 0.5780347 0.5722543
## 123 124 0.5722543 0.5722543
## 124 125 0.5664740 0.5664740
## 125 126 0.5549133 0.5664740
## 126 127 0.5606936 0.5606936
## 127 128 0.5433526 0.5491329
## 128 129 0.5375723 0.5375723
## 129 130 0.5375723 0.5375723
## 130 131 0.5317919 0.5317919
## 131 132 0.5144509 0.5317919
## 132 133 0.5144509 0.5260116
## 133 134 0.5028902 0.5202312
## 134 135 0.4913295 0.4855491
## 135 136 0.4797688 0.4797688
## 136 137 0.4739884 0.4739884
## 137 138 0.4739884 0.4739884
## 138 139 0.4739884 0.4739884
## 139 140 0.4739884 0.4739884
## 140 141 0.4739884 0.4739884
## 141 142 0.4739884 0.4739884
## 142 143 0.4739884 0.4739884
## 143 144 0.4739884 0.4739884
## 144 145 0.4739884 0.4739884
## 145 146 0.4739884 0.4739884
## 146 147 0.4739884 0.4739884
## 147 148 0.4739884 0.4739884
## 148 149 0.4739884 0.4739884
## 149 150 0.4739884 0.4739884Part 4b) Grid search graph
Create the same graph you created in part 2c, but now using the new results of the grid search in 4a. Again, if you got the graph to work in part 2c, you can copy/paste the code and it should work here as well!
k_search |>
pivot_longer(
cols = -k,
names_to = 'type',
values_to = 'accuracy'
) |>
ggplot(
mapping = aes(
x = k,
y = accuracy,
color = type
)
) +
geom_line() +
theme_bw() +
labs(
color = NULL,
x = 'k',
y = 'Accuracy'
)
k_search |>
pivot_longer(
cols = -k,
names_to = 'type',
values_to = 'accuracy'
) |>
slice_max(n = 1, order_by = accuracy)## # A tibble: 2 × 3
## k type accuracy
## <int> <chr> <dbl>
## 1 12 Normalize 0.763
## 2 14 Standardize 0.763Part 4c) Confusion matrix for best choice of k
Create the confusion matrix of the predictions made using the Normalized data and \(k = 12\)
confusionMatrix(
data = knn.cv(
train = poke_norm |> dplyr::select(defense, speed, attack),
cl = poke_norm$type,
k = 12
),
reference = poke3$type
)## Confusion Matrix and Statistics
##
## Reference
## Prediction Fighting Flying Rock
## Fighting 20 6 8
## Flying 14 72 6
## Rock 6 4 37
##
## Overall Statistics
##
## Accuracy : 0.7457
## 95% CI : (0.674, 0.8087)
## No Information Rate : 0.474
## P-Value [Acc > NIR] : 3.158e-13
##
## Kappa : 0.5914
##
## Mcnemar's Test P-Value : 0.2741
##
## Statistics by Class:
##
## Class: Fighting Class: Flying Class: Rock
## Sensitivity 0.5000 0.8780 0.7255
## Specificity 0.8947 0.7802 0.9180
## Pos Pred Value 0.5882 0.7826 0.7872
## Neg Pred Value 0.8561 0.8765 0.8889
## Prevalence 0.2312 0.4740 0.2948
## Detection Rate 0.1156 0.4162 0.2139
## Detection Prevalence 0.1965 0.5318 0.2717
## Balanced Accuracy 0.6974 0.8291 0.8218How do the results compare to the results in part 2d? Which type had the largest improvement in being predicted correctly?
The results only using the three best attributes is more accurate (about 75.7% accurate) than using all six attributed to make the predictions (about 72.8% accurate). The largest improvement is in predicting Fighting types. Now the majority of Fighting types are predicted correctly!