import pandas as pd
import matplotlib.pyplot as plt
from pandas.plotting import scatter_matrix
penguins = pd.read_csv("penguins.csv")
features = [
"bill_length_mm",
"bill_depth_mm",
"flipper_length_mm",
"body_mass_g"
]
data = penguins[
features + ["species"]
].dropna()
print("Number of observations =", len(data))
print("\nSpecies distribution:")
print(data["species"].value_counts())
# ---------------------------------------------------
# Species-specific colors
# ---------------------------------------------------
species_names = data["species"].unique()
colors = plt.rcParams[
"axes.prop_cycle"
].by_key()["color"]
color_map = {
sp: colors[i]
for i, sp in enumerate(species_names)
}
point_colors = data["species"].map(color_map)
# ---------------------------------------------------
# Cross plot
# ---------------------------------------------------
axes = scatter_matrix(
data[features],
figsize=(12, 12),
diagonal="hist",
color=point_colors,
alpha=0.65,
s=25
)
# Legend
handles = [
plt.Line2D(
[0], [0],
marker="o",
linestyle="",
color=color_map[sp],
label=sp
)
for sp in species_names
]
plt.gcf().legend(
handles=handles,
loc="upper center",
ncol=3
)
plt.suptitle(
"Palmer Penguins: Numerical Features by Species",
fontsize=16,
y=0.99
)
plt.tight_layout(
rect=[0, 0, 1, 0.96]
)
plt.show()Number of observations = 342
Species distribution:
species
Adelie 151
Gentoo 123
Chinstrap 68
Name: count, dtype: int64
png
# ================================================================
# PALMER PENGUINS
# MULTICLASS CLASSIFICATION USING TWO VARIABLES AT A TIME
#
# Response:
# Adelie / Chinstrap / Gentoo
#
# Numerical variables:
# bill_length_mm
# bill_depth_mm
# flipper_length_mm
# body_mass_g
#
# For EVERY pair of variables fit:
# 1. Softmax Regression
# 2. LDA
# 3. QDA
# 4. Ridge Softmax
#
# Uses all complete observations for each pair.
#
# Outputs:
# - Pairwise true-data scatter plot
# - Decision boundary for every method
# - Row-normalized confusion matrix for every method
# - Accuracy / Macro Precision / Recall / F1
# - Final comparison across all 6 variable pairs
# ================================================================
# ================================================================
# 1. IMPORT PACKAGES
# ================================================================
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from itertools import combinations
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import (
LinearDiscriminantAnalysis,
QuadraticDiscriminantAnalysis
)
from sklearn.metrics import (
accuracy_score,
precision_score,
recall_score,
f1_score,
confusion_matrix
)
# ================================================================
# 2. LOAD DATA
# ================================================================
# Keep penguins.csv in the same folder as this notebook.
penguins = pd.read_csv("penguins.csv")
print("Original data shape:")
print(penguins.shape)
print("\nColumns:")
print(penguins.columns.tolist())
# ================================================================
# 3. NUMERICAL VARIABLES
# ================================================================
features = [
"bill_length_mm",
"bill_depth_mm",
"flipper_length_mm",
"body_mass_g"
]
class_order = [
"Adelie",
"Chinstrap",
"Gentoo"
]
# ================================================================
# 4. CLEAN RESPONSE
# ================================================================
penguins["species"] = (
penguins["species"]
.astype(str)
.str.strip()
)
penguins = penguins[
penguins["species"].isin(class_order)
].copy()
print("\nSpecies distribution:")
print(
penguins["species"]
.value_counts()
)
# ================================================================
# 5. ALL TWO-VARIABLE COMBINATIONS
# ================================================================
feature_pairs = list(
combinations(features, 2)
)
print("\nNumber of two-variable problems:")
print(len(feature_pairs))
print("\nPairs:")
for pair in feature_pairs:
print(pair)
# ================================================================
# 6. HELPER: FIT FOUR MODELS
# ================================================================
def fit_models(X, y):
# ------------------------------------------------------------
# Model 1: approximately unregularized Softmax
# ------------------------------------------------------------
softmax = Pipeline([
(
"scaler",
StandardScaler()
),
(
"model",
LogisticRegression(
penalty="l2",
C=1e6,
solver="lbfgs",
max_iter=5000
)
)
])
softmax.fit(
X,
y
)
# ------------------------------------------------------------
# Model 2: LDA
# ------------------------------------------------------------
lda = LinearDiscriminantAnalysis()
lda.fit(
X,
y
)
# ------------------------------------------------------------
# Model 3: QDA
# ------------------------------------------------------------
qda = QuadraticDiscriminantAnalysis()
qda.fit(
X,
y
)
# ------------------------------------------------------------
# Model 4: Ridge Softmax
#
# C = 1/lambda
# ------------------------------------------------------------
ridge_softmax = Pipeline([
(
"scaler",
StandardScaler()
),
(
"model",
LogisticRegression(
penalty="l2",
C=1.0,
solver="lbfgs",
max_iter=5000
)
)
])
ridge_softmax.fit(
X,
y
)
return {
"Softmax":
softmax,
"LDA":
lda,
"QDA":
qda,
"Ridge Softmax":
ridge_softmax
}
# ================================================================
# 7. HELPER: TRUE DATA CROSS-PLOT
# ================================================================
def plot_true_data(
X,
y,
var1,
var2
):
fig, ax = plt.subplots(
figsize=(7, 5.5)
)
markers = [
"o",
"s",
"^"
]
for species, marker in zip(
class_order,
markers
):
mask = (
y == species
)
ax.scatter(
X.loc[
mask,
var1
],
X.loc[
mask,
var2
],
s=30,
alpha=0.65,
marker=marker,
label=species
)
ax.set_xlabel(
var1
)
ax.set_ylabel(
var2
)
ax.set_title(
f"True Species: {var1} vs {var2}"
)
ax.legend()
plt.tight_layout()
plt.show()
# ================================================================
# 8. HELPER: DECISION BOUNDARY
# ================================================================
def plot_decision_boundary(
model,
X,
y,
var1,
var2,
method_name
):
# ------------------------------------------------------------
# Grid
# ------------------------------------------------------------
x1_min = (
X[var1].min()
-
0.08 *
(
X[var1].max()
-
X[var1].min()
)
)
x1_max = (
X[var1].max()
+
0.08 *
(
X[var1].max()
-
X[var1].min()
)
)
x2_min = (
X[var2].min()
-
0.08 *
(
X[var2].max()
-
X[var2].min()
)
)
x2_max = (
X[var2].max()
+
0.08 *
(
X[var2].max()
-
X[var2].min()
)
)
xx, yy = np.meshgrid(
np.linspace(
x1_min,
x1_max,
300
),
np.linspace(
x2_min,
x2_max,
300
)
)
grid = pd.DataFrame({
var1:
xx.ravel(),
var2:
yy.ravel()
})
# ------------------------------------------------------------
# Predict every grid point
# ------------------------------------------------------------
pred = model.predict(
grid
)
# Convert species labels to integers for contour plot
label_map = {
"Adelie": 0,
"Chinstrap": 1,
"Gentoo": 2
}
z = np.array(
[
label_map[p]
for p in pred
]
)
z = z.reshape(
xx.shape
)
# ------------------------------------------------------------
# Plot estimated regions
# ------------------------------------------------------------
fig, ax = plt.subplots(
figsize=(7, 5.5)
)
ax.contourf(
xx,
yy,
z,
levels=[
-0.5,
0.5,
1.5,
2.5
],
alpha=0.20
)
# Explicit boundary curves
ax.contour(
xx,
yy,
z,
levels=[
0.5,
1.5
],
linewidths=1.5
)
# ------------------------------------------------------------
# Overlay TRUE species
# ------------------------------------------------------------
markers = [
"o",
"s",
"^"
]
for species, marker in zip(
class_order,
markers
):
mask = (
y == species
)
ax.scatter(
X.loc[
mask,
var1
],
X.loc[
mask,
var2
],
s=27,
alpha=0.65,
marker=marker,
label=species
)
ax.set_xlabel(
var1
)
ax.set_ylabel(
var2
)
ax.set_title(
method_name
+
"\n"
+
var1
+
" vs "
+
var2
)
ax.legend()
plt.tight_layout()
plt.show()
# ================================================================
# 9. HELPER: NORMALIZED CONFUSION MATRIX
# ================================================================
def plot_confusion_probability(
y_true,
y_pred,
method_name,
pair_name
):
cm = confusion_matrix(
y_true,
y_pred,
labels=class_order,
normalize="true"
)
fig, ax = plt.subplots(
figsize=(6, 5)
)
image = ax.imshow(
cm,
vmin=0,
vmax=1
)
# ------------------------------------------------------------
# Put probabilities inside cells
# ------------------------------------------------------------
for i in range(3):
for j in range(3):
ax.text(
j,
i,
f"{cm[i,j]:.3f}",
ha="center",
va="center",
fontsize=11
)
ax.set_xticks(
[0, 1, 2]
)
ax.set_yticks(
[0, 1, 2]
)
ax.set_xticklabels(
[
"Adelie",
"Chinstrap",
"Gentoo"
],
rotation=20
)
ax.set_yticklabels(
[
"Adelie",
"Chinstrap",
"Gentoo"
]
)
ax.set_xlabel(
"Predicted Species"
)
ax.set_ylabel(
"True Species"
)
ax.set_title(
method_name
+
"\n"
+
pair_name
+
"\nRow-Normalized Confusion Matrix"
)
fig.colorbar(
image,
ax=ax,
label="Conditional Proportion"
)
plt.tight_layout()
plt.show()
return cm
# ================================================================
# 10. STORE RESULTS FROM ALL SIX PAIRS
# ================================================================
all_results = []
all_confusions = {}
# ================================================================
# 11. LOOP OVER EVERY TWO-VARIABLE PAIR
# ================================================================
for var1, var2 in feature_pairs:
print("\n")
print("=" * 75)
print(
"VARIABLE PAIR:"
)
print(
var1,
" + ",
var2
)
print("=" * 75)
# ------------------------------------------------------------
# Use all complete observations for THIS pair
# ------------------------------------------------------------
pair_data = penguins[
[
var1,
var2,
"species"
]
].dropna().copy()
X = pair_data[
[
var1,
var2
]
]
y = pair_data[
"species"
]
pair_name = (
var1
+
" + "
+
var2
)
print(
"\nSample size =",
len(pair_data)
)
print(
"\nSpecies counts:"
)
print(
y.value_counts()
)
# ------------------------------------------------------------
# A. True class cross-plot
# ------------------------------------------------------------
plot_true_data(
X,
y,
var1,
var2
)
# ------------------------------------------------------------
# B. Fit four methods
# ------------------------------------------------------------
models = fit_models(
X,
y
)
# ------------------------------------------------------------
# C. Evaluate every method
# ------------------------------------------------------------
pair_results = []
for method_name, model in models.items():
pred = model.predict(
X
)
accuracy = accuracy_score(
y,
pred
)
precision = precision_score(
y,
pred,
labels=class_order,
average="macro",
zero_division=0
)
recall = recall_score(
y,
pred,
labels=class_order,
average="macro",
zero_division=0
)
f1 = f1_score(
y,
pred,
labels=class_order,
average="macro",
zero_division=0
)
# --------------------------------------------------------
# Save result
# --------------------------------------------------------
result = {
"Variable_1":
var1,
"Variable_2":
var2,
"Method":
method_name,
"N":
len(pair_data),
"Accuracy":
accuracy,
"Precision_macro":
precision,
"Recall_macro":
recall,
"F1_macro":
f1
}
all_results.append(
result
)
pair_results.append(
result
)
# --------------------------------------------------------
# D. Decision boundary
# --------------------------------------------------------
plot_decision_boundary(
model,
X,
y,
var1,
var2,
method_name
)
# --------------------------------------------------------
# E. Normalized confusion matrix
# --------------------------------------------------------
cm = plot_confusion_probability(
y,
pred,
method_name,
pair_name
)
all_confusions[
(
pair_name,
method_name
)
] = cm
# --------------------------------------------------------
# Print confusion matrix
# --------------------------------------------------------
print(
"\n",
method_name,
" normalized confusion matrix:"
)
print(
pd.DataFrame(
cm,
index=[
"True Adelie",
"True Chinstrap",
"True Gentoo"
],
columns=[
"Pred Adelie",
"Pred Chinstrap",
"Pred Gentoo"
]
).round(4)
)
# ------------------------------------------------------------
# F. Comparison table for current pair
# ------------------------------------------------------------
pair_results_df = pd.DataFrame(
pair_results
)
print("\n")
print("-" * 75)
print(
"MODEL COMPARISON FOR:"
)
print(
pair_name
)
print("-" * 75)
print(
pair_results_df[
[
"Method",
"Accuracy",
"Precision_macro",
"Recall_macro",
"F1_macro"
]
]
.sort_values(
"Accuracy",
ascending=False
)
.round(4)
)
# ================================================================
# 12. FINAL RESULTS FOR ALL PAIRS
# ================================================================
results_all = pd.DataFrame(
all_results
)
print("\n")
print("=" * 80)
print(
"FINAL COMPARISON: ALL VARIABLE PAIRS"
)
print("=" * 80)
print(
results_all
.sort_values(
[
"Variable_1",
"Variable_2",
"Accuracy"
],
ascending=[
True,
True,
False
]
)
.round(4)
)
# ================================================================
# 13. BEST MODEL FOR EACH VARIABLE PAIR
# ================================================================
best_by_pair = (
results_all
.sort_values(
"Accuracy",
ascending=False
)
.groupby(
[
"Variable_1",
"Variable_2"
]
)
.first()
.reset_index()
)
print("\n")
print("=" * 80)
print(
"BEST METHOD FOR EACH VARIABLE PAIR"
)
print("=" * 80)
print(
best_by_pair[
[
"Variable_1",
"Variable_2",
"Method",
"N",
"Accuracy",
"F1_macro"
]
]
.round(4)
)
# ================================================================
# 14. BEST PAIR + METHOD OVERALL
# ================================================================
best_overall = (
results_all
.sort_values(
"Accuracy",
ascending=False
)
.iloc[0]
)
print("\n")
print("=" * 80)
print(
"BEST TWO-VARIABLE CLASSIFIER OVERALL"
)
print("=" * 80)
print(
"Variable 1 :",
best_overall[
"Variable_1"
]
)
print(
"Variable 2 :",
best_overall[
"Variable_2"
]
)
print(
"Method :",
best_overall[
"Method"
]
)
print(
"Accuracy :",
round(
best_overall[
"Accuracy"
],
4
)
)
print(
"Macro F1 :",
round(
best_overall[
"F1_macro"
],
4
)
)
# ================================================================
# 15. ACCURACY TABLE:
# ROW = VARIABLE PAIR
# COLUMN = METHOD
# ================================================================
results_all[
"Pair"
] = (
results_all[
"Variable_1"
]
+
"\n+\n"
+
results_all[
"Variable_2"
]
)
accuracy_table = results_all.pivot(
index="Pair",
columns="Method",
values="Accuracy"
)
print("\n")
print("=" * 80)
print(
"ACCURACY TABLE"
)
print("=" * 80)
print(
accuracy_table.round(4)
)
# ================================================================
# 16. MACRO-F1 TABLE
# ================================================================
f1_table = results_all.pivot(
index="Pair",
columns="Method",
values="F1_macro"
)
print("\n")
print("=" * 80)
print(
"MACRO F1 TABLE"
)
print("=" * 80)
print(
f1_table.round(4)
)
# ================================================================
# 17. ACCURACY COMPARISON PLOT
# ================================================================
accuracy_table.plot(
kind="bar",
figsize=(12, 6)
)
plt.ylim(
0,
1
)
plt.ylabel(
"Accuracy"
)
plt.xlabel(
"Two-Variable Predictor Set"
)
plt.title(
"Penguin Species Classification\n"
"Two Variables at a Time"
)
plt.xticks(
rotation=35,
ha="right"
)
plt.legend(
title="Method"
)
plt.tight_layout()
plt.show()
# ================================================================
# 18. MACRO-F1 COMPARISON PLOT
# ================================================================
f1_table.plot(
kind="bar",
figsize=(12, 6)
)
plt.ylim(
0,
1
)
plt.ylabel(
"Macro F1"
)
plt.xlabel(
"Two-Variable Predictor Set"
)
plt.title(
"Penguin Species Classification\n"
"Macro-F1 for Two Variables at a Time"
)
plt.xticks(
rotation=35,
ha="right"
)
plt.legend(
title="Method"
)
plt.tight_layout()
plt.show()
# ================================================================
# 19. FINAL NOTE
# ================================================================
print("\nIMPORTANT:")
print(
"Every classifier above uses exactly TWO numerical predictors."
)
print(
"There are six possible pairs from the four numerical measurements."
)
print(
"All complete observations for the corresponding pair are used."
)
print(
"Since fitting and evaluation use the same observations,"
)
print(
"these results measure in-sample fit rather than independent"
"generalization performance."
)Original data shape:
(344, 7)
Columns:
['species', 'island', 'bill_length_mm', 'bill_depth_mm', 'flipper_length_mm', 'body_mass_g', 'sex']
Species distribution:
species
Adelie 152
Gentoo 124
Chinstrap 68
Name: count, dtype: int64
Number of two-variable problems:
6
Pairs:
('bill_length_mm', 'bill_depth_mm')
('bill_length_mm', 'flipper_length_mm')
('bill_length_mm', 'body_mass_g')
('bill_depth_mm', 'flipper_length_mm')
('bill_depth_mm', 'body_mass_g')
('flipper_length_mm', 'body_mass_g')
===========================================================================
VARIABLE PAIR:
bill_length_mm + bill_depth_mm
===========================================================================
Sample size = 342
Species counts:
species
Adelie 151
Gentoo 123
Chinstrap 68
Name: count, dtype: int64
png
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
png
png
Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9801 0.0199 0.0000
True Chinstrap 0.0441 0.8971 0.0588
True Gentoo 0.0000 0.0163 0.9837
png
png
LDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9868 0.0132 0.0000
True Chinstrap 0.0735 0.8676 0.0588
True Gentoo 0.0000 0.0163 0.9837
png
png
QDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9868 0.0132 0.0000
True Chinstrap 0.0588 0.9118 0.0294
True Gentoo 0.0000 0.0244 0.9756
png
png
Ridge Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9868 0.0132 0.0000
True Chinstrap 0.0735 0.8529 0.0735
True Gentoo 0.0000 0.0163 0.9837
---------------------------------------------------------------------------
MODEL COMPARISON FOR:
bill_length_mm + bill_depth_mm
---------------------------------------------------------------------------
Method Accuracy Precision_macro Recall_macro F1_macro
2 QDA 0.9678 0.9609 0.9580 0.9595
0 Softmax 0.9649 0.9575 0.9536 0.9555
1 LDA 0.9620 0.9573 0.9460 0.9512
3 Ridge Softmax 0.9591 0.9544 0.9411 0.9471
===========================================================================
VARIABLE PAIR:
bill_length_mm + flipper_length_mm
===========================================================================
Sample size = 342
Species counts:
species
Adelie 151
Gentoo 123
Chinstrap 68
Name: count, dtype: int64
png
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
png
png
Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9669 0.0199 0.0132
True Chinstrap 0.0882 0.8824 0.0294
True Gentoo 0.0000 0.0081 0.9919
png
png
LDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9669 0.0199 0.0132
True Chinstrap 0.0882 0.8676 0.0441
True Gentoo 0.0000 0.0081 0.9919
png
png
QDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9603 0.0199 0.0199
True Chinstrap 0.0882 0.8824 0.0294
True Gentoo 0.0000 0.0081 0.9919
png
png
Ridge Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9669 0.0199 0.0132
True Chinstrap 0.0882 0.8676 0.0441
True Gentoo 0.0000 0.0081 0.9919
---------------------------------------------------------------------------
MODEL COMPARISON FOR:
bill_length_mm + flipper_length_mm
---------------------------------------------------------------------------
Method Accuracy Precision_macro Recall_macro F1_macro
0 Softmax 0.9591 0.9554 0.9470 0.9509
1 LDA 0.9561 0.9526 0.9421 0.9468
2 QDA 0.9561 0.9528 0.9448 0.9485
3 Ridge Softmax 0.9561 0.9526 0.9421 0.9468
===========================================================================
VARIABLE PAIR:
bill_length_mm + body_mass_g
===========================================================================
Sample size = 342
Species counts:
species
Adelie 151
Gentoo 123
Chinstrap 68
Name: count, dtype: int64
png
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
png
png
Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9536 0.0066 0.0397
True Chinstrap 0.0294 0.9559 0.0147
True Gentoo 0.0407 0.0081 0.9512
png
png
LDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9603 0.0000 0.0397
True Chinstrap 0.0588 0.9118 0.0294
True Gentoo 0.0488 0.0000 0.9512
png
png
QDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9470 0.0066 0.0464
True Chinstrap 0.0588 0.9118 0.0294
True Gentoo 0.0244 0.0000 0.9756
png
png
Ridge Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9603 0.0066 0.0331
True Chinstrap 0.0588 0.9265 0.0147
True Gentoo 0.0325 0.0000 0.9675
---------------------------------------------------------------------------
MODEL COMPARISON FOR:
bill_length_mm + body_mass_g
---------------------------------------------------------------------------
Method Accuracy Precision_macro Recall_macro F1_macro
3 Ridge Softmax 0.9561 0.9614 0.9514 0.9561
0 Softmax 0.9532 0.9558 0.9536 0.9547
2 QDA 0.9503 0.9559 0.9448 0.9497
1 LDA 0.9474 0.9572 0.9411 0.9484
===========================================================================
VARIABLE PAIR:
bill_depth_mm + flipper_length_mm
===========================================================================
Sample size = 342
Species counts:
species
Adelie 151
Gentoo 123
Chinstrap 68
Name: count, dtype: int64
png
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
png
png
Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9205 0.0795 0.0
True Chinstrap 0.6471 0.3529 0.0
True Gentoo 0.0000 0.0000 1.0
png
png
LDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9073 0.0927 0.0
True Chinstrap 0.6324 0.3676 0.0
True Gentoo 0.0000 0.0000 1.0
png
png
QDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9338 0.0662 0.0
True Chinstrap 0.6912 0.3088 0.0
True Gentoo 0.0000 0.0000 1.0
png
png
Ridge Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9404 0.0596 0.0
True Chinstrap 0.7059 0.2941 0.0
True Gentoo 0.0000 0.0000 1.0
---------------------------------------------------------------------------
MODEL COMPARISON FOR:
bill_depth_mm + flipper_length_mm
---------------------------------------------------------------------------
Method Accuracy Precision_macro Recall_macro F1_macro
0 Softmax 0.8363 0.8087 0.7578 0.7646
1 LDA 0.8333 0.8007 0.7583 0.7650
2 QDA 0.8333 0.8091 0.7475 0.7520
3 Ridge Softmax 0.8333 0.8123 0.7448 0.7484
===========================================================================
VARIABLE PAIR:
bill_depth_mm + body_mass_g
===========================================================================
Sample size = 342
Species counts:
species
Adelie 151
Gentoo 123
Chinstrap 68
Name: count, dtype: int64
png
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
png
png
Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 1.0 0.0 0.0
True Chinstrap 1.0 0.0 0.0
True Gentoo 0.0 0.0 1.0
png
png
LDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 1.0 0.0 0.0
True Chinstrap 1.0 0.0 0.0
True Gentoo 0.0 0.0 1.0
png
png
QDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 1.0 0.0 0.0
True Chinstrap 1.0 0.0 0.0
True Gentoo 0.0 0.0 1.0
png
png
Ridge Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 1.0 0.0 0.0
True Chinstrap 1.0 0.0 0.0
True Gentoo 0.0 0.0 1.0
---------------------------------------------------------------------------
MODEL COMPARISON FOR:
bill_depth_mm + body_mass_g
---------------------------------------------------------------------------
Method Accuracy Precision_macro Recall_macro F1_macro
0 Softmax 0.8012 0.5632 0.6667 0.6054
1 LDA 0.8012 0.5632 0.6667 0.6054
2 QDA 0.8012 0.5632 0.6667 0.6054
3 Ridge Softmax 0.8012 0.5632 0.6667 0.6054
===========================================================================
VARIABLE PAIR:
flipper_length_mm + body_mass_g
===========================================================================
Sample size = 342
Species counts:
species
Adelie 151
Gentoo 123
Chinstrap 68
Name: count, dtype: int64
png
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/sklearn/linear_model/_logistic.py:1403: FutureWarning: 'penalty' was deprecated in version 1.8 and will be removed in 1.10. To avoid this warning, leave 'penalty' set to its default value and use 'l1_ratio' or 'C' instead. Use l1_ratio=0 instead of penalty='l2', l1_ratio=1 instead of penalty='l1', l1_ratio set to a float between 0 and 1 instead of penalty='elasticnet', and C=np.inf instead of penalty=None.
warnings.warn(
png
png
Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.8940 0.0927 0.0132
True Chinstrap 0.5882 0.3382 0.0735
True Gentoo 0.0081 0.0081 0.9837
png
png
LDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.8543 0.1258 0.0199
True Chinstrap 0.5735 0.3382 0.0882
True Gentoo 0.0081 0.0081 0.9837
png
png
QDA normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9007 0.0861 0.0132
True Chinstrap 0.5735 0.3382 0.0882
True Gentoo 0.0081 0.0081 0.9837
png
png
Ridge Softmax normalized confusion matrix:
Pred Adelie Pred Chinstrap Pred Gentoo
True Adelie 0.9139 0.0662 0.0199
True Chinstrap 0.6618 0.2647 0.0735
True Gentoo 0.0000 0.0081 0.9919
---------------------------------------------------------------------------
MODEL COMPARISON FOR:
flipper_length_mm + body_mass_g
---------------------------------------------------------------------------
Method Accuracy Precision_macro Recall_macro F1_macro
2 QDA 0.8187 0.7774 0.7409 0.7434
0 Softmax 0.8158 0.7725 0.7387 0.7413
3 Ridge Softmax 0.8129 0.7711 0.7235 0.7206
1 LDA 0.7982 0.7430 0.7254 0.7257
================================================================================
FINAL COMPARISON: ALL VARIABLE PAIRS
================================================================================
Variable_1 Variable_2 Method N Accuracy \
16 bill_depth_mm body_mass_g Softmax 342 0.8012
17 bill_depth_mm body_mass_g LDA 342 0.8012
18 bill_depth_mm body_mass_g QDA 342 0.8012
19 bill_depth_mm body_mass_g Ridge Softmax 342 0.8012
12 bill_depth_mm flipper_length_mm Softmax 342 0.8363
13 bill_depth_mm flipper_length_mm LDA 342 0.8333
14 bill_depth_mm flipper_length_mm QDA 342 0.8333
15 bill_depth_mm flipper_length_mm Ridge Softmax 342 0.8333
2 bill_length_mm bill_depth_mm QDA 342 0.9678
0 bill_length_mm bill_depth_mm Softmax 342 0.9649
1 bill_length_mm bill_depth_mm LDA 342 0.9620
3 bill_length_mm bill_depth_mm Ridge Softmax 342 0.9591
11 bill_length_mm body_mass_g Ridge Softmax 342 0.9561
8 bill_length_mm body_mass_g Softmax 342 0.9532
10 bill_length_mm body_mass_g QDA 342 0.9503
9 bill_length_mm body_mass_g LDA 342 0.9474
4 bill_length_mm flipper_length_mm Softmax 342 0.9591
5 bill_length_mm flipper_length_mm LDA 342 0.9561
6 bill_length_mm flipper_length_mm QDA 342 0.9561
7 bill_length_mm flipper_length_mm Ridge Softmax 342 0.9561
22 flipper_length_mm body_mass_g QDA 342 0.8187
20 flipper_length_mm body_mass_g Softmax 342 0.8158
23 flipper_length_mm body_mass_g Ridge Softmax 342 0.8129
21 flipper_length_mm body_mass_g LDA 342 0.7982
Precision_macro Recall_macro F1_macro
16 0.5632 0.6667 0.6054
17 0.5632 0.6667 0.6054
18 0.5632 0.6667 0.6054
19 0.5632 0.6667 0.6054
12 0.8087 0.7578 0.7646
13 0.8007 0.7583 0.7650
14 0.8091 0.7475 0.7520
15 0.8123 0.7448 0.7484
2 0.9609 0.9580 0.9595
0 0.9575 0.9536 0.9555
1 0.9573 0.9460 0.9512
3 0.9544 0.9411 0.9471
11 0.9614 0.9514 0.9561
8 0.9558 0.9536 0.9547
10 0.9559 0.9448 0.9497
9 0.9572 0.9411 0.9484
4 0.9554 0.9470 0.9509
5 0.9526 0.9421 0.9468
6 0.9528 0.9448 0.9485
7 0.9526 0.9421 0.9468
22 0.7774 0.7409 0.7434
20 0.7725 0.7387 0.7413
23 0.7711 0.7235 0.7206
21 0.7430 0.7254 0.7257
================================================================================
BEST METHOD FOR EACH VARIABLE PAIR
================================================================================
Variable_1 Variable_2 Method N Accuracy \
0 bill_depth_mm body_mass_g Softmax 342 0.8012
1 bill_depth_mm flipper_length_mm Softmax 342 0.8363
2 bill_length_mm bill_depth_mm QDA 342 0.9678
3 bill_length_mm body_mass_g Ridge Softmax 342 0.9561
4 bill_length_mm flipper_length_mm Softmax 342 0.9591
5 flipper_length_mm body_mass_g QDA 342 0.8187
F1_macro
0 0.6054
1 0.7646
2 0.9595
3 0.9561
4 0.9509
5 0.7434
================================================================================
BEST TWO-VARIABLE CLASSIFIER OVERALL
================================================================================
Variable 1 : bill_length_mm
Variable 2 : bill_depth_mm
Method : QDA
Accuracy : 0.9678
Macro F1 : 0.9595
================================================================================
ACCURACY TABLE
================================================================================
Method LDA QDA Ridge Softmax Softmax
Pair
bill_depth_mm\n+\nbody_mass_g 0.8012 0.8012 0.8012 0.8012
bill_depth_mm\n+\nflipper_length_mm 0.8333 0.8333 0.8333 0.8363
bill_length_mm\n+\nbill_depth_mm 0.9620 0.9678 0.9591 0.9649
bill_length_mm\n+\nbody_mass_g 0.9474 0.9503 0.9561 0.9532
bill_length_mm\n+\nflipper_length_mm 0.9561 0.9561 0.9561 0.9591
flipper_length_mm\n+\nbody_mass_g 0.7982 0.8187 0.8129 0.8158
================================================================================
MACRO F1 TABLE
================================================================================
Method LDA QDA Ridge Softmax Softmax
Pair
bill_depth_mm\n+\nbody_mass_g 0.6054 0.6054 0.6054 0.6054
bill_depth_mm\n+\nflipper_length_mm 0.7650 0.7520 0.7484 0.7646
bill_length_mm\n+\nbill_depth_mm 0.9512 0.9595 0.9471 0.9555
bill_length_mm\n+\nbody_mass_g 0.9484 0.9497 0.9561 0.9547
bill_length_mm\n+\nflipper_length_mm 0.9468 0.9485 0.9468 0.9509
flipper_length_mm\n+\nbody_mass_g 0.7257 0.7434 0.7206 0.7413
png
png
IMPORTANT:
Every classifier above uses exactly TWO numerical predictors.
There are six possible pairs from the four numerical measurements.
All complete observations for the corresponding pair are used.
Since fitting and evaluation use the same observations,
these results measure in-sample fit rather than independentgeneralization performance.