setwd("/Users/isaiahmireles/Desktop/Misconceptions")
cohorts <- read.csv("cohorts.csv")
cohorts$exam <- factor(cohorts$exam)
library(tidyverse)

1 `PPQ` and `Midterms_lst` object(s)

cohorts |>
  filter(unidentified==FALSE) |> 
  count(term)

cohorts |>
  filter(unidentified==FALSE) |> 
  count(exam, term)

# percent per q dat 
PPQ <- 
  cohorts |>
  # only keep students of which we may map from midterm to final
  filter(unidentified==FALSE) |> 
  pivot_longer(
    # select starts w/ Q, then 1 digit 
    cols = matches("^Q\\d+$"),
    # grab q, 
    names_to = "question",
    values_to = "correct"
  ) |>
  group_by(term, exam, question) |>
  # ct std (matches above)
  summarize(
    n_students = n(),
    PPQ = mean(correct), 
    .groups = "drop"
  )

1.1 Split `PPQ` into `term, exam` specific data frames

# add grouping
PPQ <- 
  PPQ |> 
  group_by(term, exam) 

PPQ_lst <-
  PPQ |>
  group_split() |>
  set_names(
    PPQ |>
      group_keys() |>
      transmute(name = paste(term, exam, sep = "_")) |>
      pull(name)
  )

Midterms <- PPQ_lst[str_detect(names(PPQ_lst), "_Midterm$")]
Midterms_lst <- as.list(Midterms)

2 Comment :

Exam length
Ordering
Problem type

varies across exam, below we manually label this fact

3 Manual Labels

3.1 Midterm

3.1.1 Spring

we are doing everything relative to spring so all labels are boring in a sense

Midterms_lst$S24_Midterm <- Midterms_lst$S24_Midterm |>
  mutate(
    difference = factor(
      "Same",
      levels = c("Same", "Similar", "insignificant", "Not Included")
    ),
    description = "No difference",
    comment = NA_character_,
    Question_Order = paste(question, "-- Spring")
  )

3.2 Include `Unique to Spring` label

Midterms_lst$S24_Midterm <- Midterms_lst$S24_Midterm |>
  mutate(
    difference = factor(
      if_else(
        question %in% c("Q32", "Q33", "Q34"),
        "Not Included",
        "Same"
      ),
      levels = c("Same", "Similar", "insignificant", "Not Included")
    ),
    description = if_else(
      question %in% c("Q32", "Q33", "Q34"),
      "Question unique to Spring exam",
      "No difference"
    ),
    comment = if_else(
      question %in% c("Q32", "Q33", "Q34"),
      "Not included in Fall or Winter exams",
      NA_character_
    ),
    Question_Order = paste(question, "-- Spring")
  )

Midterms_lst$S24_Midterm |>
  filter(question %in% c("Q32", "Q33", "Q34")) |>
  select(question, difference, description, comment, Question_Order)

3.2.1 Winter

difference, description, comment

Midterms_lst$W24_Midterm <- Midterms_lst$W24_Midterm |>
  mutate(
    difference = case_when(
      question == "Q27" ~ "insignificant",
      question %in% c("Q32", "Q33") ~ "Not Included",
      TRUE ~ "Same"
    ),
    difference = factor(
      difference,
      levels = c("Same", "Similar", "insignificant", "Not Included")
    ),
    description = case_when(
      difference == "Same" ~ "No difference",
      difference == "Similar" ~ "Similar type of problem with different wording",
      difference == "insignificant" ~ "Very small changes in wording",
      difference == "Not Included" ~ "Question not included in Spring class"
    ),
    comment = case_when(
      question == "Q27" ~ "He vs. Ry : histogram main title naming difference",
      question == "Q32" ~ "Same Q Across Fall, Winter Quarter",
      question == "Q33" ~ "Unique to Winter Exam",
      TRUE ~ NA_character_
    )
  )

Midterms_lst$W24_Midterm |>
  filter(question %in% c("Q27", "Q32", "Q33")) |>
  select(term, exam, question, difference, description, comment)

Question_Order

Midterms_lst$W24_Midterm <- Midterms_lst$W24_Midterm |>
  mutate(
    Question_Order = case_when(
      question %in% c("Q32", "Q33") ~ NA_character_,
      TRUE ~ paste(question, "-- Spring")
    )
  )

3.2.2 Fall

Midterms_lst$F23_Midterm <- Midterms_lst$F23_Midterm |>
  mutate(
    Question_Order = case_when(
      question %in% c("Q11", "Q24", "Q30", "Q32", "Q33", "Q34") ~ NA_character_,
      question == "Q2" ~ "Q3 -- Spring",
      question == "Q3" ~ "Q4 -- Spring",
      question == "Q4" ~ "Q7 -- Spring",
      question == "Q5" ~ "Q8 -- Spring",
      question == "Q6" ~ "Q9 -- Spring",
      question == "Q7" ~ "Q11 -- Spring",
      question == "Q8" ~ "Q12 -- Spring",
      question == "Q9" ~ "Q13 -- Spring",
      question == "Q10" ~ "Q14 -- Spring",
      question == "Q12" ~ "Q6 -- Spring",
      question == "Q13" ~ "Q10 -- Spring",
      question == "Q14" ~ "Q23 -- Spring",
      question == "Q15" ~ "Q24 -- Spring",
      question == "Q16" ~ "Q25 -- Spring",
      question == "Q17" ~ "Q26 -- Spring",
      question == "Q18" ~ "Q27 -- Spring",
      question == "Q19" ~ "Q28 -- Spring",
      question == "Q21" ~ "Q30 -- Spring",
      question == "Q22" ~ "Q31 -- Spring",
      question == "Q23" ~ "Q16 -- Spring",
      question == "Q25" ~ "Q16 -- Spring",
      question == "Q26" ~ "Q18 -- Spring",
      question == "Q27" ~ "Q19 -- Spring",
      question == "Q28" ~ "Q17 -- Spring",
      question == "Q31" ~ "Q22 -- Spring",
      TRUE ~ paste(question, "-- Spring")
    ),
    difference = case_when(
      question %in% c("Q11", "Q24", "Q30", "Q32", "Q33", "Q34") ~ "Not Included",
      question %in% c("Q17", "Q18", "Q19", "Q21") ~ "insignificant",
      question %in% c("Q23", "Q25", "Q26", "Q27", "Q28", "Q31") ~ "Similar",
      TRUE ~ "Same"
    ),
    difference = factor(
      difference,
      levels = c("Same", "Similar", "insignificant", "Not Included")
    ),
    description = case_when(
      difference == "Same" ~ "No difference",
      difference == "Similar" ~ "Similar type of problem with different wording",
      difference == "insignificant" ~ "Very small changes in wording",
      difference == "Not Included" ~ "Question not included in Spring class"
    ),
    comment = case_when(
      question %in% c("Q11", "Q30", "Q33", "Q34") ~ "Unique to Fall Exam",
      question == "Q17" ~ "Different amount (400 not 500) and different correct response (30% not 15%)",
      question == "Q18" ~ "He vs. Ry : histogram main title naming difference; Same as winter",
      question == "Q19" ~ "problem context : Ice cream instead of tea",
      question == "Q21" ~ "correlation of .128 instead of .129",
      question == "Q23" ~ "Type of Study Problem (experiment vs. observational study)",
      question == "Q24" ~ "Best visualized by which plot Problem",
      question == "Q25" ~ "Reg. Estimate given x-value, context difference",
      question == "Q26" ~ "Reg. intercept interpretation, context difference",
      question == "Q27" ~ "Reg. slope interpretation, context difference",
      question == "Q28" ~ "coef of determination interpretation, context difference",
      question == "Q31" ~ "Choosing the correct mdl",
      question == "Q32" ~ "Same Q Across Fall, Winter Quarter",
      TRUE ~ NA_character_
    )
  )

Midterms_lst$F23_Midterm |>
  filter(
    difference != "Same" |
      Question_Order != paste(question, "-- Spring")
  ) |>
  select(term, exam, question, difference, description, comment)

4 Brief Note

# Average question performance
Midterms_lst$W24_Midterm |>
  summarize(avr = mean(PPQ, na.rm = TRUE))

cohorts <- cohorts |> rename(pct_rounded = pct)

# Average student performance
cohorts |>
  filter(term == "W24", exam == "Midterm") |>
  summarize(avr = mean(pct_rounded, na.rm = TRUE))

These should be and are equivalent^

5 Refine Data Structure

The following website, refers to the data cleaning process used in generating cohorts.csv. A first attempt at data cleaning.
The purpose of this section is to contextualize the data & generate a codebook.pdf

# view data structure of cohorts.csv
cohorts |> str()

## 'data.frame':    1690 obs. of  44 variables:
##  $ X           : int  1 2 3 4 5 6 7 8 9 10 ...
##  $ unidentified: logi  FALSE FALSE FALSE FALSE FALSE FALSE ...
##  $ term        : chr  "F23" "F23" "F23" "F23" ...
##  $ exam        : Factor w/ 2 levels "Final","Midterm": 2 2 2 2 2 2 2 2 2 2 ...
##  $ version     : chr  "A" "A" "A" "A" ...
##  $ student_id  : chr  "student_1" "student_2" "student_3" "student_4" ...
##  $ Sections    : chr  "23F-STATS-10-DIS-2B" "23F-STATS-10-DIS-2C" "23F-STATS-10-DIS-2D" "23F-STATS-10-DIS-2A" ...
##  $ Total.Score : int  26 21 27 9 34 33 33 17 30 31 ...
##  $ Max.Points  : int  34 34 34 34 34 34 34 34 34 34 ...
##  $ Q1          : int  1 1 1 1 1 1 1 1 1 1 ...
##  $ Q2          : int  1 0 1 0 1 1 1 1 1 1 ...
##  $ Q3          : int  0 1 1 0 1 1 1 0 1 1 ...
##  $ Q4          : int  1 1 1 0 1 1 1 1 1 1 ...
##  $ Q5          : int  1 1 1 0 1 1 1 0 0 1 ...
##  $ Q6          : int  1 1 1 0 1 1 1 1 1 1 ...
##  $ Q7          : int  1 1 1 1 1 1 1 1 1 1 ...
##  $ Q8          : int  1 1 1 0 1 1 1 1 1 1 ...
##  $ Q9          : int  1 0 0 0 1 1 1 0 1 1 ...
##  $ Q10         : int  1 1 1 1 1 1 1 1 1 1 ...
##  $ Q11         : int  1 0 1 0 1 1 1 0 1 1 ...
##  $ Q12         : int  1 1 1 1 1 1 1 1 1 1 ...
##  $ Q13         : int  0 0 1 0 1 1 1 1 1 1 ...
##  $ Q14         : int  0 1 1 0 1 0 1 0 1 0 ...
##  $ Q15         : int  0 1 1 0 1 1 1 1 1 0 ...
##  $ Q16         : int  1 1 1 0 1 1 1 1 1 1 ...
##  $ Q17         : int  1 1 1 1 1 1 1 0 1 1 ...
##  $ Q18         : int  1 1 1 1 1 1 1 1 1 1 ...
##  $ Q19         : int  0 0 1 0 1 1 1 1 1 0 ...
##  $ Q20         : int  1 0 1 0 1 1 1 0 1 1 ...
##  $ Q21         : int  1 1 1 0 1 1 1 1 1 1 ...
##  $ Q22         : int  1 0 1 0 1 1 1 0 1 1 ...
##  $ Q23         : int  0 1 1 0 1 1 1 0 1 1 ...
##  $ Q24         : int  1 0 0 1 1 1 1 1 1 1 ...
##  $ Q25         : int  1 0 1 0 1 1 1 1 1 1 ...
##  $ Q26         : int  0 0 0 1 1 1 1 0 1 1 ...
##  $ Q27         : int  1 1 1 0 1 1 1 0 1 1 ...
##  $ Q28         : int  1 1 0 0 1 1 1 0 0 1 ...
##  $ Q29         : int  1 1 1 1 1 1 1 1 1 1 ...
##  $ Q30         : int  0 1 1 0 1 1 1 0 1 1 ...
##  $ Q31         : int  1 1 0 0 1 1 1 0 1 1 ...
##  $ Q32         : int  1 0 1 0 1 1 1 0 1 1 ...
##  $ Q33         : int  1 0 0 0 1 1 1 0 0 1 ...
##  $ Q34         : int  1 0 0 0 1 1 0 0 0 1 ...
##  $ pct_rounded : num  0.765 0.618 0.794 0.265 1 ...

cohorts$Q1 |> class() # here we see it as class integer

## [1] "integer"

5.1 Categorical Variables w/ Levels

# convert all chr types to factor() -- except std_id
cohorts <- cohorts |>
  mutate(
    across(
      c(term, exam, version, Sections),
      as.factor
    )
  )

except std_id because there aren’t levels, there are not categorical variables but rather an ID variable

# levels(cohorts$Sections)
cohorts |> count(term, exam, Sections)

notice some students belong to multiple sections
- ie a student changed sections
Some students dont belong to a section
Discussion levels are overly wordy

5.1.1 `Sections` Variable

cohorts <- cohorts |>
  mutate(
    Sections = Sections |>
      str_extract_all("DIS-\\d+[A-Z]") |>
      map_chr(~ if (length(.x) == 0) {
        "Missing"
      } else {
        str_c(.x, collapse = ", ")
      }) |>
      factor()
  )

# new naming!
cohorts$Sections |> levels()

##  [1] "DIS-1A"         "DIS-1A, DIS-2C" "DIS-1B"         "DIS-1C"        
##  [5] "DIS-1D"         "DIS-2A"         "DIS-2B"         "DIS-2B, DIS-2A"
##  [9] "DIS-2C"         "DIS-2D"         "DIS-4A"         "DIS-4B"        
## [13] "DIS-4B, DIS-4A" "DIS-4C"         "DIS-4D"         "Missing"

cohorts |> count(term, exam, Sections)

5.1.2 Investigate `Sections == "Missing"`

cohorts |> 
  count(Sections, unidentified) |> 
  filter(Sections == "Missing")

notice that most of unidentified==TRUE students are also those whom are missing their sections. Which makes sense as these are students whom dropped the class after the midterm.

cohorts |> count(unidentified, term)

Note Fall is the only course where we have "unidentified==FALSE" ; in this case these are students whom dropped the course. Other students did drop the course in the other course however were not labeled as unidentified

5.2 Investigate `Sections == "Missing" AND unidentified == FALSE`

cohorts |> 
  filter(Sections == "Missing", unidentified == FALSE) |>
  select(student_id, term, exam, version, pct_rounded)

unknown cause of Sections == "Missing" AND unidentified == FALSE, most come from S24.
For the sake of brevity we will move on an investigate this more closely at another time.
- NOTE TO SELF : Review Data Cleaning Procedure for AB Section listing

# same & write new dat 
# write.csv(cohorts, "cohorts.csv")

Midterms_lst <- Midterms_lst |>
  map(~ mutate(.x, across(where(is.character), as.factor)))

# allows us to apply operation across vectors/df in list
?map

6 Save R objects

save(
  cohorts,
  Midterms_lst,
  file = "S10 Data.RData"
)

# overide previus cohorts version
write.csv(cohorts, "cohorts.csv")

rm(list=ls()); setwd("/Users/isaiahmireles/Desktop/Misconceptions"); load("S10 Data.RData")

7 Graph(s)

7.1 Bar Plots (Relative Ordering)

“Relative Ordering” : we do so so we may compare apples to apples and not apples to oranges – ie. question ordering varies.

cohorts |> 
  dplyr::filter(exam == "Midterm") |> 
  group_by(term) |> 
  summarize(med = round(median(pct_rounded), 2), 
            xbar = round(mean(pct_rounded), 2))

medians <- tibble(
  term = c("F23", "S24", "W24"),
  med = c(0.85, 0.79, 0.82)
)

Midterms_lst <- Midterms_lst |>
  map(~ .x |>
    mutate(term = as.character(term)) |>
    left_join(medians, by = "term") |>
    mutate(
      below_median = as.integer(PPQ < med)
    ) |>
    select(-med)
  )

7.1.1 Spring

S24_plot <- Midterms_lst$S24_Midterm |>
  mutate(
    question = factor(
      as.character(question),
      levels = paste0("Q", 1:34)
    )
  ) |>
  ggplot(aes(x = question, y = PPQ, fill = difference)) +
  geom_col(
    color = "black",
    linewidth = 0.2,
    show.legend = TRUE
  ) +
  # Add percentage labels above bars
  geom_text(
    aes(
      label = scales::percent(PPQ, accuracy = 1),
      color = factor(below_median)
    ),
    vjust = -0.4,
    size = 3,
    show.legend = FALSE
  ) +
  # Red if below median, black otherwise
  scale_color_manual(
    values = c(
      "0" = "black",
      "1" = "red"
    ),
    guide = "none"
  ) +
  scale_y_continuous(
    breaks = seq(0, 1, 0.2),
    expand = expansion(mult = c(0, 0.08))
  ) +
  coord_cartesian(
    ylim = c(0, 1.08),
    clip = "off"
  ) +
  # Question classification colors
  scale_fill_manual(
    values = c(
      "Same" = "steelblue",
      "Similar" = "orange",
      "insignificant" = "skyblue",
      "Not Included" = "firebrick"
    ),
    drop = FALSE
  ) +
  labs(
    title = "Spring 2024 Midterm: Question Performance",
    subtitle = "Reference examination (n = 149 students)",
    x = "Question",
    y = "Proportion Correct (PPQ)",
    fill = "Question Classification"
  ) +
  theme_minimal() +
  theme(
    axis.text.x = element_text(
      angle = 45,
      hjust = 1
    ),
    panel.grid.major.x = element_blank(),
    plot.title = element_text(face = "bold")
  )

S24_plot

ggsave(
  filename = "S24_Midterm_PPQ.png",
  plot = S24_plot,
  width = 14,
  height = 6,
  dpi = 300
)

7.1.2 Winter

W24_plot <- Midterms_lst$W24_Midterm |>
  mutate(
    # Order questions relative to Spring
    question_num = readr::parse_number(as.character(question)),
    spring_order_num = readr::parse_number(as.character(Question_Order)),
    spring_order_num = if_else(
      is.na(spring_order_num),
      Inf,
      spring_order_num
    )
  ) |>
  arrange(spring_order_num, question_num) |>
  mutate(
    question = factor(
      as.character(question),
      levels = as.character(question)
    )
  ) |>
  ggplot(aes(x = question, y = PPQ, fill = difference)) +
  geom_col(
    color = "black",
    linewidth = 0.2,
    show.legend = TRUE
  ) +
  # Add percentage labels above bars
  geom_text(
    aes(
      label = scales::percent(PPQ, accuracy = 1),
      color = factor(below_median)
    ),
    vjust = -0.4,
    size = 3,
    show.legend = FALSE
  ) +
  # Red if below median, black otherwise
  scale_color_manual(
    values = c(
      "0" = "black",
      "1" = "red"
    ),
    guide = "none"
  ) +
  scale_y_continuous(
    breaks = seq(0, 1, 0.2),
    expand = expansion(mult = c(0, 0.08))
  ) +
  coord_cartesian(
    ylim = c(0, 1.08),
    clip = "off"
  ) +
  # Question classification colors
  scale_fill_manual(
    values = c(
      "Same" = "steelblue",
      "Similar" = "orange",
      "insignificant" = "skyblue",
      "Not Included" = "firebrick"
    ),
    drop = FALSE
  ) +
  labs(
    title = "Winter 2024 Midterm: Question Performance",
    subtitle = "Ordered by Spring-relative question alignment (n = 297 students)",
    x = "Winter question (ordered by Spring correspondence)",
    y = "Proportion Correct (PPQ)",
    fill = "Question Classification"
  ) +
  theme_minimal() +
  theme(
    axis.text.x = element_text(
      angle = 45,
      hjust = 1
    ),
    panel.grid.major.x = element_blank(),
    plot.title = element_text(face = "bold")
  )

W24_plot

note that this exam doesnt include Q34 and has 32, 33 unique to it.

Midterms_lst$W24_Midterm  |>
  arrange(readr::parse_number(as.character(question))) |>
  select(difference, question, Question_Order)

ggsave(
  filename = "W24_Midterm_PPQ.png",
  plot = W24_plot,
  width = 14,
  height = 6,
  dpi = 300
)

7.1.3 Fall

F23_plot_data <- Midterms_lst$F23_Midterm |>
  mutate(
    question_num = readr::parse_number(as.character(question)),
    question_label = if_else(
      is.na(Question_Order),
      paste0(as.character(question), "*"),
      sub(" -- Spring$", "", as.character(Question_Order))
    )
  ) |>
  arrange(question_num) |>
  mutate(
    question = factor(
      as.character(question),
      levels = unique(as.character(question))
    )
  )

F23_plot <- F23_plot_data |>
  ggplot(aes(x = question, y = PPQ, fill = difference)) +
  geom_col(
    color = "black",
    linewidth = 0.2,
    show.legend = TRUE
  ) +
  geom_text(
    aes(
      label = scales::percent(PPQ, accuracy = 1),
      color = factor(below_median)
    ),
    vjust = -0.4,
    size = 3,
    show.legend = FALSE
  ) +
  scale_color_manual(
    values = c("0" = "black", "1" = "red"),
    guide = "none"
  ) +
  scale_x_discrete(
    labels = function(x) {
      F23_plot_data$question_label[
        match(x, as.character(F23_plot_data$question))
      ]
    }
  ) +
  scale_y_continuous(
    breaks = seq(0, 1, 0.2),
    expand = expansion(mult = c(0, 0.08))
  ) +
  coord_cartesian(ylim = c(0, 1.08), clip = "off") +
  scale_fill_manual(
    values = c(
      "Same" = "steelblue",
      "Similar" = "orange",
      "insignificant" = "skyblue",
      "Not Included" = "firebrick"
    ),
    drop = FALSE
  ) +
  labs(
    title = "Fall 2023 Midterm: Question Performance",
    subtitle = "Bars are in Fall exam order; labels show Spring question numbers (n = 297 students)",
    x = "Spring question number",
    y = "Proportion Correct (PPQ)",
    fill = "Question Classification",
    caption = "* No Spring counterpart"
  ) +
  theme_minimal() +
  theme(
    axis.text.x = element_text(angle = 45, hjust = 1),
    panel.grid.major.x = element_blank(),
    plot.title = element_text(face = "bold")
  )

F23_plot

Midterms_lst$F23_Midterm |>
  arrange(readr::parse_number(as.character(question))) |>
  select(difference, question, Question_Order)

ggsave(
  filename = "F23_Midterm_PPQ.png",
  plot = F23_plot,
  width = 14,
  height = 6,
  dpi = 300
)

7.1.4 global

library(patchwork)
p <- S24_plot / W24_plot / F23_plot
p

ggsave(
  filename = "Global_Midterm_PPQ.png",
  plot = p,
  width = 14,
  height = 6,
  dpi = 300
)

Ricky Report

Isaiah C. Mireles

2026-10-08

1 `PPQ` and `Midterms_lst` object(s)

1.1 Split `PPQ` into `term, exam` specific data frames

2 Comment :

3 Manual Labels

3.1 Midterm

3.1.1 Spring

3.2 Include `Unique to Spring` label

3.2.1 Winter

3.2.2 Fall

4 Brief Note

5 Refine Data Structure

5.1 Categorical Variables w/ Levels

5.1.1 `Sections` Variable

5.1.2 Investigate `Sections == "Missing"`

5.2 Investigate `Sections == "Missing" AND unidentified == FALSE`

6 Save R objects

7 Graph(s)

7.1 Bar Plots (Relative Ordering)

7.1.1 Spring

7.1.2 Winter

7.1.3 Fall

7.1.4 global

Ricky Report

Isaiah C. Mireles

2026-10-08

1 PPQ and Midterms_lst object(s)

1.1 Split PPQ into term, exam specific data frames

2 Comment :

3 Manual Labels

3.1 Midterm

3.1.1 Spring

3.2 Include Unique to Spring label

3.2.1 Winter

3.2.2 Fall

4 Brief Note

5 Refine Data Structure

5.1 Categorical Variables w/ Levels

5.1.1 Sections Variable

5.1.2 Investigate Sections == "Missing"

5.2 Investigate Sections == "Missing" AND unidentified == FALSE

6 Save R objects

7 Graph(s)

7.1 Bar Plots (Relative Ordering)

7.1.1 Spring

7.1.2 Winter

7.1.3 Fall

7.1.4 global

1 `PPQ` and `Midterms_lst` object(s)

1.1 Split `PPQ` into `term, exam` specific data frames

3.2 Include `Unique to Spring` label

5.1.1 `Sections` Variable

5.1.2 Investigate `Sections == "Missing"`

5.2 Investigate `Sections == "Missing" AND unidentified == FALSE`