# Overview of candidates
pilot_candidates <- pilot_call1 %>%
  filter(!is.na(candidates_json)) %>%
  mutate(candidates = map(candidates_json, function(j) {
    parsed <- fromJSON(j, simplifyVector = FALSE)
    if (length(parsed) == 0) return(tibble(cand_name = character(), cand_code = character()))
    map_dfr(parsed, ~tibble(cand_name = .x$cand_name, cand_code = .x$cand_code))
  })) %>%
  unnest(candidates) %>%
  select(folder, cand_name, cand_code)

candidate_summary <- pilot_candidates %>%
  count(cand_name, sort = TRUE) %>%
  rename(Candidate = cand_name, `Mailers Mentioning` = n)

# Known candidate-party mapping - this is for the imputed data in "Archive", which I didn't end up using. 
known_parties <- tribble(
  ~cand_name_clean, ~cand_party,
  "donald trump",   "R",
  "kamala harris",  "D",
  "tim sheehy",     "R",
  "jon tester",     "D",
  "swanson",        "R",
  "mccormick",      "R",
  "hovde",          "R",
  "tammy baldwin",  "D",
  "bob casey",      "D",
  "j.d. vance",     "R",
  "tim walz",       "D"
)

# Match candidate name to known party
match_party <- function(name) {
  name_lower <- str_to_lower(name)
  for (i in seq_len(nrow(known_parties))) {
    if (str_detect(name_lower, fixed(known_parties$cand_name_clean[i]))) {
      return(known_parties$cand_party[i])
    }
  }
  NA_character_
}

# Build folder-level lookup: supp_party from any Candidate A*, opp_party from any Candidate B*
folder_parties <- pilot_candidates %>%
  filter(folder != "114721283581") %>%
  mutate(cand_party = map_chr(cand_name, match_party)) %>%
  filter(!is.na(cand_party)) %>%
  mutate(role = case_when(
    str_detect(cand_code, "^Candidate A") ~ "supp_party",
    str_detect(cand_code, "^Candidate B") ~ "opp_party"
  )) %>%
  filter(!is.na(role)) %>%
  distinct(folder, role, cand_party) %>%
  pivot_wider(names_from = role, values_from = cand_party, values_fn = first)

# Ensure folder_parties has both columns even if pivot_wider didn't produce them
if (!"supp_party" %in% names(folder_parties)) folder_parties$supp_party <- NA_character_
if (!"opp_party" %in% names(folder_parties)) folder_parties$opp_party <- NA_character_

# Add party and issue-role columns directly to pilot_all
pilot_all <- pilot_all %>%
  left_join(folder_parties, by = "folder") %>%
  mutate(
    supp_party = replace_na(supp_party, "Unknown"),
    opp_party = replace_na(opp_party, "Unknown"),
    # Infer supp_party from opp_party (and vice versa) when one is Unknown
    supp_party = case_when(
      supp_party == "Unknown" & opp_party == "D" ~ "R",
      supp_party == "Unknown" & opp_party == "R" ~ "D",
      TRUE ~ supp_party
    ),
    opp_party = case_when(
      opp_party == "Unknown" & supp_party == "D" ~ "R",
      opp_party == "Unknown" & supp_party == "R" ~ "D",
      TRUE ~ opp_party
    ),
    is_supp_issue = str_detect(candidate_code, "^Candidate A"),
    is_opp_issue = str_detect(candidate_code, "^Candidate B")
  )

# Recreate pilot_merged now that pilot_all has party/issue-role columns
pilot_merged <- mintt_df %>%
  inner_join(pilot_all, by = "folder") %>%
  mutate(mintt_party = case_when(
    mintt_partisanship == "Republican/Conservative" ~ "R",
    mintt_partisanship == "Democratic/Progressive" ~ "D",
    TRUE ~ "Other"
  ))
mintt_tbl <- table(pilot_merged$mintt_party, useNA = "ifany")

tibble(
  Metric = c("pilot_all rows",
             "Pilot + mintt_df rows",
             paste0("mintt_party: ", names(mintt_tbl))),
  Value = c(as.character(nrow(pilot_all)),
            as.character(nrow(pilot_merged)),
            as.character(mintt_tbl))
) %>% kable()

mintt_text <- mintt_df %>% distinct(folder, original_text = text)

pilot_all_display <- pilot_all %>%
  left_join(mintt_text, by = "folder") %>%
  select(-anon_text, -reasoning_c1, -reasoning_c2, -candidates_json) %>%
  mutate(across(where(is.character), ~str_trunc(., 150)))

issue_text_idx <- which(colnames(pilot_all_display) == "issue_text") - 1

datatable(pilot_all_display,
          caption = "Full pilot_all dataset",
          options = list(
            pageLength = 5,
            columnDefs = list(
              list(width = '500px', targets = issue_text_idx)
            )
          ))

# Get mintt_partisanship party per folder (one row per folder, no primary candidate filter needed)
mintt_party_lookup <- pilot_merged %>%
  distinct(folder, mintt_party)

# Compare with our supp_party (one row per folder)
party_comparison <- pilot_all %>%
  distinct(folder, supp_party) %>%
  inner_join(mintt_party_lookup, by = "folder") %>%
  filter(supp_party != "Unknown", mintt_party %in% c("R", "D")) %>%
  mutate(match = supp_party == mintt_party)

tibble(
  Metric = c("Folders with both party sources", "Match", "Mismatch", "Mismatch rate"),
  Value = c(as.character(nrow(party_comparison)), as.character(sum(party_comparison$match)),
            as.character(sum(!party_comparison$match)),
            paste0(round(mean(!party_comparison$match) * 100, 1), "%"))
) %>% kable()

# Shared ridge plot helper: maps -99 → -1, 99 → 0 for x-axis positioning
prep_ideology <- function(data, col = "policy_ideology") {
  data %>%
    mutate(ideology_numeric = case_when(
      .data[[col]] == -99 ~ -1,
      .data[[col]] == 99  ~  0,
      TRUE ~ as.numeric(.data[[col]])
    ))
}

ridge_x_scale <- scale_x_continuous(
  breaks = c(-1, 0, 1:9),
  labels = c("Bad\n(-99)", "Good\n(99)", as.character(1:9))
)

make_ridge_plot_merged <- function(data, party_label, color, ideology_col = "policy_ideology", ideology_label = "Policy") {
  opposite <- ifelse(party_label == "R", "D", ifelse(party_label == "D", "R", NA))

  # Own party's messaging about their candidates (A* issues from own mailers)
  own_data <- data %>%
    filter(mintt_party == party_label, is_supp_issue,
           !is.na(issue), !is.na(.data[[ideology_col]])) %>%
    mutate(source = paste0(party_label, " about own candidates"))

  # Opposing party's messaging about this party's candidates (B* issues from opponent mailers)
  opp_data <- if (!is.na(opposite)) {
    data %>%
      filter(mintt_party == opposite, is_opp_issue,
             !is.na(issue), !is.na(.data[[ideology_col]])) %>%
      mutate(source = paste0(opposite, " about ", party_label, " candidates"))
  } else {
    tibble()
  }

  # Top 10 issues from own messaging + top 5 from opposing (union to avoid duplicates)
  top_own <- own_data %>% count(issue, sort = TRUE) %>% slice_head(n = 10) %>% pull(issue)
  top_opp <- if (nrow(opp_data) > 0) {
    opp_data %>% count(issue, sort = TRUE) %>% slice_head(n = 5) %>% pull(issue)
  } else {
    character()
  }
  top_iss <- unique(c(top_own, top_opp))

  plot_data <- bind_rows(own_data, opp_data) %>%
    filter(issue %in% top_iss) %>%
    prep_ideology(col = ideology_col) %>%
    mutate(issue = factor(issue, levels = rev(top_iss)))

  # Per-issue, per-source means for vertical reference lines (excluding valence)
  means_df <- plot_data %>%
    filter(ideology_numeric >= 1, ideology_numeric <= 9) %>%
    group_by(issue, source) %>%
    summarise(mean_val = mean(ideology_numeric, na.rm = TRUE), .groups = "drop")

  opp_color <- if (!is.na(opposite)) {
    c("D" = "dodgerblue", "R" = "firebrick1")[[opposite]]
  } else {
    "grey50"
  }

  fill_values <- setNames(c(color, "transparent"),
                           c(unique(own_data$source), unique(opp_data$source)))
  color_values <- setNames(c(color, opp_color),
                            c(unique(own_data$source), unique(opp_data$source)))
  linetype_values <- setNames(c("solid", "dashed"),
                               c(unique(own_data$source), unique(opp_data$source)))

  ggplot(plot_data, aes(x = ideology_numeric, y = issue,
                        fill = source, color = source, linetype = source)) +
    geom_density_ridges(alpha = 0.4, bandwidth = 0.6) +
    geom_segment(data = means_df,
      aes(x = mean_val, xend = mean_val,
          y = as.numeric(issue) - 0.1, yend = as.numeric(issue) + 0.5,
          color = source, linetype = source),
      linewidth = 0.4, alpha = 0.7, inherit.aes = FALSE, show.legend = FALSE) +
    scale_fill_manual(values = fill_values) +
    scale_color_manual(values = color_values) +
    scale_linetype_manual(values = linetype_values) +
    ridge_x_scale +
    labs(
      title = paste0(ideology_label, " Ideology — ", party_label, " Candidates (Mintt data)"),
      subtitle = paste0("Solid = ", party_label, "'s own mailers | Dashed = opposing party's mailers\nVertical lines = mean score"),
      x = paste0(ideology_label, " Ideology (-99 = bad, 99 = good; 1 = most liberal, 9 = most conservative)"),
      y = NULL,
      fill = NULL, color = NULL, linetype = NULL
    ) +
    theme_minimal() +
    theme(legend.position = "bottom")
}

fmt_ms <- function(m, s) {
  case_when(
    is.nan(m) | is.na(m) ~ "—",
    is.na(s) ~ as.character(round(m, 2)),
    TRUE ~ paste0(round(m, 2), " (", round(s, 2), ")")
  )
}

# Deduplicate to one row per folder-issue (mintt_party is folder-level, not candidate-level)
merged_dedup <- pilot_merged %>%
  distinct(folder, issue, candidate_code, .keep_all = TRUE) %>%
  filter(!is.na(issue))

total_rows_merged <- nrow(merged_dedup)
total_pilot_folders <- n_distinct(pilot_merged$folder)

summary_tbl_merged <- merged_dedup %>%
  group_by(issue) %>%
  summarise(
    Count = n(),
    Pct_Mailers = round(n_distinct(folder) / total_pilot_folders * 100, 1),
    Pct_Total = round(n() / total_rows_merged * 100, 1),
    R_self_m = mean(policy_ideology[mintt_party == "R" & is_supp_issue &
                        policy_ideology >= 1 & policy_ideology <= 9], na.rm = TRUE),
    R_self_s = sd(policy_ideology[mintt_party == "R" & is_supp_issue &
                        policy_ideology >= 1 & policy_ideology <= 9], na.rm = TRUE),
    D_self_m = mean(policy_ideology[mintt_party == "D" & is_supp_issue &
                        policy_ideology >= 1 & policy_ideology <= 9], na.rm = TRUE),
    D_self_s = sd(policy_ideology[mintt_party == "D" & is_supp_issue &
                        policy_ideology >= 1 & policy_ideology <= 9], na.rm = TRUE),
    R_other_m = mean(policy_ideology[mintt_party == "D" & is_opp_issue &
                        policy_ideology >= 1 & policy_ideology <= 9], na.rm = TRUE),
    R_other_s = sd(policy_ideology[mintt_party == "D" & is_opp_issue &
                        policy_ideology >= 1 & policy_ideology <= 9], na.rm = TRUE),
    D_other_m = mean(policy_ideology[mintt_party == "R" & is_opp_issue &
                        policy_ideology >= 1 & policy_ideology <= 9], na.rm = TRUE),
    D_other_s = sd(policy_ideology[mintt_party == "R" & is_opp_issue &
                        policy_ideology >= 1 & policy_ideology <= 9], na.rm = TRUE),
    Pct_Ideological = round(sum(policy_ideology >= 1 & policy_ideology <= 9, na.rm = TRUE) /
                              sum(!is.na(policy_ideology)) * 100, 1),
    R_Pct_Valence = round(sum(mintt_party == "R" & policy_ideology %in% c(-99, 99), na.rm = TRUE) /
                            max(sum(mintt_party == "R" & !is.na(policy_ideology)), 1) * 100, 1),
    D_Pct_Valence = round(sum(mintt_party == "D" & policy_ideology %in% c(-99, 99), na.rm = TRUE) /
                            max(sum(mintt_party == "D" & !is.na(policy_ideology)), 1) * 100, 1),
    .groups = "drop"
  ) %>%
  mutate(
    R_self = fmt_ms(R_self_m, R_self_s),
    D_self = fmt_ms(D_self_m, D_self_s),
    R_other = fmt_ms(R_other_m, R_other_s),
    D_other = fmt_ms(D_other_m, D_other_s)
  ) %>%
  select(issue, Count, Pct_Mailers, Pct_Total, R_self, D_self, R_other, D_other,
         Pct_Ideological, R_Pct_Valence, D_Pct_Valence) %>%
  arrange(desc(Count))

datatable(summary_tbl_merged,
          colnames = c("Issue", "Count", "% Mailers", "% of Total",
                       "R Self Avg (SD)", "D Self Avg (SD)",
                       "R Other Avg (SD)", "D Other Avg (SD)",
                       "% Ideological (1-9)", "R % Valence", "D % Valence"),
          caption = "Policy Ideology — Issue Summary by mintt_partisanship",
          options = list(pageLength = 15))

summary_tbl_framing <- merged_dedup %>%
  group_by(issue) %>%
  summarise(
    Count = n(),
    Pct_Mailers = round(n_distinct(folder) / total_pilot_folders * 100, 1),
    Pct_Total = round(n() / total_rows_merged * 100, 1),
    R_self_m = mean(framing_ideology[mintt_party == "R" & is_supp_issue &
                        framing_ideology >= 1 & framing_ideology <= 9], na.rm = TRUE),
    R_self_s = sd(framing_ideology[mintt_party == "R" & is_supp_issue &
                        framing_ideology >= 1 & framing_ideology <= 9], na.rm = TRUE),
    D_self_m = mean(framing_ideology[mintt_party == "D" & is_supp_issue &
                        framing_ideology >= 1 & framing_ideology <= 9], na.rm = TRUE),
    D_self_s = sd(framing_ideology[mintt_party == "D" & is_supp_issue &
                        framing_ideology >= 1 & framing_ideology <= 9], na.rm = TRUE),
    R_other_m = mean(framing_ideology[mintt_party == "D" & is_opp_issue &
                        framing_ideology >= 1 & framing_ideology <= 9], na.rm = TRUE),
    R_other_s = sd(framing_ideology[mintt_party == "D" & is_opp_issue &
                        framing_ideology >= 1 & framing_ideology <= 9], na.rm = TRUE),
    D_other_m = mean(framing_ideology[mintt_party == "R" & is_opp_issue &
                        framing_ideology >= 1 & framing_ideology <= 9], na.rm = TRUE),
    D_other_s = sd(framing_ideology[mintt_party == "R" & is_opp_issue &
                        framing_ideology >= 1 & framing_ideology <= 9], na.rm = TRUE),
    Pct_Ideological = round(sum(framing_ideology >= 1 & framing_ideology <= 9, na.rm = TRUE) /
                              sum(!is.na(framing_ideology)) * 100, 1),
    R_Pct_Valence = round(sum(mintt_party == "R" & framing_ideology %in% c(-99, 99), na.rm = TRUE) /
                            max(sum(mintt_party == "R" & !is.na(framing_ideology)), 1) * 100, 1),
    D_Pct_Valence = round(sum(mintt_party == "D" & framing_ideology %in% c(-99, 99), na.rm = TRUE) /
                            max(sum(mintt_party == "D" & !is.na(framing_ideology)), 1) * 100, 1),
    .groups = "drop"
  ) %>%
  mutate(
    R_self = fmt_ms(R_self_m, R_self_s),
    D_self = fmt_ms(D_self_m, D_self_s),
    R_other = fmt_ms(R_other_m, R_other_s),
    D_other = fmt_ms(D_other_m, D_other_s)
  ) %>%
  select(issue, Count, Pct_Mailers, Pct_Total, R_self, D_self, R_other, D_other,
         Pct_Ideological, R_Pct_Valence, D_Pct_Valence) %>%
  arrange(desc(Count))

datatable(summary_tbl_framing,
          colnames = c("Issue", "Count", "% Mailers", "% of Total",
                       "R Self Avg (SD)", "D Self Avg (SD)",
                       "R Other Avg (SD)", "D Other Avg (SD)",
                       "% Ideological (1-9)", "R % Valence", "D % Valence"),
          caption = "Framing Ideology — Issue Summary by mintt_partisanship",
          options = list(pageLength = 15))