Pairwise Similarity

# Load libraries
library(tidyverse)
library(text2vec)
library(tm)
library(pheatmap)
library(dendextend)

# Load and clean MAHA report text
maha_text <- read_file("MAHA-Report-text.txt") %>%
  str_to_lower() %>%
  str_replace_all("[^[:alnum:] ]", " ") %>%
  str_squish()

# Load and preprocess articles
articles <- read_csv("C:/Users/ERosa/Github/maha/project-articles.csv") %>%
  mutate(
    title = str_to_lower(title),
    article_types = str_to_lower(replace_na(article_types, "")),
    content_clean = str_squish(str_replace_all(paste(title, article_types), "[^[:alnum:] ]", " "))
  )

Rows: 8113 Columns: 27── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr  (8): url, published_date, author_name, domain_name, thumbnail, title, article_types, author_details
dbl (18): total_shares, linking_domains, pinterest_shares, twitter_shares, total_facebook_shares, evergreen_score, total_reddit_engag...
lgl  (1): author_image
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

# Combine content and tokenize
texts_all <- c(articles$content_clean, maha_text)
it_all <- itoken(texts_all, preprocessor = tolower, tokenizer = word_tokenizer)

# Create vocabulary and DTM
vocab <- create_vocabulary(it_all, stopwords = stopwords("en")) %>%
  prune_vocabulary(term_count_min = 10, doc_proportion_max = 0.3) %>%
  filter(nchar(term) >= 3)
dtm <- create_dtm(it_all, vocab_vectorizer(vocab))

# Split article/report matrices
n_articles <- nrow(articles)
dtm_articles <- dtm[1:n_articles, ]
dtm_maha <- dtm[n_articles + 1, , drop = FALSE]

# Similarity to MAHA
similarity_scores <- sim2(x = dtm_articles, y = dtm_maha, method = "cosine", norm = "l2") %>%
  as.vector()
articles <- articles %>% mutate(similarity_to_maha = similarity_scores)

# Select top N
top_n <- 50
top_articles <- articles %>% arrange(desc(similarity_to_maha)) %>% slice(1:top_n)
top_indices <- which(articles$title %in% top_articles$title)
article_subset_dtm <- dtm_articles[top_indices, ]

# Pairwise similarity and clustering
pairwise_sim <- sim2(article_subset_dtm, method = "cosine", norm = "l2")
dist_matrix <- as.dist(1 - pairwise_sim)
hc <- hclust(dist_matrix, method = "average")
dend <- as.dendrogram(hc) %>% color_branches(k = 4)

# Add custom labels to dendrogram
labels(dend) <- articles$title[top_indices]

# Assign cluster membership
articles$cluster <- NA_integer_
articles$cluster[top_indices] <- cutree(hc, k = 4)

# Plot dendrogram
par(mar = c(10, 4, 4, 2))  # More space for rotated labels
plot(dend, cex = 0.5, las = 2,
     main = "Hierarchical Clustering of Top Articles",
     xlab = "", sub = "", ylab = "Dissimilarity")

# Plot heatmap
pheatmap(
  mat = pairwise_sim,
  clustering_distance_rows = dist_matrix,
  clustering_distance_cols = dist_matrix,
  main = "Pairwise Similarity Between Top Articles",
  fontsize_row = 6,
  fontsize_col = 6,
  legend = TRUE,
  display_numbers = FALSE
)

# Optional: Save result
# write_csv(articles %>% filter(!is.na(cluster)), "top_articles_with_clusters.csv")