Overview

This document runs a basic text analysis and sentiment analysis on four Bluesky post collections (two for each candidate in the California governor’s race) and shows how both change over time. All charts are interactive (Plotly).

Workflow

  1. Read the four CSVs (the *2.csv files contain a few invalid UTF-8 bytes, handled below)
  2. Parse the post text out of the flattened record column
  3. Clean text, detect language, tokenize
  4. Basic text analysis: top words, bigrams, distinctive words, top accounts
  5. Sentiment analysis with the Bing lexicon
  6. Trends over time: volume, sentiment, positive/negative share, term trends

Install once (not run when knitting):

install.packages(c("tidyverse", "tidytext", "plotly", "kableExtra", "cld2", "scales"))
library(tidyverse)
library(lubridate)
library(tidytext)
library(plotly)
library(kableExtra)
library(cld2)      # language detection
library(scales)

Settings

tz_local          <- "C:/Users/zxu3/Documents/R/bluesky"   # time zone used to assign posts to days
election_day      <- as.Date("2026-11-03")   # vertical reference line on trend charts
min_posts_per_day <- 5                       # hide days with fewer posts in trend charts
pal <- c(Becerra = "#2C7BB6", Hilton = "#D7191C")

files <- tribble(
  ~file,              ~candidate, ~batch,
  "post_Xavier.csv",  "Becerra",  "batch 1",
  "post_Xavier2.csv", "Becerra",  "batch 2",
  "post_steve.csv",   "Hilton",   "batch 1",
  "post_steve2.csv",  "Hilton",   "batch 2"
) %>%
  mutate(path = file.path(params$data_dir, file))

# Words the Bing lexicon scores in ways that are misleading for political posts
# (e.g., "trump" is listed as positive, "lead" as positive). Edit freely.
sent_exclude <- c("trump", "like", "lead", "leads", "leading", "right", "well",
                  "work", "top", "won")

# Words that are uninformative here (candidate names, site boilerplate)
custom_stop <- tibble(word = c("steve", "hilton", "xavier", "becerra", "california",
                               "californians", "californian", "governor", "gov",
                               "gubernatorial", "race", "election", "candidate",
                               "bsky", "social", "amp", "nbsp", "publisher", "via"))

Load and combine the data

The CSVs were exported from R with nested fields flattened into text, and the two *2.csv files contain a few invalid UTF-8 bytes. read_bsky() drops the bad bytes before parsing.

read_bsky <- function(path) {
  raw_txt <- rawToChar(readBin(path, "raw", file.size(path)))
  Encoding(raw_txt) <- "UTF-8"
  clean_txt <- iconv(raw_txt, from = "UTF-8", to = "UTF-8", sub = "")  # drop invalid bytes
  tmp <- tempfile(fileext = ".csv")
  writeLines(clean_txt, tmp, useBytes = TRUE)
  read_csv(tmp, col_types = cols(.default = col_character()), show_col_types = FALSE)
}

raw_posts <- files %>%
  mutate(data = map(path, read_bsky)) %>%
  select(-path) %>%
  unnest(data) %>%
  mutate(
    created_utc   = ymd_hms(indexed_at, tz = "UTC"),
    created_local = with_tz(created_utc, tz_local),
    day           = as_date(created_local),
    handle        = str_split_fixed(author, "; ", 4)[, 2],
    across(c(like_count, repost_count, reply_count, quote_count), as.integer)
  )

raw_posts %>%
  group_by(candidate, batch, file) %>%
  summarise(rows = n(), unique_posts = n_distinct(uri),
            first_day = min(day), last_day = max(day), .groups = "drop") %>%
  kbl(caption = "Raw files") %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)
Raw files
candidate batch file rows unique_posts first_day last_day
Becerra batch 1 post_Xavier.csv 500 500 2026-10-15 2026-10-31
Becerra batch 2 post_Xavier2.csv 488 488 2026-10-22 2026-11-03
Hilton batch 1 post_steve.csv 500 500 2026-10-17 2026-10-31
Hilton batch 2 post_steve2.csv 486 486 2026-10-21 2026-11-03

Extract the post text

The record column is a flattened string such as app.bsky.feed.post; <date>; list(<embed>); list(<facets>); en; <post text>. The helper below removes the type, date, nested list(...) blocks and language code, leaving the text. Posts bridged from Mastodon have a different order and are handled separately.

# Remove a leading balanced "list( ... )" block, respecting quoted strings and escapes
strip_list_block <- function(s) {
  pos <- gregexpr('[()"\\\\]', s, perl = TRUE)[[1]]
  if (pos[1] == -1) return(s)
  depth <- 0; in_str <- FALSE; skip_until <- 0
  for (p in pos) {
    if (p <= skip_until) next
    ch <- substr(s, p, p)
    if (ch == "\\") { skip_until <- p + 1; next }
    if (in_str) { if (ch == '"') in_str <- FALSE; next }
    if (ch == '"') { in_str <- TRUE; next }
    if (ch == "(") depth <- depth + 1
    if (ch == ")") {
      depth <- depth - 1
      if (depth == 0) return(substr(s, p + 1, nchar(s)))
    }
  }
  s
}

extract_post_text <- function(rec) {
  if (is.na(rec)) return(NA_character_)
  date_rx <- "^\\d{4}-\\d{2}-\\d{2}T[^;]*?; "
  lang_rx <- "^[a-z]{2,3}(?:-[A-Za-z0-9]+)?(?:; |$)"
  x <- sub("^app\\.bsky\\.feed\\.post; ", "", rec)
  if (grepl(date_rx, x, perl = TRUE)) {
    x <- sub(date_rx, "", x, perl = TRUE)
    for (i in 1:10) {
      if (startsWith(x, "list(")) {
        x <- sub("^; ", "", strip_list_block(x))
      } else if (grepl(lang_rx, x, perl = TRUE)) {
        x <- sub(lang_rx, "", x, perl = TRUE)
      } else break
    }
    x
  } else {  # bridged posts: text; original url; date; ...
    m <- regmatches(x, regexec("(?s)^(.*?); https?://[^;\\s]+; \\d{4}-\\d{2}-\\d{2}T[^;]*?; ",
                               x, perl = TRUE))[[1]]
    if (length(m) > 1) m[2] else x
  }
}

clean_post_text <- function(x) {
  x %>%
    str_replace_all("<[^>]+>", " ") %>%                       # html tags (bridged posts)
    str_replace_all("&amp;", "&") %>%
    str_replace_all("&nbsp;|&quot;|&#39;|&lt;|&gt;", " ") %>%
    str_replace_all("[\u2018\u2019]", "'") %>%                # curly apostrophes
    str_replace_all("@[\\w.-]+", " ") %>%                     # @mentions
    str_replace_all("https?://\\S+|www\\.\\S+", " ") %>%      # urls
    str_replace_all("\\b[\\w-]+(\\.[\\w-]+)*\\.(com|org|net|gov|edu|us|io|co|tv|news|live)\\b\\S*", " ") %>%  # truncated links
    str_replace_all("#", "") %>%                              # keep hashtag words
    str_replace_all("[^\\p{L}\\p{N}\\s'-]", " ") %>%          # emojis / punctuation
    str_squish()
}

Clean, de-duplicate and filter

A post that appears in both candidates’ collections (it mentions both) is kept once per candidate, and flagged with in_both.

posts_0 <- raw_posts %>%
  distinct(candidate, uri, .keep_all = TRUE)            # batch 1 and batch 2 overlap

posts_1 <- posts_0 %>%
  mutate(text_raw = map_chr(record, extract_post_text),
         text     = clean_post_text(text_raw)) %>%
  filter(!is.na(text), nchar(text) > 0)                 # drops image-only / link-only posts

posts_2 <- posts_1 %>%
  mutate(lang = detect_language(text)) %>%
  filter(is.na(lang) | lang == "en")                    # keep English (and undetermined short posts)

posts <- posts_2 %>%
  group_by(uri) %>% mutate(in_both = n_distinct(candidate) > 1) %>% ungroup() %>%
  mutate(pid = paste(candidate, uri, sep = "|"))

tibble(
  step  = c("Raw rows", "Unique posts per candidate", "With usable text", "English / undetermined"),
  posts = c(nrow(raw_posts), nrow(posts_0), nrow(posts_1), nrow(posts_2))
) %>%
  kbl(caption = "Cleaning log") %>%
  kable_styling(bootstrap_options = c("striped", "hover"), full_width = FALSE)
Cleaning log
step posts
Raw rows 1974
Unique posts per candidate 1282
With usable text 1174
English / undetermined 1149
posts %>% select(candidate, day, handle, text) %>% head(10)

Tokenize

tokens_all <- posts %>%
  select(pid, candidate, day, in_both, text) %>%
  unnest_tokens(word, text) %>%
  mutate(word = str_remove(word, "'s$")) %>%
  filter(!str_detect(word, "^\\d+$"), nchar(word) > 1)

tokens <- tokens_all %>%
  anti_join(stop_words, by = "word") %>%
  anti_join(custom_stop, by = "word")

post_len <- tokens_all %>% count(pid, name = "n_words")

tibble(tokens_total = nrow(tokens_all), tokens_after_stopwords = nrow(tokens),
       vocabulary = n_distinct(tokens$word))

Basic text analysis

Post length

posts %>%
  left_join(post_len, by = "pid") %>%
  plot_ly(x = ~n_words, color = ~candidate, colors = pal, type = "histogram",
          opacity = 0.65, nbinsx = 40) %>%
  layout(barmode = "overlay", title = "Words per post",
         xaxis = list(title = "Words (after cleaning)"), yaxis = list(title = "Posts"))

Most frequent words

make_bar <- function(d, color, title) {
  d <- d %>% mutate(term = fct_reorder(term, n))
  plot_ly(d, x = ~n, y = ~term, type = "bar", orientation = "h",
          marker = list(color = color),
          hovertemplate = "%{y}: %{x}<extra></extra>") %>%
    layout(title = title, xaxis = list(title = "Count"), yaxis = list(title = ""),
           margin = list(l = 120))
}

top_words <- tokens %>%
  count(candidate, word, sort = TRUE) %>%
  group_by(candidate) %>% slice_max(n, n = 15, with_ties = FALSE) %>% ungroup() %>%
  rename(term = word)

Becerra

make_bar(filter(top_words, candidate == "Becerra"), pal[["Becerra"]], "Top words: Becerra posts")

Hilton

make_bar(filter(top_words, candidate == "Hilton"), pal[["Hilton"]], "Top words: Hilton posts")

Most frequent bigrams

bigrams <- posts %>%
  select(candidate, text) %>%
  unnest_tokens(bigram, text, token = "ngrams", n = 2) %>%
  filter(!is.na(bigram)) %>%
  separate(bigram, c("w1", "w2"), sep = " ", remove = FALSE) %>%
  filter(!w1 %in% stop_words$word, !w2 %in% stop_words$word,
         !w1 %in% custom_stop$word, !w2 %in% custom_stop$word,
         !str_detect(w1, "^\\d+$"), !str_detect(w2, "^\\d+$"))

top_bigrams <- bigrams %>%
  count(candidate, bigram, sort = TRUE) %>%
  group_by(candidate) %>% slice_max(n, n = 15, with_ties = FALSE) %>% ungroup() %>%
  rename(term = bigram)

Becerra

make_bar(filter(top_bigrams, candidate == "Becerra"), pal[["Becerra"]], "Top bigrams: Becerra posts")

Hilton

make_bar(filter(top_bigrams, candidate == "Hilton"), pal[["Hilton"]], "Top bigrams: Hilton posts")

Distinctive words (log-odds ratio)

Which words are most characteristic of one candidate’s posts versus the other’s? Only posts that appear in one collection are used, and words need at least 10 uses. Positive values lean Becerra; negative values lean Hilton.

word_counts <- tokens %>%
  filter(!in_both) %>%
  count(candidate, word) %>%
  pivot_wider(names_from = candidate, values_from = n, values_fill = 0)

tot_b <- sum(word_counts$Becerra)
tot_h <- sum(word_counts$Hilton)

word_odds <- word_counts %>%
  filter(Becerra + Hilton >= 10) %>%
  mutate(log_ratio = log2(((Becerra + 1) / (tot_b + 1)) / ((Hilton + 1) / (tot_h + 1))))

top_odds <- bind_rows(
  slice_max(word_odds, log_ratio, n = 12, with_ties = FALSE) %>% mutate(favors = "Becerra"),
  slice_min(word_odds, log_ratio, n = 12, with_ties = FALSE) %>% mutate(favors = "Hilton")
) %>% mutate(word = fct_reorder(word, log_ratio))

plot_ly(top_odds, x = ~log_ratio, y = ~word, color = ~favors, colors = pal,
        type = "bar", orientation = "h") %>%
  layout(title = "Most distinctive words", xaxis = list(title = "log2 odds ratio (Becerra / Hilton)"),
         yaxis = list(title = ""), margin = list(l = 110))

Most active accounts

posts %>%
  count(candidate, handle, sort = TRUE) %>%
  group_by(candidate) %>% slice_max(n, n = 8, with_ties = FALSE) %>% ungroup() %>%
  kbl(col.names = c("Collection", "Account", "Posts")) %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)
Collection Account Posts
Becerra andyheppelle.bsky.social 74
Becerra bluegitana.bsky.social 47
Becerra davidsicairos.bsky.social 22
Becerra nypost.com 17
Becerra plaintanjane.bsky.social 13
Becerra bsfavourite.bsky.social 8
Becerra pollresults.org 8
Becerra sfpoliticsbriefly.bsky.social 8
Hilton davidsicairos.bsky.social 22
Hilton christopretensism.bsky.social 16
Hilton judgement-bot.bsky.social 15
Hilton bsfavourite.bsky.social 13
Hilton nypost.com 13
Hilton porcellato.bsky.social 9
Hilton chicagonewsbench.bsky.social 7
Hilton pollresults.org 7

Sentiment analysis

Sentiment is scored with the Bing lexicon (built into tidytext, no download needed). For each post: score = (# positive words) - (# negative words), and the post is labeled Positive (> 0), Negative (< 0) or Neutral / none (= 0). This is a simple word-count method: it does not detect negation or sarcasm.

bing <- get_sentiments("bing") %>%
  distinct(word, .keep_all = TRUE) %>%
  filter(!word %in% sent_exclude)

post_sent <- tokens_all %>%
  inner_join(bing, by = "word") %>%
  count(pid, sentiment) %>%
  pivot_wider(names_from = sentiment, values_from = n, values_fill = 0)

posts <- posts %>%
  left_join(post_sent, by = "pid") %>%
  mutate(
    positive = replace_na(positive, 0L),
    negative = replace_na(negative, 0L),
    score    = positive - negative,
    label    = case_when(score > 0 ~ "Positive", score < 0 ~ "Negative", TRUE ~ "Neutral / none"),
    label    = factor(label, levels = c("Negative", "Neutral / none", "Positive"))
  )

posts %>%
  group_by(candidate) %>%
  summarise(posts = n(),
            mean_score = round(mean(score), 3),
            pct_positive = percent(mean(label == "Positive"), accuracy = 0.1),
            pct_neutral  = percent(mean(label == "Neutral / none"), accuracy = 0.1),
            pct_negative = percent(mean(label == "Negative"), accuracy = 0.1)) %>%
  kbl(caption = "Sentiment summary by candidate") %>%
  kable_styling(bootstrap_options = c("striped", "hover"), full_width = FALSE)
Sentiment summary by candidate
candidate posts mean_score pct_positive pct_neutral pct_negative
Becerra 576 -0.201 19.1% 45.0% 35.9%
Hilton 573 -0.293 20.1% 43.6% 36.3%

A quick significance check on posts that appear in only one collection (so the two groups do not share posts):

excl <- posts %>% filter(!in_both)
wilcox.test(score ~ candidate, data = excl)
## 
##  Wilcoxon rank sum test with continuity correction
## 
## data:  score by candidate
## W = 86266, p-value = 0.7148
## alternative hypothesis: true location shift is not equal to 0

Share of positive / neutral / negative posts

label_cols <- c("Negative" = "#D95F02", "Neutral / none" = "#BDBDBD", "Positive" = "#1B9E77")

posts %>%
  count(candidate, label) %>%
  group_by(candidate) %>% mutate(share = n / sum(n)) %>% ungroup() %>%
  plot_ly(x = ~candidate, y = ~share, color = ~label, colors = label_cols, type = "bar",
          text = ~percent(share, accuracy = 0.1), textposition = "inside",
          hovertemplate = "%{x}, %{fullData.name}: %{text}<extra></extra>") %>%
  layout(barmode = "stack", title = "Post sentiment by candidate",
         yaxis = list(title = "Share of posts", tickformat = ".0%"), xaxis = list(title = ""))

Distribution of post scores

posts %>%
  mutate(score_clip = pmax(pmin(score, 5), -5)) %>%
  plot_ly(x = ~score_clip, color = ~candidate, colors = pal, type = "histogram",
          histnorm = "probability") %>%
  layout(barmode = "group", title = "Distribution of post sentiment scores (clipped at ±5)",
         xaxis = list(title = "Score (positive words - negative words)", dtick = 1),
         yaxis = list(title = "Share of posts", tickformat = ".0%"))

Words driving sentiment

sent_words <- tokens_all %>%
  inner_join(bing, by = "word") %>%
  count(candidate, word, sentiment) %>%
  group_by(candidate, sentiment) %>% slice_max(n, n = 10, with_ties = FALSE) %>% ungroup()

make_sent_bar <- function(cand) {
  d <- sent_words %>%
    filter(candidate == cand) %>%
    mutate(n_signed = if_else(sentiment == "negative", -n, n),
           word = fct_reorder(word, n_signed))
  plot_ly(d, x = ~n_signed, y = ~word, color = ~sentiment,
          colors = c(negative = "#D95F02", positive = "#1B9E77"),
          type = "bar", orientation = "h", text = ~n, hoverinfo = "text+y") %>%
    layout(title = paste("Top sentiment words:", cand),
           xaxis = list(title = "\u2190 negative words | positive words \u2192 (count)"),
           yaxis = list(title = ""), margin = list(l = 110))
}

Becerra

make_sent_bar("Becerra")

Hilton

make_sent_bar("Hilton")

Notes and limitations

  • Sampling: each file holds at most about 500 posts (the most recent matches at the time of collection), so the daily counts depend on when and how the data were pulled. Compare rates and averages across days more than raw volumes.
  • Overlap: batch 1 and batch 2 overlap in time and are de-duplicated by post URI. Posts that mention both candidates appear in both collections (in_both); the head-to-head tests exclude them.
  • Lexicon method: Bing counts positive and negative words but ignores negation, sarcasm and context. Many posts are news headlines or link cards, which tend to score near zero. Review sent_exclude and custom_stop in the Settings section, and consider adding a second lexicon or a transformer model for a robustness check.
  • Language: cld2 removes clearly non-English posts; very short posts with no detected language are kept.
  • Time: timestamps come from indexed_at (when Bluesky indexed the post, in UTC) and are converted to C:/Users/zxu3/Documents/R/bluesky.
sessionInfo()
## R version 4.3.2 (2023-10-31 ucrt)
## Platform: x86_64-w64-mingw32/x64 (64-bit)
## Running under: Windows 11 x64 (build 22631)
## 
## Matrix products: default
## 
## 
## locale:
## [1] LC_COLLATE=English_United States.utf8 
## [2] LC_CTYPE=English_United States.utf8   
## [3] LC_MONETARY=English_United States.utf8
## [4] LC_NUMERIC=C                          
## [5] LC_TIME=English_United States.utf8    
## 
## time zone: America/Los_Angeles
## tzcode source: internal
## 
## attached base packages:
## [1] stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] scales_1.4.0     cld2_1.2.6       kableExtra_1.4.0 plotly_4.12.1   
##  [5] tidytext_0.4.1   lubridate_1.9.5  forcats_1.0.1    stringr_1.6.0   
##  [9] dplyr_1.1.4      purrr_1.2.2      readr_2.2.0      tidyr_1.3.2     
## [13] tibble_3.3.0     ggplot2_4.0.3    tidyverse_2.0.0 
## 
## loaded via a namespace (and not attached):
##  [1] gtable_0.3.6       xfun_0.60          bslib_0.12.0       htmlwidgets_1.6.4 
##  [5] lattice_0.22-9     tzdb_0.5.0         vctrs_0.6.5        tools_4.3.2       
##  [9] crosstalk_1.2.1    generics_0.1.4     parallel_4.3.2     janeaustenr_1.0.0 
## [13] pkgconfig_2.0.3    tokenizers_0.3.0   Matrix_1.6-1.1     data.table_1.17.8 
## [17] RColorBrewer_1.1-3 S7_0.2.2           lifecycle_1.0.5    compiler_4.3.2    
## [21] farver_2.1.2       textshaping_1.0.5  SnowballC_0.7.1    htmltools_0.5.9   
## [25] sass_0.4.9         yaml_2.3.10        pillar_1.11.1      crayon_1.5.3      
## [29] jquerylib_0.1.4    cachem_1.1.0       tidyselect_1.2.1   digest_0.6.39     
## [33] stringi_1.8.9      fastmap_1.2.0      grid_4.3.2         cli_3.6.5         
## [37] magrittr_2.0.4     dichromat_2.0-0.1  withr_3.0.3        bit64_4.8.4       
## [41] timechange_0.4.0   rmarkdown_2.31     httr_1.4.8         bit_4.6.0         
## [45] otel_0.2.0         hms_1.1.4          evaluate_1.0.5     knitr_1.51        
## [49] viridisLite_0.4.3  rlang_1.1.6        Rcpp_1.1.0         glue_1.8.0        
## [53] xml2_1.5.2         svglite_2.2.2      rstudioapi_0.19.0  vroom_1.7.1       
## [57] jsonlite_2.0.0     R6_2.6.1           systemfonts_1.3.2

Jimmy Zhenning Xu, Ph.D. | github.com/utjimmyx