Initial Data Analysis and Visualizations

The following visualizations present the preliminary findings from a lexical sentiment analysis of AI-related music headlines published by Billboard and Rolling Stone. The analysis examines the distribution of retrieved headlines, frequently occurring words, and positive and negative language identified using the Bing sentiment lexicon.

1. Distribution of Headlines by Publications

This bar chart illustrates the distribution of AI-related music headlines retrieved from Billboard and Rolling Stone through MediaCloud. Of the 522 headlines collected, 406 were published by Billboard and 116 by Rolling Stone. The difference in sample sizes is important to consider when comparing sentiment patterns between the two publications.

2. Most Frequently Occurring Headline Words

This visualization displays the ten most frequently occurring words in headlines from each publication after removing common English stop words. The chart provides an overview of recurring vocabulary associated with AI-related music coverage and allows for an initial comparison of word usage between Billboard and Rolling Stone.

3. Comparison of Positive and Negative Sentiment

This chart compares the proportions of positive and negative words identified in each publication’s headlines using the Bing sentiment lexicon. Rather than comparing raw word counts, the visualization presents percentages to account for differences in the number of headlines collected. These results reflect the sentiment of individual words rather than the overall tone or position of an article toward artificial intelligence.

4. Most Frequently Occurring Sentiment Words

This visualization identifies the five most frequently occurring positive and negative words in headlines from each publication, as classified by the Bing sentiment lexicon. By examining the specific words contributing to the sentiment analysis, the chart provides additional context for understanding the differences and similarities in lexical sentiment between Billboard and Rolling Stone.

Code

# ==================================================
# Step 0: Save Your Media Cloud API Key (Run Once) ----
# ==================================================

# key <- "PASTE_YOUR_MEDIACLOUD_API_KEY_BETWEEN_THESE_QUOTES"
#
# write(
#   paste0("MEDIACLOUD_KEY=", key),
#   file = "~/.Renviron",
#   append = TRUE
# )
#
# readRenviron("~/.Renviron")
# nzchar(Sys.getenv("MEDIACLOUD_KEY"))

# ==================================================
# Step 1: Install and Load Required Packages ----
# ==================================================
required_packages <- c(
  "tidyverse",
  "httr2",
  "lubridate",
  "plotly"
)

for (pkg in required_packages) {
  if (!requireNamespace(pkg, quietly = TRUE)) {
    install.packages(pkg)
  }
}

library(tidyverse)
library(httr2)
library(lubridate)
library(plotly)

music_ai_data <- readRDS("music_ai_headlines_2023_2026.rds")

# Prepare publication names
music_ai_data <- music_ai_data |>
  mutate(
    publication = recode(
      media_name,
      "billboard.com" = "Billboard",
      "rollingstone.com" = "Rolling Stone"
    )
  )

# Visual 1: Headline counts
headline_counts <- music_ai_data |>
  count(publication)

visual_1 <- ggplot(
  headline_counts,
  aes(x = publication, y = n, fill = publication)
) +
  geom_col(show.legend = FALSE) +
  geom_text(aes(label = n), vjust = -0.4) +
  labs(
    title = "AI-Related Music Headlines by Publication",
    x = "Publication",
    y = "Number of Headlines"
  ) +
  theme_minimal()

# ==================================================
# Step 2: Load Your Media Cloud API Key ----
# ==================================================

mc_key <- Sys.getenv("MEDIACLOUD_KEY")

# Optional check: Should return TRUE.
nzchar(mc_key)

# ==================================================
# Step 3: Define the Date Range ----
# ==================================================

start_date <- "2025-01-01"
end_date <- "2026-06-30"

# ==================================================
# Step 4: Define Your Search Query ----
# ==================================================

query <-'("artificial intelligence" OR "generative AI" OR "AI-generated") AND music'

# ==================================================
# Step 5: Create a Function to Retrieve Stories 
# ==================================================
mc_story_list <- function(
    query,
    start_date,
    end_date,
    collection_id = 34412234,
    checkpoint_file = "music_ai_20205_2026_backups.rds"
) {
  
  all_stories <- list()
  token <- NULL
  page_number <- 1
  
  # Resume from previous checkpoint if one exists
  if (file.exists(checkpoint_file)) {
    checkpoint <- readRDS(checkpoint_file)
    
    all_stories <- checkpoint$stories
    token <- checkpoint$token
    page_number <- checkpoint$page_number
    
    message("Resuming from page ", page_number)
  }
  
  repeat {
    
    message("Downloading page ", page_number)
    
    req <-
      request("https://search.mediacloud.org/api/search/story-list") |>
      req_headers(
        Authorization = paste("Token", mc_key)
      )
    
    if (is.null(token)) {
      
      req <-
        req |>
        req_url_query(
          q = query,
          start = start_date,
          end = end_date,
          platform = "onlinenews-mediacloud",
          cs = collection_id
        )
      
    } else {
      
      req <-
        req |>
        req_url_query(
          q = query,
          start = start_date,
          end = end_date,
          platform = "onlinenews-mediacloud",
          cs = collection_id,
          pagination_token = token
        )
    }
    
    success <- FALSE
    attempt_number <- 1
    
    while (!success) {
      
      message("Attempt ", attempt_number)
      
      attempt <-
        tryCatch(
          {
            req |>
              req_perform() |>
              resp_body_json()
          },
          
          error = function(e) {
            
            if (grepl("429", e$message)) {
              
              message(
                "Rate limit reached. Waiting 60 seconds..."
              )
              
              Sys.sleep(60)
              return(NULL)
            }
            
            if (grepl("502|503|504", e$message)) {
              
              wait_time <-
                min(
                  300,
                  attempt_number * 60
                )
              
              message(
                "Server timeout/error. Waiting ",
                wait_time,
                " seconds and retrying..."
              )
              
              Sys.sleep(wait_time)
              return(NULL)
            }
            
            stop(e)
          }
        )
      
      if (!is.null(attempt)) {
        
        results <- attempt
        success <- TRUE
        
      } else {
        
        attempt_number <- attempt_number + 1
        
        if (attempt_number > 10) {
          stop("Maximum retry attempts exceeded.")
        }
      }
    }
    
    all_stories <-
      append(
        all_stories,
        results$stories
      )
    
    message(
      "Total stories collected: ",
      length(all_stories)
    )
    
    token <- results$pagination_token
    
    # Save checkpoint after every page
    saveRDS(
      list(
        stories = all_stories,
        token = token,
        page_number = page_number + 1
      ),
      file = checkpoint_file
    )
    
    message(
      "Checkpoint saved after page ",
      page_number
    )
    
    if (is.null(token)) {
      message("Download complete.")
      break
    }
    
    page_number <- page_number + 1
    Sys.sleep(30)
  }
  
  story_data <-
    map_dfr(
      all_stories,
      as_tibble
    ) |>
    mutate(
      publish_date = as.Date(publish_date),
      indexed_date = ymd_hms(indexed_date)
    ) |>
    arrange(publish_date)
  
  # Save final dataset
  saveRDS(
    story_data,
    file = "music_ai.rds"
  )
  
  # Remove checkpoint after successful completion
  if (file.exists(checkpoint_file)) {
    file.remove(checkpoint_file)
  }
  
  return(story_data)
}

# ==================================================
# Step 6: Download Matching Stories 
# ==================================================

story_data <-
  mc_story_list(
    query = query,
    start_date = start_date,
    end_date = end_date
  )

# ==================================================
# Step 7: Check the Results 
# ==================================================

glimpse(story_data)

# ==================================================
# Step 8: Eliminate Any Duplicates 
# ==================================================
message(
  "Unique story IDs: ",
  n_distinct(story_data$id)
)

story_data <-
  story_data |>
  distinct(id, .keep_all = TRUE)

# ==================================================
# Step 9: Filter for English-Language Articles 
# ==================================================
music_ai_data <-
  story_data |>
  filter(
    media_name %in% c(
      "billboard.com",
      "rollingstone.com"
    )
  )

# Check the number of stories from each publication

music_ai_data |>
  count(media_name)
# ==================================================
# Step 10: Save Stories as an RDS File 
# ==================================================

saveRDS(
  music_ai_data,
  file = "music_ai_headlines_2025_2026.rds"
)

# ==================================================
# Step 11: Display Summary Information
# ==================================================

cat(
  "\nStories after filtering:",
  nrow(story_data),
  "\n"
)

cat(
  "Date range:",
  min(story_data$publish_date, na.rm = TRUE),
  "to",
  max(story_data$publish_date, na.rm = TRUE),
  "\n"
)

# Confirm dataset dimensions
dim(music_ai_data)

# Confirm publication counts
music_ai_data |>
  count(media_name)

# Check date range
range(music_ai_data$publish_date, na.rm = TRUE)

# Check categorical values
unique(music_ai_data$language)
unique(music_ai_data$media_name)
unique(music_ai_data$media_url)

# Check for missing values
colSums(is.na(music_ai_data))

# Check for duplicate story IDs
nrow(music_ai_data) - n_distinct(music_ai_data$id)

# ================================================== 
# Step 12: Prepare Headlines for Text Analysis
# ================================================== 
library(tidytext)

# Add readable publication names
music_ai_data <- music_ai_data |>
  mutate(
    publication = recode(
      media_name,
      "billboard.com" = "Billboard",
      "rollingstone.com" = "Rolling Stone"
    )
  )

# Separate headlines into individual words
headline_words <- music_ai_data |>
  select(id, publication, title) |>
  unnest_tokens(word, title) |>
  anti_join(stop_words, by = "word")

# Check the results
glimpse(headline_words)

# ================================================== 
# Step 13: Visualize Headline Counts ----
# ================================================== 

headline_counts <- music_ai_data |>
  count(publication)

ggplot(
  headline_counts,
  aes(x = publication, y = n, fill = publication)
) +
  geom_col(show.legend = FALSE) +
  geom_text(aes(label = n), vjust = -0.4) +
  labs(
    title = "AI-Related Music Headlines by Publication",
    x = "Publication",
    y = "Number of Headlines"
  ) +
  theme_minimal()

# ================================================== 
# Step 14: Visualize Most Frequent Words 
# ================================================== 

top_words <- headline_words |>
  count(publication, word) |>
  group_by(publication) |>
  slice_max(n, n = 10, with_ties = FALSE) |>
  ungroup()

ggplot(
  top_words,
  aes(
    x = tidytext::reorder_within(word, n, publication),
    y = n
  )
) +
  geom_col() +
  tidytext::scale_x_reordered() +
  coord_flip() +
  facet_wrap(~ publication, scales = "free_y") +
  labs(
    title = "Most Frequent Headline Words",
    x = "Word",
    y = "Frequency"
  ) +
  theme_minimal()
# ================================================== 
# Step 15: Analyze and Visualize Sentiment 
# ================================================== 

bing_words <- headline_words |>
  inner_join(
    get_sentiments("bing"),
    by = "word"
  )

sentiment_summary <- bing_words |>
  count(publication, sentiment) |>
  group_by(publication) |>
  mutate(
    proportion = n / sum(n)
  ) |>
  ungroup()

# Display sentiment results
print(sentiment_summary)

# Create sentiment chart
ggplot(
  sentiment_summary,
  aes(
    x = publication,
    y = proportion,
    fill = sentiment
  )
) +
  geom_col() +
  scale_y_continuous(labels = scales::percent) +
  labs(
    title = "Positive vs. Negative Sentiment Words",
    x = "Publication",
    y = "Percentage of Sentiment-Matched Words",
    fill = "Sentiment"
  ) +
  theme_minimal()

# ================================================== 
# Step 16: Visualize Common Sentiment Words ----
# ================================================== 

top_sentiment_words <- bing_words |>
  count(publication, sentiment, word) |>
  group_by(publication, sentiment) |>
  slice_max(n, n = 5, with_ties = FALSE) |>
  ungroup()

ggplot(
  top_sentiment_words,
  aes(
    x = tidytext::reorder_within(
      word,
      n,
      interaction(publication, sentiment)
    ),
    y = n,
    fill = sentiment
  )
) +
  geom_col(show.legend = FALSE) +
  tidytext::scale_x_reordered() +
  coord_flip() +
  facet_grid(
    sentiment ~ publication,
    scales = "free_y",
    space = "free_y"
  ) +
  labs(
    title = "Most Common Positive and Negative Words",
    x = "Word",
    y = "Frequency"
  ) +
  theme_minimal()