Week 9 NY Times API

# httr sends the request to the API.
# jsonlite changes the JSON response into R objects.
# dplyr, stringr, and tidyr help clean and summarize the data.
# ggplot2 creates the chart, and knitr formats the tables.
library(httr)
library(jsonlite)
library(dplyr)
library(stringr)
library(tidyr)
library(ggplot2)
library(knitr)

# Read the private key from the computer's environment.
nyt_key <- Sys.getenv("NYT_API_KEY")

# This value is used to decide whether the live API code can run.
key_available <- nzchar(nyt_key)

# Record the date only when the live request can be made.
retrieval_date <- if (key_available) Sys.Date() else as.Date(NA)

if (!key_available) {
  message(
    "NYT_API_KEY was not found. Add it to .Renviron, restart R, and render again."
  )
}

nyt_url <- "https://api.nytimes.com/svc/mostpopular/v2/viewed/30.json"

if (key_available) {
  nyt_response <- GET(
    url = nyt_url,
    query = list(`api-key` = nyt_key)
  )

  # Stop here if the request was not successful.
  stop_for_status(nyt_response)

  request_status <- status_code(nyt_response)
} else {
  nyt_response <- NULL
  request_status <- NA_integer_
}

request_status

if (key_available) {
  response_headers <- headers(nyt_response)

  http_summary <- tibble(
    response_part = c("Status", "Header", "Body"),
    value = c(
      paste(request_status, http_status(nyt_response)$message),
      paste("Content-Type:", response_headers[["content-type"]]),
      "JSON article data returned by the API"
    )
  )

  kable(
    http_summary,
    col.names = c("Response part", "What R received"),
    caption = "Three parts of the New York Times API response",
    align = c("l", "l")
  )
} else {
  cat("The response summary will appear after a valid API key is added.\n")
}

if (key_available) {
  response_text <- content(
    nyt_response,
    as = "text",
    encoding = "UTF-8"
  )

  nyt_json <- fromJSON(response_text, flatten = TRUE)

  # The articles are stored inside the results part of the response.
  raw_articles <- as_tibble(nyt_json$results) %>%
    mutate(api_rank = row_number(), .before = 1)

  # Stop with a clear message if the endpoint structure changes.
  required_fields <- c(
    "id", "published_date", "section", "subsection", "title",
    "byline", "abstract", "source", "type", "url"
  )

  missing_fields <- setdiff(required_fields, names(raw_articles))

  if (length(missing_fields) > 0) {
    stop(
      "The API response is missing required fields: ",
      paste(missing_fields, collapse = ", ")
    )
  }
} else {
  raw_articles <- tibble()
}

if (nrow(raw_articles) > 0) {
  cat("The API returned", nrow(raw_articles), "article records.\n")
} else {
  cat("No live records are displayed because NYT_API_KEY is not available.\n")
}

if (nrow(raw_articles) > 0) {
  articles_tidy <- raw_articles %>%
    transmute(
      api_rank,
      article_id = as.character(id),
      published_date = as.Date(published_date),
      section = str_squish(section),
      subsection = str_squish(subsection),
      title = str_squish(title),
      byline = str_squish(str_remove(byline, regex("^By\\s+", ignore_case = TRUE))),
      abstract = str_squish(abstract),
      source,
      article_type = type,
      url
    ) %>%
    mutate(
      section = na_if(section, ""),
      section = replace_na(section, "Not listed"),
      subsection = na_if(subsection, ""),
      byline = na_if(byline, "")
    ) %>%
    distinct(article_id, .keep_all = TRUE)
} else {
  articles_tidy <- tibble()
}

if (nrow(articles_tidy) > 0) {
  articles_tidy %>%
    select(api_rank, published_date, section, title, byline) %>%
    head(10) %>%
    kable(
      col.names = c("Rank", "Published", "Section", "Article title", "Author"),
      caption = "First 10 articles in the cleaned data frame"
    )
} else {
  cat("The cleaned table will appear after a valid API key is added.\n")
}

if (nrow(articles_tidy) > 0) {
  validation_summary <- tibble(
    check = c(
      "Rows returned by the API",
      "Rows in the cleaned data",
      "Duplicate article IDs after cleaning",
      "Missing article titles",
      "Sections recorded as Not listed"
    ),
    result = c(
      nrow(raw_articles),
      nrow(articles_tidy),
      sum(duplicated(articles_tidy$article_id)),
      sum(is.na(articles_tidy$title) | articles_tidy$title == ""),
      sum(articles_tidy$section == "Not listed")
    )
  )
} else {
  validation_summary <- tibble()
}

if (nrow(validation_summary) > 0) {
  kable(
    validation_summary,
    col.names = c("Validation check", "Result"),
    caption = "Checks performed before the analysis",
    align = c("l", "r")
  )
} else {
  cat("The validation table will appear after a valid API key is added.\n")
}

if (nrow(articles_tidy) > 0) {
  section_summary <- articles_tidy %>%
    count(section, name = "article_count", sort = TRUE) %>%
    mutate(
      percentage = round(100 * article_count / sum(article_count), 1)
    )
} else {
  section_summary <- tibble()
}

if (nrow(section_summary) > 0) {
  section_summary %>%
    kable(
      col.names = c("NY Times section", "Number of articles", "Percentage"),
      caption = "Most-viewed articles by section",
      align = c("l", "r", "r")
    )
} else {
  cat("The section summary will appear after a valid API key is added.\n")
}

if (nrow(section_summary) > 0) {
  ggplot(
    section_summary,
    aes(x = reorder(section, article_count), y = article_count)
  ) +
    geom_col(fill = "#0072B2", width = 0.7) +
    geom_text(
      aes(label = article_count),
      hjust = -0.15,
      size = 3.5
    ) +
    coord_flip() +
    scale_y_continuous(
      expand = expansion(mult = c(0, 0.12))
    ) +
    labs(
      title = "Most-Viewed NY Times Articles by Section",
      subtitle = "Articles returned by the 30-day Most Popular endpoint",
      x = "Section",
      y = "Number of articles",
      caption = "Source: New York Times Most Popular API"
    ) +
    theme_minimal(base_size = 12) +
    theme(
      panel.grid.major.y = element_blank(),
      plot.title.position = "plot"
    )
} else {
  cat("The chart will appear after a valid API key is added.\n")
}

if (nrow(section_summary) > 0) {
  top_section <- section_summary %>%
    slice_head(n = 1)

  cat(
    "Based on data retrieved on ", format(retrieval_date, "%B %d, %Y"),
    ", ", top_section$section,
    " appeared most often among the articles returned by the 30-day ",
    "Most Popular endpoint, with ", top_section$article_count,
    " articles representing ", top_section$percentage,
    "% of the results.",
    sep = ""
  )
} else {
  cat("The finding will appear after a valid API key is added and the document is rendered.\n")
}