fetchJson <- function(url, query = list()) {
resp <- httr::GET(url, query = query, httr::user_agent("web-apis-assignment/1.0"))
httr::stop_for_status(resp)
jsonlite::fromJSON(httr::content(resp, as = "text", encoding = "UTF-8"), flatten = TRUE)
}
pick_col <- function(df, candidates, fill = NA_character_) {
for (cn in candidates) if (cn %in% names(df)) return(df[[cn]])
if (is.na(fill)) rep(NA, nrow(df)) else rep(fill, nrow(df))
}
# ---- NYT Article Search ----
fetchArticlesPage <- function(queryString, beginDate = NULL, endDate = NULL, page = 0) {
baseUrl <- "https://api.nytimes.com/svc/search/v2/articlesearch.json"
fetchJson(
url = baseUrl,
query = list(
q = queryString,
begin_date = beginDate,
end_date = endDate,
`api-key` = apiKey, # <-- pass your key here
page = page
)
)
}
# ---- Run: fetch, tidy, display ----
res <- fetchArticlesPage("climate", beginDate = "20200101", endDate = format(Sys.Date(), "%Y%m%d"), page = 0)
docs <- res$response$docs
stopifnot(!is.null(docs), length(docs) > 0)
df <- as.data.frame(docs, stringsAsFactors = FALSE)
articlesDF <- data.frame(
headline = pick_col(df, c("headline.main", "title", "name")),
pubDate = as.POSIXct(pick_col(df, c("pub_date", "published_date", "date")), tz = "UTC"),
sectionName = pick_col(df, c("section_name", "section", "subsection", "category")),
byline = pick_col(df, c("byline.original", "byline")),
abstract = pick_col(df, c("abstract", "description", "summary")),
webUrl = pick_col(df, c("web_url", "url", "link")),
stringsAsFactors = FALSE
)
kwList <- if ("keywords" %in% names(df)) {
lapply(df$keywords, function(x) {
if (is.null(x)) character()
else if (is.data.frame(x) && "value" %in% names(x)) x$value
else if (is.list(x) && length(x) > 0 && all(vapply(x, is.character, logical(1)))) unlist(x)
else if (is.character(x)) x
else character()
})
} else replicate(nrow(df), character(), simplify = FALSE)
keywordsDF <- tidyr::unnest_longer(
data = data.frame(webUrl = articlesDF$webUrl, keywords = I(kwList)),
col = keywords,
values_to = "keyword",
indices_include = FALSE
)
# Display in knit output
print(utils::head(articlesDF, 8))
## headline
## 1 Exxon Sues California Over New Climate Disclosure Laws
## 2 An E.P.A. Plan to Kill a Major Climate Rule Is Worrying Business Leaders
## 3 What Happens When a Hurricane Like Melissa Doesn’t Move
## 4 How Long Will it Take to Build a Nuclear Power ‘Renaissance’ in the U.S.?
## 5 Trump Opens Pristine Alaska Wilderness to Drilling in Long-Running Feud
## 6 New Yorkers on Their Hopes for the Future of the City
## 7 Heat Has Essentially Wiped Out 2 Key Coral Species on Florida Reefs
## 8 She Made Sure That Tsunami Warnings Reached the Public
## pubDate sectionName byline
## 1 2025-10-25 Climate By Karen Zraick
## 2 2025-10-25 Climate By Karen Zraick and Lisa Friedman
## 3 2025-10-24 Weather By Amy Graff
## 4 2025-10-23 Climate By Claire Brown
## 5 2025-10-23 Climate By Maxine Joselow
## 6 2025-10-23 New York
## 7 2025-10-23 Climate By Catrin Einhorn
## 8 2025-10-23 Climate By Rebecca Dzombak
## abstract
## 1 The oil giant said requirements that companies calculate new details about greenhouse gas emissions and climate risks violate Exxon’s free speech rights.
## 2 Some carmakers and energy executives say the plan would trigger costly litigation and spur individual states to create a patchwork of tighter rules.
## 3 Melissa is moving at a sluggish pace and could dump a lot of rain in one place. Jamaica and Haiti are likely to see the most.
## 4 The Trump administration wants to sharply speed up the construction of nuclear power plants, but fixing the industry’s bottlenecks could take years.
## 5 The Interior Department also said it would allow a contentious road to be built through the Izembek National Wildlife Refuge in southwestern Alaska.
## 6 Readers share their wishes, predictions and wildest dreams for what’s to come in transit, housing, arts and more.
## 7 Elkhorn and staghorn coral are now functionally extinct around the state, researchers say, meaning they no longer play any significant role in their ecosystem.
## 8 “I always wanted to be a public servant and do science for the good of the people,” said Corina Allen, who lost her job at NOAA.
## webUrl
## 1 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-free-speech.html
## 2 https://www.nytimes.com/2025/10/25/climate/endangerment-finding-auto-energy-lawsuits.html
## 3 https://www.nytimes.com/2025/10/24/weather/hurricane-melissa-slow-forecast-haiti-jamaica-rain.html
## 4 https://www.nytimes.com/2025/10/23/climate/us-nuclear-industry-growth.html
## 5 https://www.nytimes.com/2025/10/23/climate/trump-arctic-national-wildlife-refuge-oil-drilling.html
## 6 https://www.nytimes.com/2025/10/23/nyregion/new-york-future-wishes-predictions.html
## 7 https://www.nytimes.com/2025/10/23/climate/coral-bleaching-florida-staghorn-elkorn.html
## 8 https://www.nytimes.com/2025/10/23/climate/tsunami-warnings-alerts-noaa.html
print(utils::head(keywordsDF, 10))
## # A tibble: 10 × 2
## webUrl keyword
## <chr> <chr>
## 1 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Califo…
## 2 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Law an…
## 3 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Exxon …
## 4 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Oil (P…
## 5 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Suits …
## 6 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Global…
## 7 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Greenh…
## 8 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Fuel E…
## 9 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Regula…
## 10 https://www.nytimes.com/2025/10/25/climate/exxon-california-lawsuit-… Freedo…
knitr::kable(utils::head(articlesDF, 8), caption = "Sample Articles")
knitr::kable(utils::head(keywordsDF, 10), caption = "Top Keywords")