The following libraries are used in this assignment
jsonlite
httr
# knitr::opts_chunk$set(echo = TRUE)
# Clear the console
cat("\014")
# Check if the package is installed. If not, install the package
if(!require('jsonlite')) {
install.packages('jsonlite')
library(jsonlite)
}## Loading required package: jsonlite# Check if the package is installed. If not, install the package
if(!require('httr')) {
install.packages('httr')
library(httr)
}## Loading required package: httr## Warning: package 'httr' was built under R version 3.3.3# Read API Key from file
APIKey <- readLines("https://raw.githubusercontent.com/kalyanparthasarathy/DATA607/master/APIKey.txt")In this section, the books API data from NY Times is retrieved using the API keys stored in Github.
# NYTimes Books Data
# Check if the URL is returning data
URL <- paste0("https://api.nytimes.com/svc/books/v3/lists.json?api_key=", APIKey, "&list=hardcover-fiction")
nytimes_books_get_method <- GET(URL);
# Need to get status code 200 - means the URL call worked fine
nytimes_books_get_method$status_code## [1] 200# Read the data in JSON format and the results will be in nested Dataframe object
nytimes_books_json_data <- jsonlite::fromJSON(
URL
, flatten = TRUE
)
# Names from the JSON call - top level dataframes
# names(nytimes_books_json_data)
# Check the results dataframe
# names(nytimes_books_json_data$results)
# Structure of results dataframe
# str(nytimes_books_json_data)
# Store the books dataframe results
nytimes_books_results_df <- nytimes_books_json_data$results
# Book details dataframe
nytimes_books_details_df <- nytimes_books_results_df$book_details
nytimes_books_df_flat <- flatten(nytimes_books_results_df, recursive = TRUE)
# str(nytimes_books_results_df)
head(nytimes_books_df_flat, 10)## list_name display_name bestsellers_date published_date
## 1 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 2 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 3 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 4 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 5 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 6 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 7 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 8 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 9 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## 10 Hardcover Fiction Hardcover Fiction 2017-10-21 2017-11-05
## rank rank_last_week weeks_on_list asterisk dagger
## 1 1 1 3 0 0
## 2 2 0 1 0 0
## 3 3 0 1 0 0
## 4 4 2 4 0 0
## 5 5 3 2 0 0
## 6 6 4 6 0 0
## 7 7 7 3 0 0
## 8 8 9 4 0 0
## 9 9 8 6 0 0
## 10 10 0 1 0 0
## amazon_product_url
## 1 https://www.amazon.com/Origin-Novel-Dan-Brown-ebook/dp/B01LY7FD0D?tag=NYTBS-20
## 2 https://www.amazon.com/Deep-Freeze-Virgil-Flowers-Novel-ebook/dp/B01NBU65Z1?tag=NYTBS-20
## 3 https://www.amazon.com/Uncommon-Type-Stories-Tom-Hanks/dp/1101946156?tag=NYTBS-20
## 4 https://www.amazon.com/Sleeping-Beauties-Novel-Stephen-King/dp/150116340X?tag=NYTBS-20
## 5 https://www.amazon.com/Fairytale-Novel-Danielle-Steel/dp/1101884061?tag=NYTBS-20
## 6 https://www.amazon.com/Column-Fire-Kingsbridge-Ken-Follett-ebook/dp/B01N1U8IB3?tag=NYTBS-20
## 7 https://www.amazon.com/Manhattan-Beach-Novel-Jennifer-Egan/dp/1476716730?tag=NYTBS-20
## 8 https://www.amazon.com/Dont-Let-Go-Harlan-Coben-ebook/dp/B01NH7GION?tag=NYTBS-20
## 9 https://www.amazon.com/Girl-Who-Takes-Eye-continuing-ebook/dp/B01MR8VA0P?tag=NYTBS-20
## 10 https://www.amazon.com/Devours-Welcome-Night-Vale-Novel/dp/006247605X?tag=NYTBS-20
## isbns
## 1 0385514239, 0385542690, 0525434305, 9780385514231, 9780385542692, 9780525434306
## 2 0399176063, 0698407113, 9780399176067, 9780698407114
## 3 1101946156, 1101946164, 0525590501, 9781101946152, 9781101946169, 9780525590507
## 4 150116340X, 1501163426, 9781501163401, 9781501163425
## 5 1101884061, 110188407X, 0525501274, 9781101884065, 9781101884072, 9780525501275
## 6 052595497X, 0735224471, 9780525954972, 9780735224476
## 7 1476716730, 1476716757, 1432843419, 9781476716732, 9781476716756, 9781432843410
## 8 0525955119, 0698411668, 9780525955115, 9780698411661
## 9 0451494326, 0451494334, 0525499407, 9780451494320, 9780451494337, 9780525499404
## 10 006247605X, 9780062476050
## book_details
## 1 ORIGIN, A symbology professor goes on a perilous quest with a beautiful museum director., by Dan Brown, Dan Brown, , 0, , Doubleday, 9780385514231, 0385514239
## 2 DEEP FREEZE, Virgil Flowers is called to investigate in Trippton, Minn., when a local bank's president is found dead in a nearly frozen river., by John Sandford, John Sandford, , 0, , Putnam, 9780399176067, 0399176063
## 3 UNCOMMON TYPE, Seventeen short stories, each incorporating a typewriter, by the Academy Award-winning actor., by Tom Hanks, Tom Hanks, , 0, , Knopf, 9781101946152, 1101946156
## 4 SLEEPING BEAUTIES, Women who fall asleep become shrouded in mysterious cocoons while the men battle one another., by Stephen King and Owen King, Stephen King and Owen King, , 0, , Scribner, 9781501163401, 150116340X
## 5 FAIRYTALE, Tragedy and unexpected dangers come to life at a Napa Valley winery., by Danielle Steel, Danielle Steel, , 0, , Delacorte, 9781101884065, 1101884061
## 6 A COLUMN OF FIRE, A pair of lovers find themselves on opposite sides of a conflict while Queen Elizabeth fights to maintain her throne., by Ken Follett, Ken Follett, , 0, , Viking, 9780525954972, 052595497X
## 7 MANHATTAN BEACH, The first female diver at the Brooklyn Naval Yard during World War II tries to understand why her father disappeared., by Jennifer Egan, Jennifer Egan, , 0, , Scribner, 9781476716732, 1476716730
## 8 DON'T LET GO, Det. Napoleon Dumas investigates a murder and uncovers clues about personal events that happened 15 years ago., by Harlan Coben, Harlan Coben, , 0, , Dutton, 9780525955115, 0525955119
## 9 THE GIRL WHO TAKES AN EYE FOR AN EYE, Stieg Larsson's character Lisbeth Salander seeks to uncover the secrets of her childhood., by David Lagercrantz, David Lagercrantz, , 0, , Knopf, 9780451494320, 0451494326
## 10 IT DEVOURS!, A female scientist investigates an unusual rumbling in an American Southwest desert and comes across a religious congregation planning a dangerous ritual., by Joseph Fink and Jeffrey Cranor, Joseph Fink and Jeffrey Cranor, , 0, , Harper Perennial, 9780062476050, 006247605X
## reviews
## 1 , , ,
## 2 , , ,
## 3 , , ,
## 4 , , ,
## 5 , , ,
## 6 , , ,
## 7 https://www.nytimes.com/2017/10/03/books/review/jennifer-egan-manhattan-beach.html, , ,
## 8 , , ,
## 9 , , ,
## 10 , , ,In this section, the Top Stories API data from NY Times is retrieved using the API keys stored in Github.
# NYTimes Top Stories Data
# Check if the URL is returning data
URL <- paste0("https://api.nytimes.com/svc/topstories/v2/technology.json?api_key=", APIKey, "§ion=technology")
nytimes_tech_topstories_get_method <- GET(URL);
# Need to get status code 200 - means the URL call worked fine
nytimes_tech_topstories_get_method$status_code## [1] 200# Read the data in JSON format and the results will be in nested Dataframe object
nytimes_tech_topstories_json_data <- jsonlite::fromJSON(
URL
, flatten = TRUE
)
nytimes_tech_topstories_json_data <- nytimes_tech_topstories_json_data[nytimes_tech_topstories_json_data$section == 'technology']
# Names from the JSON call - top level dataframes
names(nytimes_tech_topstories_json_data)## [1] "status" "copyright" "section" "last_updated"
## [5] "num_results" "results"# Check the results dataframe
names(nytimes_tech_topstories_json_data$results)## [1] "section" "subsection" "title"
## [4] "abstract" "url" "byline"
## [7] "item_type" "updated_date" "created_date"
## [10] "published_date" "material_type_facet" "kicker"
## [13] "des_facet" "org_facet" "per_facet"
## [16] "geo_facet" "multimedia" "short_url"# Structure of results dataframe
# str(nytimes_tech_topstories_json_data$results)
# Store the top stories dataframe results
nytimes_tech_topstories_results_df <- nytimes_tech_topstories_json_data$results
nytimes_tech_topstories_df_flat <- flatten(nytimes_tech_topstories_results_df, recursive = TRUE)
# str(nytimes_tech_topstories_df_flat)
nytimes_tech_topstories_df_flat <- nytimes_tech_topstories_df_flat[, c(1, 2, 3, 5, 7, 8, 9, 10)]
head(nytimes_tech_topstories_df_flat, 10)## section subsection
## 1 Business Day
## 2 Technology
## 3 Technology
## 4 U.S.
## 5 Style
## 6 Style
## 7 Technology
## 8 Technology
## 9 World Asia Pacific
## 10 World Europe
## title
## 1 What Worries? Big Tech Companies Post Glowing Quarterly Profits
## 2 Russia Fanned Flames With Twitter, Which Faces a Blowback
## 3 Twitter Bans Two Kremlin-Backed News Outlets From Advertising
## 4 Reddit Bans Nazi Groups and Others in Crackdown on Violent Content
## 5 Virtual Reality Gets Naughty
## 6 Amazon Key Is a Lot Less Scary Than My Post-1-Click Remorse
## 7 Hearing Amazon<U+0092>s Footsteps, the Health Care Industry Shudders
## 8 At Facebook, Hand-Wringing Over a Fix for Fake Content
## 9 A War of Words Puts Facebook at the Center of Myanmar<U+0092>s Rohingya Crisis
## 10 Britain Says North Korea Was Behind Cyberattack on Health Service
## url
## 1 https://www.nytimes.com/2017/10/26/business/big-tech-company-earnings.html
## 2 https://www.nytimes.com/2017/10/26/technology/twitter-russia.html
## 3 https://www.nytimes.com/2017/10/26/technology/twitter-russia-today-sputnik.html
## 4 https://www.nytimes.com/2017/10/26/us/reddit-violence-policy.html
## 5 https://www.nytimes.com/2017/10/28/style/virtual-reality-porn.html
## 6 https://www.nytimes.com/2017/10/28/style/amazon-key-1-click-buyers-remorse.html
## 7 https://www.nytimes.com/2017/10/27/technology/amazon-pharmacy-drugs.html
## 8 https://www.nytimes.com/2017/10/27/technology/facebook-fake-content-employees.html
## 9 https://www.nytimes.com/2017/10/27/world/asia/myanmar-government-facebook-rohingya.html
## 10 https://www.nytimes.com/2017/10/27/world/europe/uk-ransomware-hack-north-korea.html
## item_type updated_date created_date
## 1 Article 2017-10-26T20:25:31-04:00 2017-10-26T19:43:02-04:00
## 2 Article 2017-10-26T17:33:09-04:00 2017-10-26T15:40:07-04:00
## 3 Article 2017-10-26T20:43:29-04:00 2017-10-26T10:18:01-04:00
## 4 Article 2017-10-26T18:03:41-04:00 2017-10-26T15:58:15-04:00
## 5 Article 2017-10-28T06:00:29-04:00 2017-10-28T06:00:29-04:00
## 6 Article 2017-10-28T06:00:02-04:00 2017-10-28T06:00:02-04:00
## 7 Article 2017-10-27T20:32:38-04:00 2017-10-27T19:57:31-04:00
## 8 Article 2017-10-27T18:59:59-04:00 2017-10-27T18:59:59-04:00
## 9 Article 2017-10-27T15:46:06-04:00 2017-10-27T15:46:06-04:00
## 10 Article 2017-10-27T15:45:36-04:00 2017-10-27T15:45:36-04:00
## published_date
## 1 2017-10-26T19:43:02-04:00
## 2 2017-10-26T15:40:07-04:00
## 3 2017-10-26T10:18:01-04:00
## 4 2017-10-26T15:58:15-04:00
## 5 2017-10-28T06:00:29-04:00
## 6 2017-10-28T06:00:02-04:00
## 7 2017-10-27T19:57:31-04:00
## 8 2017-10-27T18:59:59-04:00
## 9 2017-10-27T15:46:06-04:00
## 10 2017-10-27T15:45:36-04:00