Activity

Author

Emma Wang

library(tidyverse)

── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.2.1     ✔ readr     2.2.0
✔ forcats   1.0.1     ✔ stringr   1.6.0
✔ ggplot2   4.0.3     ✔ tibble    3.3.1
✔ lubridate 1.9.5     ✔ tidyr     1.3.2
✔ purrr     1.2.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

library(rvest)


Attaching package: 'rvest'

The following object is masked from 'package:readr':

    guess_encoding

library(jsonlite)


Attaching package: 'jsonlite'

The following object is masked from 'package:purrr':

    flatten

library(robotstxt)
library(dplyr)
library(ggplot2)

url = "https://www.goodreads.com/list/show/1.Best_Books_Ever"

books = read_html(url)
paths_allowed(url)


 www.goodreads.com

[1] TRUE

titles = books %>%
  html_elements(".bookTitle span") %>%
  html_text(trim = TRUE) # get text, remove spaces
titles

  [1] "The Hunger Games (The Hunger Games, #1)"                                        
  [2] "Pride and Prejudice"                                                            
  [3] "To Kill a Mockingbird"                                                          
  [4] "Harry Potter and the Order of the Phoenix (Harry Potter, #5)"                   
  [5] "The Book Thief"                                                                 
  [6] "Twilight (The Twilight Saga, #1)"                                               
  [7] "Animal Farm"                                                                    
  [8] "J.R.R. Tolkien 4-Book Boxed Set: The Hobbit and The Lord of the Rings"          
  [9] "The Chronicles of Narnia (The Chronicles of Narnia, #1-7)"                      
 [10] "The Fault in Our Stars"                                                         
 [11] "The Picture of Dorian Gray"                                                     
 [12] "The Lightning Thief (Percy Jackson and the Olympians, #1)"                      
 [13] "Wuthering Heights"                                                              
 [14] "The Giving Tree"                                                                
 [15] "Harry Potter and the Order of the Phoenix (Harry Potter, #5)"                   
 [16] "The Perks of Being a Wallflower"                                                
 [17] "The Little Prince"                                                              
 [18] "Gone with the Wind"                                                             
 [19] "Jane Eyre"                                                                      
 [20] "The Great Gatsby"                                                               
 [21] "Crime and Punishment"                                                           
 [22] "The Da Vinci Code (Robert Langdon, #2)"                                         
 [23] "Divergent (Divergent, #1)"                                                      
 [24] "Alice’s Adventures in Wonderland / Through the Looking-Glass"                   
 [25] "Les Misérables"                                                                 
 [26] "Anne of Green Gables (Anne of Green Gables, #1)"                                
 [27] "Memoirs of a Geisha"                                                            
 [28] "The Alchemist"                                                                  
 [29] "Lord of the Flies"                                                              
 [30] "Romeo and Juliet"                                                               
 [31] "Brave New World"                                                                
 [32] "1984"                                                                           
 [33] "The Hitchhiker's Guide to the Galaxy (The Hitchhiker's Guide to the Galaxy, #1)"
 [34] "City of Bones (The Mortal Instruments, #1)"                                     
 [35] "Fahrenheit 451"                                                                 
 [36] "Dracula"                                                                        
 [37] "The Help"                                                                       
 [38] "Charlotte’s Web"                                                                
 [39] "Of Mice and Men"                                                                
 [40] "Little Women (Little Women, #1)"                                                
 [41] "The Catcher in the Rye"                                                         
 [42] "Ender’s Game (Ender's Saga, #1)"                                                
 [43] "The Outsiders"                                                                  
 [44] "A Thousand Splendid Suns"                                                       
 [45] "One Hundred Years of Solitude"                                                  
 [46] "A Game of Thrones (A Song of Ice and Fire, #1)"                                 
 [47] "The Secret Garden"                                                              
 [48] "The Princess Bride"                                                             
 [49] "Harry Potter and the Deathly Hallows (Harry Potter, #7)"                        
 [50] "The Time Traveler's Wife"                                                       
 [51] "Frankenstein"                                                                   
 [52] "The Odyssey"                                                                    
 [53] "The Handmaid’s Tale (The Handmaid's Tale, #1)"                                  
 [54] "The Girl With the Dragon Tattoo (Millennium, #1)"                               
 [55] "The Giver (Giver, #1)"                                                          
 [56] "The Kite Runner"                                                                
 [57] "Harry Potter and the Prisoner of Azkaban (Harry Potter, #3)"                    
 [58] "A Wrinkle in Time (Time Quintet, #1)"                                           
 [59] "Dune (Dune, #1)"                                                                
 [60] "Where the Wild Things Are"                                                      
 [61] "Green Eggs and Ham"                                                             
 [62] "The Lovely Bones"                                                               
 [63] "The Adventures of Huckleberry Finn (Adventures of Tom and Huck, #2)"            
 [64] "Life of Pi"                                                                     
 [65] "Lolita"                                                                         
 [66] "A Tale of Two Cities"                                                           
 [67] "Slaughterhouse-Five"                                                            
 [68] "The Bell Jar"                                                                   
 [69] "Matilda"                                                                        
 [70] "Harry Potter and the Philosopher's Stone (Harry Potter, #1)"                    
 [71] "Water for Elephants"                                                            
 [72] "The Stand"                                                                      
 [73] "Catch-22"                                                                       
 [74] "The Adventures of Sherlock Holmes (Sherlock Holmes, #3)"                        
 [75] "Rebecca"                                                                        
 [76] "A Clockwork Orange"                                                             
 [77] "The Pillars of the Earth (Kingsbridge, #1)"                                     
 [78] "Watership Down (Watership Down, #1)"                                            
 [79] "The Color Purple"                                                               
 [80] "Great Expectations"                                                             
 [81] "The Fellowship of the Ring (The Lord of the Rings, #1)"                         
 [82] "Outlander (Outlander, #1)"                                                      
 [83] "Anna Karenina"                                                                  
 [84] "The Elephant Tree"                                                              
 [85] "The Brothers Karamazov"                                                         
 [86] "One Flew Over the Cuckoo’s Nest"                                                
 [87] "To Kill a Mockingbird"                                                          
 [88] "My Sister's Keeper"                                                             
 [89] "The Zombie Room"                                                                
 [90] "A Tree Grows in Brooklyn"                                                       
 [91] "The Road"                                                                       
 [92] "Harry Potter and the Goblet of Fire (Harry Potter, #4)"                         
 [93] "Gone Girl"                                                                      
 [94] "The Golden Compass (His Dark Materials, #1)"                                    
 [95] "And Then There Were None"                                                       
 [96] "Siddhartha"                                                                     
 [97] "Vampire Academy (Vampire Academy, #1)"                                          
 [98] "Don Quixote"                                                                    
 [99] "The Old Man and the Sea"                                                        
[100] "Angela's Ashes (Frank McCourt, #1)"

rank = books %>%
  html_elements(".number") %>%
  html_text(trim = TRUE) # get text, remove spaces
rank

  [1] "1"   "2"   "3"   "4"   "5"   "6"   "7"   "8"   "9"   "10"  "11"  "12" 
 [13] "13"  "14"  "15"  "16"  "17"  "18"  "19"  "20"  "21"  "22"  "23"  "24" 
 [25] "25"  "26"  "27"  "28"  "29"  "30"  "31"  "32"  "33"  "34"  "35"  "36" 
 [37] "37"  "38"  "39"  "40"  "41"  "42"  "43"  "44"  "45"  "46"  "47"  "48" 
 [49] "49"  "50"  "51"  "52"  "53"  "54"  "55"  "56"  "57"  "58"  "59"  "60" 
 [61] "61"  "62"  "63"  "64"  "65"  "66"  "67"  "68"  "69"  "70"  "71"  "72" 
 [73] "73"  "74"  "75"  "76"  "77"  "78"  "79"  "80"  "81"  "82"  "83"  "84" 
 [85] "85"  "86"  "87"  "88"  "89"  "90"  "91"  "92"  "93"  "94"  "95"  "96" 
 [97] "97"  "98"  "99"  "100"

author = books %>% 
  html_elements(".authorName span") %>%
  html_text(trim = TRUE) # get text, remove spaces
author

  [1] "Suzanne Collins"              "Jane Austen"                 
  [3] "Harper Lee"                   "J.K. Rowling"                
  [5] "Markus Zusak"                 "Stephenie Meyer"             
  [7] "George Orwell"                "J.R.R. Tolkien"              
  [9] "C.S. Lewis"                   "John  Green"                 
 [11] "Oscar Wilde"                  "Rick Riordan"                
 [13] "Emily Brontë"                 "Shel Silverstein"            
 [15] "J.K. Rowling"                 "Stephen Chbosky"             
 [17] "Antoine de Saint-Exupéry"     "Margaret Mitchell"           
 [19] "Charlotte Brontë"             "F. Scott Fitzgerald"         
 [21] "Fyodor Dostoevsky"            "Dan    Brown"                
 [23] "Veronica Roth"                "Lewis Carroll"               
 [25] "Victor Hugo"                  "L.M. Montgomery"             
 [27] "Arthur Golden"                "Paulo Coelho"                
 [29] "William Golding"              "William Shakespeare"         
 [31] "Aldous Huxley"                "George Orwell"               
 [33] "Douglas Adams"                "Cassandra Clare"             
 [35] "Ray Bradbury"                 "Bram Stoker"                 
 [37] "Kathryn Stockett"             "E.B. White"                  
 [39] "John Steinbeck"               "Louisa May Alcott"           
 [41] "J.D. Salinger"                "Orson Scott Card"            
 [43] "S.E. Hinton"                  "Khaled Hosseini"             
 [45] "Gabriel García Márquez"       "George R.R. Martin"          
 [47] "Frances Hodgson Burnett"      "William Goldman"             
 [49] "J.K. Rowling"                 "Audrey Niffenegger"          
 [51] "Mary Wollstonecraft Shelley"  "Homer"                       
 [53] "Margaret Atwood"              "Stieg Larsson"               
 [55] "Lois Lowry"                   "Khaled Hosseini"             
 [57] "J.K. Rowling"                 "Madeleine L'Engle"           
 [59] "Frank Herbert"                "Maurice Sendak"              
 [61] "Dr. Seuss"                    "Alice Sebold"                
 [63] "Mark Twain"                   "Yann Martel"                 
 [65] "Vladimir Nabokov"             "Charles Dickens"             
 [67] "Kurt Vonnegut Jr."            "Sylvia Plath"                
 [69] "Roald Dahl"                   "J.K. Rowling"                
 [71] "Sara Gruen"                   "Stephen  King"               
 [73] "Joseph Heller"                "Arthur Conan Doyle"          
 [75] "Daphne du Maurier"            "Anthony Burgess"             
 [77] "Ken Follett"                  "Richard  Adams"              
 [79] "Alice Walker"                 "Charles Dickens"             
 [81] "J.R.R. Tolkien"               "Diana Gabaldon"              
 [83] "Leo Tolstoy"                  "R.D. Ronald"                 
 [85] "Fyodor Dostoevsky"            "Ken Kesey"                   
 [87] "Harper Lee"                   "Jodi Picoult"                
 [89] "R.D. Ronald"                  "Betty  Smith"                
 [91] "Cormac McCarthy"              "J.K. Rowling"                
 [93] "Gillian Flynn"                "Philip Pullman"              
 [95] "Agatha Christie"              "Hermann Hesse"               
 [97] "Richelle Mead"                "Miguel de Cervantes Saavedra"
 [99] "Ernest Hemingway"             "Frank McCourt"

scores = books %>%
  html_elements(".uitext a:nth-child(1)") %>%
  html_text(trim = TRUE) %>%
  gsub("score:", "", .)
scores

  [1] " 4,566,990" " 3,095,908" " 2,683,165" " 2,129,002" " 2,032,473"
  [6] " 1,807,409" " 1,797,238" " 1,729,884" " 1,569,766" " 1,456,683"
 [11] " 1,429,671" " 1,338,505" " 1,290,202" " 1,271,527" " 1,246,895"
 [16] " 1,239,093" " 1,231,936" " 1,206,012" " 1,186,581" " 1,167,085"
 [21] " 1,108,276" " 1,045,335" " 1,029,380" " 1,028,857" " 989,921"  
 [26] " 987,838"   " 981,753"   " 979,881"   " 966,740"   " 944,982"  
 [31] " 941,775"   " 907,563"   " 901,488"   " 881,938"   " 879,659"  
 [36] " 874,144"   " 868,112"   " 857,557"   " 840,366"   " 825,427"  
 [41] " 821,120"   " 817,135"   " 797,708"   " 786,446"   " 786,356"  
 [46] " 746,799"   " 745,571"   " 742,405"   " 720,181"   " 712,364"  
 [51] " 705,826"   " 696,064"   " 680,668"   " 677,320"   " 663,969"  
 [56] " 663,113"   " 662,958"   " 653,019"   " 613,324"   " 590,696"  
 [61] " 589,191"   " 579,067"   " 572,733"   " 566,400"   " 560,666"  
 [66] " 554,660"   " 549,017"   " 540,167"   " 532,304"   " 522,361"  
 [71] " 508,869"   " 468,269"   " 462,485"   " 452,336"   " 446,465"  
 [76] " 446,101"   " 445,297"   " 436,623"   " 426,017"   " 422,299"  
 [81] " 419,500"   " 417,216"   " 415,813"   " 409,750"   " 397,118"  
 [86] " 396,711"   " 383,283"   " 382,257"   " 377,187"   " 357,978"  
 [91] " 355,924"   " 353,620"   " 346,028"   " 337,809"   " 336,464"  
 [96] " 329,071"   " 324,511"   " 320,887"   " 317,361"   " 315,498"

books = tibble(
  titles    = titles,
  rank    = rank,
  author = author,
  scores   = scores
)

books

# A tibble: 100 × 4
   titles                                                    rank  author scores
   <chr>                                                     <chr> <chr>  <chr> 
 1 The Hunger Games (The Hunger Games, #1)                   1     Suzan… " 4,5…
 2 Pride and Prejudice                                       2     Jane … " 3,0…
 3 To Kill a Mockingbird                                     3     Harpe… " 2,6…
 4 Harry Potter and the Order of the Phoenix (Harry Potter,… 4     J.K. … " 2,1…
 5 The Book Thief                                            5     Marku… " 2,0…
 6 Twilight (The Twilight Saga, #1)                          6     Steph… " 1,8…
 7 Animal Farm                                               7     Georg… " 1,7…
 8 J.R.R. Tolkien 4-Book Boxed Set: The Hobbit and The Lord… 8     J.R.R… " 1,7…
 9 The Chronicles of Narnia (The Chronicles of Narnia, #1-7) 9     C.S. … " 1,5…
10 The Fault in Our Stars                                    10    John … " 1,4…
# ℹ 90 more rows

write_csv(books, "books_simple.csv")

getwd()

[1] "/Users/emmawang"

Example 2

url1 = "https://en.wikipedia.org/wiki/List_of_countries_by_GDP_(nominal)_per_capita"

nominal = read_html(url1)

paths_allowed(url1)


 en.wikipedia.org

[1] TRUE

tab = nominal |> html_nodes("table")
tab

{xml_nodeset (6)}
[1] <table width="100%h" id="mwDg"><tbody id="mwDw"><tr id="mwEA">\n<td valig ...
[2] <table class="wikitable sortable static-row-numbers sort-under-center" st ...
[3] <table class="nowraplinks mw-collapsible uncollapsed navbox-inner" style= ...
[4] <table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" ...
[5] <table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbod ...
[6] <table class="nowraplinks mw-collapsible autocollapse navbox-inner" style ...

html_table(tab[[2]]) |> names()

[1] "Country/Territory"          "IMF(2026)[b][5]"           
[3] "World Bank(2025)[6]"        "United Nations(2023-24)[7]"

tab2 = tab[[2]] %>% html_table 
tab2

# A tibble: 223 × 4
   `Country/Territory` `IMF(2026)[b][5]`                   `World Bank(2025)[6]`
   <chr>               <chr>                               <chr>                
 1 Monaco              —.mw-parser-output .sr-only{border… 288,002 (2024)       
 2 Liechtenstein       230,417                             220,167 (2024)       
 3 Luxembourg          155,733                             147,252              
 4 Bermuda             —N/a                                142,250 (2024)       
 5 Ireland             127,837                             131,593              
 6 Switzerland         124,478                             114,769              
 7 Norway              115,188                             94,594               
 8 Singapore           109,117                             98,814               
 9 Iceland             107,464                             98,324               
10 Cayman Islands      —N/a                                104,293 (2024)       
# ℹ 213 more rows
# ℹ 1 more variable: `United Nations(2023-24)[7]` <chr>

Note: The original link on the class slide didn’t work, and instead a substitute article was provided. From my knowledge, data (regarding GDP and such) provided from the World Bank tends to be more accurate so I will use the World Bank column to count nominal GDP.

tab2_clean = tab2 %>% 
  #select only needed columns 
  select(`Country/Territory`, `World Bank(2025)[6]`) %>%
  #rename world bank
  rename(
    `World Bank` = `World Bank(2025)[6]`
  ) %>%
  filter(`World Bank` !="NA") %>%
  filter(`World Bank` !="—N/a") 
tab2_clean

# A tibble: 216 × 2
   `Country/Territory` `World Bank`  
   <chr>               <chr>         
 1 Monaco              288,002 (2024)
 2 Liechtenstein       220,167 (2024)
 3 Luxembourg          147,252       
 4 Bermuda             142,250 (2024)
 5 Ireland             131,593       
 6 Switzerland         114,769       
 7 Norway              94,594        
 8 Singapore           98,814        
 9 Iceland             98,324        
10 Cayman Islands      104,293 (2024)
# ℹ 206 more rows

final_tab2 = tab2_clean |> 
  filter(`Country/Territory` %in% c("Ireland", "Switzerland", "Iceland", "Norway"))
final_tab2

# A tibble: 4 × 2
  `Country/Territory` `World Bank`
  <chr>               <chr>       
1 Ireland             131,593     
2 Switzerland         114,769     
3 Norway              94,594      
4 Iceland             98,324

ggplot(final_tab2, aes(x = `Country/Territory`, y = `World Bank`)) +
  geom_col(fill = "lightsteelblue4") +
  theme_minimal() +
  labs(
    title = "Comparing Nominal GDP of Four Countries", 
    subtitle = "Iceland, Ireland, Norway, Switzerland", 
    caption = "GDP displayed in $USD", 
    y = "Nominal GDP "
  )