Assigment 5

#install.packages('rvest') 
library(rvest) 
library(tidyverse)

── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.2.1     ✔ readr     2.2.0
✔ forcats   1.0.1     ✔ stringr   1.6.0
✔ ggplot2   4.0.3     ✔ tibble    3.3.1
✔ lubridate 1.9.5     ✔ tidyr     1.3.2
✔ purrr     1.2.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter()         masks stats::filter()
✖ readr::guess_encoding() masks rvest::guess_encoding()
✖ dplyr::lag()            masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

# Store the URL 
url <- "https://books.toscrape.com" 
# Read and parse the HTML 
webpage <- read_html(url)

books <- webpage |> 
  html_elements("article.product_pod")

length(books) 

[1] 20

# Titles are in the "title" attribute of the <a> tag inside <h3> 
titles <- books |> 
  html_element("h3 > a") |> 
  html_attr("title") 
head(titles) 

[1] "A Light in the Attic"                 
[2] "Tipping the Velvet"                   
[3] "Soumission"                           
[4] "Sharp Objects"                        
[5] "Sapiens: A Brief History of Humankind"
[6] "The Requiem Red"                      

ratings_raw <- books |> 
  html_element("p.star-rating") |> 
  html_attr("class")

ratings <- gsub("star-rating ", "", ratings_raw) 
head(ratings)

[1] "Three" "One"   "One"   "Four"  "Five"  "One"  

# Prices sit in a <p> with class "price_color" 
prices <- books |> 
  html_element("p.price_color") |> 
  html_text2() 
head(prices) 

[1] "£51.77" "£53.74" "£50.10" "£47.82" "£54.23" "£22.65"

prices <- as.numeric(gsub("£", "", prices)) 
head(prices) 

[1] 51.77 53.74 50.10 47.82 54.23 22.65

stock <- books |> 
  html_element("p.availability") |> 
  html_text2() |> 
  trimws() 
head(stock) 

[1] "In stock" "In stock" "In stock" "In stock" "In stock" "In stock"

images <- books |> 
  html_element("img.thumbnail") |> 
  html_attr("src") |> 
  gsub("../../", "https://books.toscrape.com/", x = _) 
head(images)

[1] "media/cachttps://books.toscrape.com/da/2cdad67c44b002e7ead0cc35693c0e8b.jpg"
[2] "media/cachttps://books.toscrape.com/0c/260c6ae16bce31c8f8c95daddd9f4a1c.jpg"
[3] "media/cachttps://books.toscrape.com/ef/3eef99c9d9adef34639f510662022830.jpg"
[4] "media/cachttps://books.toscrape.com/51/3251cf3a3412f53f339e42cac2134093.jpg"
[5] "media/cachttps://books.toscrape.com/a5/bea5697f2534a2f86a3ef27b5a8c12a6.jpg"
[6] "media/cachttps://books.toscrape.com/33/68339b4c9bc034267e1da611ab3b34f8.jpg"

test_df <- tibble( 
  title  = titles, 
  rating = ratings, 
  price  = prices, 
  stock  = stock, 
  image  = images 
) 
head(test_df) 

# A tibble: 6 × 5
  title                                 rating price stock    image             
  <chr>                                 <chr>  <dbl> <chr>    <chr>             
1 A Light in the Attic                  Three   51.8 In stock media/cachttps://…
2 Tipping the Velvet                    One     53.7 In stock media/cachttps://…
3 Soumission                            One     50.1 In stock media/cachttps://…
4 Sharp Objects                         Four    47.8 In stock media/cachttps://…
5 Sapiens: A Brief History of Humankind Five    54.2 In stock media/cachttps://…
6 The Requiem Red                       One     22.6 In stock media/cachttps://…

# Empty list to collect each page's data 
all_books <- list() 

for (i in 1:50) { 
  
  # Page 1 uses the root URL; pages 2+ use /catalogue/page-N.html 
  if (i == 1) { 
    url <- "https://books.toscrape.com" 
  } else { 
    url <- paste0("https://books.toscrape.com/catalogue/page-", i, ".html") 
  }
  page  <- read_html(url) 
  books <- html_elements(page, "article.product_pod") 
  
  all_books[[i]] <- tibble( 
    title  = html_attr(html_element(books, "h3 > a"), "title"), 
    rating = gsub("star-rating ", "", html_attr(html_element(books, "p.star-rating"), "class")), 
    price  = html_text2(html_element(books, "p.price_color")), 
    stock  = trimws(html_text2(html_element(books, "p.availability"))), 
    image  = gsub("../../", "https://books.toscrape.com/", 
                  html_attr(html_element(books, "img.thumbnail"), "src")) 
  ) 
  
  # Be polite — pause briefly between requests 
  Sys.sleep(0.5) 
  cat("Scraped page", i, "\n") 
} 

Scraped page 1 
Scraped page 2 
Scraped page 3 
Scraped page 4 
Scraped page 5 
Scraped page 6 
Scraped page 7 
Scraped page 8 
Scraped page 9 
Scraped page 10 
Scraped page 11 
Scraped page 12 
Scraped page 13 
Scraped page 14 
Scraped page 15 
Scraped page 16 
Scraped page 17 
Scraped page 18 
Scraped page 19 
Scraped page 20 
Scraped page 21 
Scraped page 22 
Scraped page 23 
Scraped page 24 
Scraped page 25 
Scraped page 26 
Scraped page 27 
Scraped page 28 
Scraped page 29 
Scraped page 30 
Scraped page 31 
Scraped page 32 
Scraped page 33 
Scraped page 34 
Scraped page 35 
Scraped page 36 
Scraped page 37 
Scraped page 38 
Scraped page 39 
Scraped page 40 
Scraped page 41 
Scraped page 42 
Scraped page 43 
Scraped page 44 
Scraped page 45 
Scraped page 46 
Scraped page 47 
Scraped page 48 
Scraped page 49 
Scraped page 50 

full_df <- bind_rows(all_books) 
nrow(full_df)  # Should be 1000

[1] 1000

# Save to your working directory 
write.csv(full_df, "bookstoscrape.csv", row.names = FALSE)

full_df$price <- as.numeric(gsub("£", "", full_df$price)) 
head(full_df) 

# A tibble: 6 × 5
  title                                 rating price stock    image             
  <chr>                                 <chr>  <dbl> <chr>    <chr>             
1 A Light in the Attic                  Three   51.8 In stock media/cachttps://…
2 Tipping the Velvet                    One     53.7 In stock media/cachttps://…
3 Soumission                            One     50.1 In stock media/cachttps://…
4 Sharp Objects                         Four    47.8 In stock media/cachttps://…
5 Sapiens: A Brief History of Humankind Five    54.2 In stock media/cachttps://…
6 The Requiem Red                       One     22.6 In stock media/cachttps://…

rating_map <- c(One=1, Two=2, Three=3, Four=4, Five=5) 
full_df1 <- full_df |> 
  mutate(rating_num = rating_map[rating])

table1 <- full_df1 |>  
  count(rating, sort = TRUE) 
table1

# A tibble: 5 × 2
  rating     n
  <chr>  <int>
1 One      226
2 Three    203
3 Five     196
4 Two      196
5 Four     179

full_df1 |> 
  group_by(rating) |> 
  summarise(avg_price = mean(price, na.rm = TRUE)) |> 
  arrange(desc(avg_price))

# A tibble: 5 × 2
  rating avg_price
  <chr>      <dbl>
1 Four        36.1
2 Five        35.4
3 Two         34.8
4 Three       34.7
5 One         34.6

Problem 1

library(ggplot2)
library(dplyr)

ggplot(full_df1, aes(x = rating_num, y = price)) +
  geom_point(color = "purple", alpha = 0.6) +
  geom_smooth(
    method = "lm",
    se = FALSE,
    color = "red",
    linetype = "dotdash"
  ) +
  labs(
    title = "Book Ratings vs Prices",
    x = "Book Rating",
    y = "Price (£)",
    caption = "Source: Books to Scrape (books.toscrape.com)"
  ) +
  theme_minimal()

`geom_smooth()` using formula = 'y ~ x'

Problem 2

rating_counts <- full_df1 |>
  count(rating_num)

ggplot(rating_counts, aes(x = rating_num, y = n)) +
  geom_col(fill = "pink") +
  labs(
    title = "Number of Books by Star Rating",
    x = "Book Rating",
    y = "Number of Books",
    caption = "Source: Books to Scrape (books.toscrape.com)"
  ) +
  theme_classic()

Problem 3

avg_prices <- full_df1 |>
  group_by(rating) |>
  summarise(avg_price = mean(price, na.rm = TRUE))


avg_prices$rating <- factor(
  avg_prices$rating,
  levels = c("One", "Two", "Three", "Four", "Five")) # got external help to correct the order, so my rating could appear in order

  
  ggplot( avg_prices, aes(x = rating, y = avg_price, fill = rating)) +
  geom_col() +
  scale_fill_brewer(palette = "Set3") +
  labs(
    title = "Average Book Price by Star Rating",
    x = "Book Rating",
    y = "Average Price (£)",
    caption = "Source: Books to Scrape (books.toscrape.com)"
  ) +
  theme_classic() +
  theme(legend.position = "none")

avg_prices

# A tibble: 5 × 2
  rating avg_price
  <fct>      <dbl>
1 Five        35.4
2 Four        36.1
3 One         34.6
4 Three       34.7
5 Two         34.8