web-scrape-single.R

# Loading Packages ----
library(tidyverse)

## Warning: package 'ggplot2' was built under R version 4.3.3

## Warning: package 'tibble' was built under R version 4.3.3

## Warning: package 'tidyr' was built under R version 4.3.1

## Warning: package 'readr' was built under R version 4.3.1

## Warning: package 'dplyr' was built under R version 4.3.1

## Warning: package 'stringr' was built under R version 4.3.1

## Warning: package 'lubridate' was built under R version 4.3.3

## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.4     ✔ readr     2.1.5
## ✔ forcats   1.0.0     ✔ stringr   1.5.1
## ✔ ggplot2   3.5.2     ✔ tibble    3.3.0
## ✔ lubridate 1.9.4     ✔ tidyr     1.3.1
## ✔ purrr     1.0.2     
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

library(rvest)

## Warning: package 'rvest' was built under R version 4.3.1

## 
## Attaching package: 'rvest'
## 
## The following object is masked from 'package:readr':
## 
##     guess_encoding

library(polite)

#check for scrape-ability ----

bow("https://www.scrapingcourse.com")

## <polite session> https://www.scrapingcourse.com
##     User-agent: polite R package
##     robots.txt: 1 rules are defined for 1 bots
##    Crawl delay: 5 sec
##   The path is scrapable for this user-agent

#read page ----
url <- "https://www.scrapingcourse.com/ecommerce/"
page <- read_html(url)
# 
# # titles ----
titles <- page |>
  html_elements(".woocommerce-loop-product__title") |>
  html_text()


# #urls ----
urls <- page |>
  html_elements(".woocommerce-loop-product__link") |>
  html_attr("href")

# # prices ----
prices <- page |>
  html_elements(".woocommerce-Price-amount") |>
  html_text(trim = TRUE) |>
  str_extract("\\d+\\.\\d+") |>   # Extract only numbers
  as.numeric()
 
 

items <- page |>
  html_elements(".product") |>  # Parent container for each product
  map_df(function(product) {
    
    title <- product |>
      html_element(".woocommerce-loop-product__title") |>
      html_text(trim = TRUE)
    
    url <- product |>
      html_element(".woocommerce-loop-product__link") |>
      html_attr("href")
    
    price <- product |>
      html_element(".woocommerce-Price-amount") |>
      html_text(trim = TRUE) |>
      str_extract("\\d+\\.\\d+") |>
      as.numeric()
    
    tibble(title = title, url = url, price = price)
  })



# write items ----
write_csv(items, "data/items.csv")

read.csv("data/items.csv")

##                           title
## 1             Abominable Hoodie
## 2          Adrienne Trek Jacket
## 3                    Aeon Capri
## 4        Aero Daily Fitness Tee
## 5               Aether Gym Pant
## 6           Affirm Water Bottle
## 7              Aim Analog Watch
## 8      Ajax Full-Zip Sweatshirt
## 9             Ana Running Short
## 10    Angel Light Running Short
## 11           Antonia Racer Tank
## 12         Apollo Running Short
## 13            Arcadio Gym Short
## 14       Argus All-Weather Tank
## 15 Ariel Roll Sleeve Sweatshirt
## 16        Artemis Running Short
##                                                                               url
## 1             https://www.scrapingcourse.com/ecommerce/product/abominable-hoodie/
## 2          https://www.scrapingcourse.com/ecommerce/product/adrienne-trek-jacket/
## 3                    https://www.scrapingcourse.com/ecommerce/product/aeon-capri/
## 4        https://www.scrapingcourse.com/ecommerce/product/aero-daily-fitness-tee/
## 5               https://www.scrapingcourse.com/ecommerce/product/aether-gym-pant/
## 6           https://www.scrapingcourse.com/ecommerce/product/affirm-water-bottle/
## 7              https://www.scrapingcourse.com/ecommerce/product/aim-analog-watch/
## 8      https://www.scrapingcourse.com/ecommerce/product/ajax-full-zip-sweatshirt/
## 9             https://www.scrapingcourse.com/ecommerce/product/ana-running-short/
## 10    https://www.scrapingcourse.com/ecommerce/product/angel-light-running-short/
## 11           https://www.scrapingcourse.com/ecommerce/product/antonia-racer-tank/
## 12         https://www.scrapingcourse.com/ecommerce/product/apollo-running-short/
## 13            https://www.scrapingcourse.com/ecommerce/product/arcadio-gym-short/
## 14       https://www.scrapingcourse.com/ecommerce/product/argus-all-weather-tank/
## 15 https://www.scrapingcourse.com/ecommerce/product/ariel-roll-sleeve-sweatshirt/
## 16        https://www.scrapingcourse.com/ecommerce/product/artemis-running-short/
##    price
## 1   69.0
## 2   57.0
## 3   48.0
## 4   24.0
## 5   74.0
## 6    7.0
## 7   45.0
## 8   69.0
## 9   40.0
## 10  42.0
## 11  34.0
## 12  32.5
## 13  20.0
## 14  22.0
## 15  39.0
## 16  45.0