library(rvest)
library(stringr)
library(httr2)
library(dplyr)##
## Attaching package: 'dplyr'## The following objects are masked from 'package:stats':
##
## filter, lag## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union# List of URLs to scrape
urls <- c(
"https://www.llbean.com",
"https://www.patagonia.com",
"https://www.nike.com",
"https://www.gap.com/",
"https://www.ralphlauren.com/",
"https://www.shein.com/"
)
# Empty list to collect results
results <- list()
for (i in seq_along(urls)) {
Sys.sleep(1.5) # polite delay between requests
page <- read_html(urls[i])
title_tag <- page %>%
html_element("title") %>%
html_text2() %>%
str_squish()
title_tag
meta_desc <- page %>%
html_element("meta[name='description']") %>%
html_attr("content") %>%
str_squish()
meta_desc
h1_tags <- page %>%
html_elements("h1") %>%
html_text2() %>%
str_squish() %>%
str_c(collapse = " | ")
h1_tags
h2_tags <- page %>%
html_elements("h2") %>%
html_text2() %>%
str_squish() %>%
str_c(collapse = " | ")
h2_tags
h3_tags <- page %>%
html_elements("h3") %>%
html_text2() %>%
str_squish() %>%
str_c(collapse = " | ")
h3_tags
# Store this page's results as a one-row tibble
results[[i]] <- tibble(
url = urls[i],
title = title_tag,
meta_description = meta_desc,
h1 = h1_tags,
h2 = h2_tags,
h3 = h3_tags
)
}
# Combine all rows into one data frame
seo_data <- bind_rows(results)
seo_data## # A tibble: 6 × 6
## url title meta_description h1 h2 h3
## <chr> <chr> <chr> <chr> <chr> <chr>
## 1 https://www.llbean.com L.L.Bean - Ou… Enjoy Free Ship… "L.L… "It'… ""
## 2 https://www.patagonia.com Hang Tight! R… <NA> "" "Sit… "U.S…
## 3 https://www.nike.com Nike. Just Do… Inspiring the w… "" "Run… "ROW…
## 4 https://www.gap.com/ Gap Individua… Discover casual… "" "" ""
## 5 https://www.ralphlauren.com/ Access to thi… px-captcha "" "" ""
## 6 https://www.shein.com/ Women's Cloth… Exclusive disco… "" "" ""write.csv(seo_data, file = "seo_data.csv", row.names = FALSE)
seo_data$title## [1] "L.L.Bean - Outside Together Since 1912"
## [2] "Hang Tight! Routing to checkout..."
## [3] "Nike. Just Do It. Nike.com"
## [4] "Gap Individual style for Women's, Men's, Baby & Kids' Clothing"
## [5] "Access to this page has been denied"
## [6] "Women's Clothing, Women Fashion Sale | SHEIN USA"seo_data$meta_description## [1] "Enjoy Free Shipping with $75 purchase on clothing, shoes, outdoor gear and more, all made for the shared joy of the outdoors."
## [2] NA
## [3] "Inspiring the world's athletes, Nike delivers innovative products, experiences and services."
## [4] "Discover casual clothing for women, men, kids, and baby at Gap. Stylish, comfy, and modern pieces for the whole family. Enjoy special deals on jeans, tees & more!"
## [5] "px-captcha"
## [6] "Exclusive discounts and the latest trends at SHEIN — Women's Dresses, Tops, Bottoms, Shoes, Plus Size and more. ✓ Free Shipping ✓ App only. 15% Off With No Min.+✓Easy returns ✓ New Arrivals Dropped Daily ✓"seo_data$h1## [1] "L.L.Bean - Outside Together Since 1912. Free Shipping with $75 purchase."
## [2] ""
## [3] ""
## [4] ""
## [5] ""
## [6] ""seo_data$h2## [1] "It's time to upgrade your browser | Best Worn to Make Summer Memories | Best Worn to Make Summer Memories"
## [2] "Sit tight"
## [3] "Run Free, Play Forever | Nike Vapor Edge 360 \"Untouchable\" | Run Free, Play Forever | Nike Vapor Edge 360 \"Untouchable\" | Lock In. Level Up. | Lock In. Level Up. | TRENDING | TRENDING | Africa | Americas | Asia Pacific | Europe | Middle East"
## [4] ""
## [5] ""
## [6] ""seo_data$h3## [1] ""
## [2] "U.S. & CANADA | EUROPE | JAPAN"
## [3] "ROWING STRONG | HIS LEGEND LIVES ON | IRRATIONAL BELIEF | THREE LION HEARTED | ROWING STRONG | HIS LEGEND LIVES ON | SWEAT. PROOF. | SWEAT. PROOF. | ACG Radical AirFlow | Play outside the lines with Nike Free Ride’s flexible innovation, now in fresh new colors. | The cleat that took the world by storm, modernized with our greatest innovations. | Nothing sweet About it | A Striking Approach | One on Anyone | Play outside the lines with Nike Free Ride’s flexible innovation, now in fresh new colors. | The cleat that took the world by storm, modernized with our greatest innovations. | Nothing sweet About it | A Striking Approach | One on Anyone | Practice Ready | First Day Footwear | Classroom Ready | Bring serious style to their everyday game. | FULL HORSEPOWER | FULL HORSEPOWER"
## [4] ""
## [5] ""
## [6] ""write.csv(seo_data, file = "seo_data_v3.csv", row.names = FALSE)library(readr)##
## Attaching package: 'readr'## The following object is masked from 'package:rvest':
##
## guess_encodinglibrary(knitr)
df <- read_csv("seo_data_v3.csv")## Rows: 6 Columns: 6## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (6): url, title, meta_description, h1, h2, h3
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.kable(df)| url | title | meta_description | h1 | h2 | h3 |
|---|---|---|---|---|---|
| https://www.llbean.com | L.L.Bean - Outside Together Since 1912 | Enjoy Free Shipping with $75 purchase on clothing, shoes, outdoor gear and more, all made for the shared joy of the outdoors. | L.L.Bean - Outside Together Since 1912. Free Shipping with $75 purchase. | It’s time to upgrade your browser | Best Worn to Make Summer Memories | Best Worn to Make Summer Memories | NA |
| https://www.patagonia.com | Hang Tight! Routing to checkout… | NA | NA | Sit tight | U.S. & CANADA | EUROPE | JAPAN |
| https://www.nike.com | Nike. Just Do It. Nike.com | Inspiring the world’s athletes, Nike delivers innovative products, experiences and services. | NA | Run Free, Play Forever | Nike Vapor Edge 360 “Untouchable” | Run Free, Play Forever | Nike Vapor Edge 360 “Untouchable” | Lock In. Level Up. | Lock In. Level Up. | TRENDING | TRENDING | Africa | Americas | Asia Pacific | Europe | Middle East | ROWING STRONG | HIS LEGEND LIVES ON | IRRATIONAL BELIEF | THREE LION HEARTED | ROWING STRONG | HIS LEGEND LIVES ON | SWEAT. PROOF. | SWEAT. PROOF. | ACG Radical AirFlow | Play outside the lines with Nike Free Ride’s flexible innovation, now in fresh new colors. | The cleat that took the world by storm, modernized with our greatest innovations. | Nothing sweet About it | A Striking Approach | One on Anyone | Play outside the lines with Nike Free Ride’s flexible innovation, now in fresh new colors. | The cleat that took the world by storm, modernized with our greatest innovations. | Nothing sweet About it | A Striking Approach | One on Anyone | Practice Ready | First Day Footwear | Classroom Ready | Bring serious style to their everyday game. | FULL HORSEPOWER | FULL HORSEPOWER |
| https://www.gap.com/ | Gap Individual style for Women’s, Men’s, Baby & Kids’ Clothing | Discover casual clothing for women, men, kids, and baby at Gap. Stylish, comfy, and modern pieces for the whole family. Enjoy special deals on jeans, tees & more! | NA | NA | NA |
| https://www.ralphlauren.com/ | Access to this page has been denied | px-captcha | NA | NA | NA |
| https://www.shein.com/ | Women’s Clothing, Women Fashion Sale | SHEIN USA | Exclusive discounts and the latest trends at SHEIN — Women’s Dresses, Tops, Bottoms, Shoes, Plus Size and more. ✓ Free Shipping ✓ App only. 15% Off With No Min.+✓Easy returns ✓ New Arrivals Dropped Daily ✓ | NA | NA | NA |
Fawole, J. (Sep. 3, 2025). “Web Scraping in SEO: The 7 Most Common Use Cases.” ScraperAPI. https://www.scraperapi.com/blog/seo-scraping/
Reyes, F. (n.d.) “Web Scraping in SEO: What It Is and How It’s Used.” Lupage Digital. https://www.lupagedigital.com/blog/scraping-seo/
“Web Scraping for SEO: Competitor Keywords & Backlink Analysis.” (Jun. 2, 2026). Tendem. https://tendem.ai/blog/web-scraping-for-seo-keywords-backlinks?segment_id=broad