In Class Text Analysis & NLP 3 - Word Co-Occurrences for SEO

library(rvest)
library(stringr)
library(httr2)
library(dplyr)

## 
## Attaching package: 'dplyr'

## The following objects are masked from 'package:stats':
## 
##     filter, lag

## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union

# List of URLs to scrape
urls <- c(
  "https://www.target.com",
  "https://www.walmart.com",
  "https://www.amazon"
)

# Empty list to collect results
results <- list()

for (i in seq_along(urls)) {
  
  Sys.sleep(1.5) # polite delay between requests
  
  page <- read_html(urls[i])
  
  title_tag <- page %>%
    html_element("title") %>%
    html_text2() %>%
    str_squish()
  
  meta_desc <- page %>%
    html_element("meta[name='description']") %>%
    html_attr("content") %>%
    str_squish()
  
  h1_tags <- page %>%
    html_elements("h1") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")
  
  h2_tags <- page %>%
    html_elements("h2") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")
  
  h3_tags <- page %>%
    html_elements("h3") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")
  
  # Store this page's results as a one-row tibble
  results[[i]] <- tibble(
    url = urls[i],
    title = title_tag,
    meta_description = meta_desc,
    h1 = h1_tags,
    h2 = h2_tags,
    h3 = h3_tags
  )
}

# Combine all rows into one data frame
seo_data <- bind_rows(results)

seo_data

## # A tibble: 3 × 6
##   url                     title               meta_description h1    h2    h3   
##   <chr>                   <chr>               <chr>            <chr> <chr> <chr>
## 1 https://www.target.com  Target : Expect Mo… Shop Target onl… "Hom… "End… "Loa…
## 2 https://www.walmart.com Walmart | Save Mon… Shop Walmart.co… "Wal… "Dis… "Bri…
## 3 https://www.amazon      Amazon.com          <NA>             ""    ""    ""

write.csv(seo_data, file = "seo_data.csv", row.names = FALSE)

seo_data

## # A tibble: 3 × 6
##   url                     title               meta_description h1    h2    h3   
##   <chr>                   <chr>               <chr>            <chr> <chr> <chr>
## 1 https://www.target.com  Target : Expect Mo… Shop Target onl… "Hom… "End… "Loa…
## 2 https://www.walmart.com Walmart | Save Mon… Shop Walmart.co… "Wal… "Dis… "Bri…
## 3 https://www.amazon      Amazon.com          <NA>             ""    ""    ""

Results

For this lab exercise, I scraped the h1, h2, and h3 for Target, Walmart, and Amazon. Target and Walmart differed in their h1 values. Target showed “homepage”, whereas Walmart showed their slogan. For h2, Target shows their trending deals and new products, whereas Walmart has their different departments listed. For h3, Target is showing “Loading…” which I assumes is supposed to be links for their popular products, whereas Walmart shows their current deals on items. Amazon did not return anything for h1, h2, and h3.