URL Scrapping for SEO Insights

Universities

library(rvest)
library(stringr)
library(httr2)
library(dplyr)

## 
## Attaching package: 'dplyr'

## The following objects are masked from 'package:stats':
## 
##     filter, lag

## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union

# List of URLs to scrape
urls <- c(
  "https://www.csuci.edu/",
  "https://www.usc.edu/", 
  "https://www.chapman.edu/",
  "https://www.csun.edu/",
  "https://www.harvard.edu/",
  "https://www.upenn.edu/"
  
)

# Empty list to collect results

results <- list()

for (i in seq_along(urls)) {
  
  Sys.sleep(1.5) # polite delay between requests
  
  page <- read_html(urls[i])
  
  title_tag <- page %>%
    html_element("title") %>%
    html_text2() %>%
    str_squish()
  
  meta_desc <- page %>%
    html_element("meta[name='description']") %>%
    html_attr("content") %>%
    str_squish()
  
  h1_tags <- page %>%
    html_elements("h1") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")
  
  h2_tags <- page %>%
    html_elements("h2") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")
  
  h3_tags <- page %>%
    html_elements("h3") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")
  
  # Store this page's results as a one-row tibble
  results[[i]] <- tibble(
    url = urls[i],
    title = title_tag,
    meta_description = meta_desc,
    h1 = h1_tags,
    h2 = h2_tags,
    h3 = h3_tags
  )
}

# Combine all rows into one data frame
seo_data <- bind_rows(results)

write.csv(seo_data, file = "seo_data.csv", row.names = FALSE) 

seo_data

## # A tibble: 6 × 6
##   url                      title              meta_description h1    h2    h3   
##   <chr>                    <chr>              <chr>            <chr> <chr> <chr>
## 1 https://www.csuci.edu/   CSU Channel Islan… Welcome to Cali… Welc… Appl… Foll…
## 2 https://www.usc.edu/     University of Sou… University of S… Rese… What… Sugg…
## 3 https://www.chapman.edu/ Chapman Universit… <NA>             Home… Tour… Expe…
## 4 https://www.csun.edu/    California State … CSUN stands out… Tran… Abou… Mata…
## 5 https://www.harvard.edu/ Harvard University Harvard Univers… Harv… 1776… John…
## 6 https://www.upenn.edu/   University of Pen… Topics           Penn… News… Camp…