scraping urls for quick SEO inisghts

library(rvest)
library(stringr)
library(httr2)
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
# List of URLs to scrape
urls <- c(
  "https://www.callutheran.edu/",
  "https://www.csuci.edu/", 
  "https://www.csun.edu/"
)

# Empty list to collect results
results <- list()

for (i in seq_along(urls)) {
  
  Sys.sleep(1.5) # polite delay between requests
  
  page <- read_html(urls[i])
  
  title_tag <- page %>%
    html_element("title") %>%
    html_text2() %>%
    str_squish()
 title_tag 
  meta_desc <- page %>%
    html_element("meta[name='description']") %>%
    html_attr("content") %>%
    str_squish()
 meta_desc
 
  h1_tags <- page %>%
    html_elements("h1") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")
h1_tags
  
  h2_tags <- page %>%
    html_elements("h2") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")

  h2_tags
    
  h3_tags <- page %>%
    html_elements("h3") %>%
    html_text2() %>%
    str_squish() %>%
    str_c(collapse = " | ")
  
  h3_tags
  
  # Store this page's results as a one-row tibble
  results[[i]] <- tibble(
    url = urls[i],
    title = title_tag,
    meta_description = meta_desc,
    h1 = h1_tags,
    h2 = h2_tags,
    h3 = h3_tags
  )
}

# Combine all rows into one data frame
seo_data <- bind_rows(results)

seo_data
## # A tibble: 3 × 6
##   url                          title          meta_description h1    h2    h3   
##   <chr>                        <chr>          <chr>            <chr> <chr> <chr>
## 1 https://www.callutheran.edu/ California Lu… Cal Lutheran of… Wher… What… Unde…
## 2 https://www.csuci.edu/       CSU Channel I… Welcome to Cali… Welc… Appl… Foll…
## 3 https://www.csun.edu/        California St… CSUN stands out… Tran… Abou… Mata…
write.csv(seo_data, file = "seo_data.csv", row.names = FALSE)

seo_data$title
## [1] "California Lutheran University"         
## [2] "CSU Channel Islands"                    
## [3] "California State University, Northridge"
seo_data$meta_description
## [1] "Cal Lutheran offers undergraduate and graduate programs to students who are seeking to grow as individuals while they pursue their passions and discover their purpose."
## [2] "Welcome to California State University Channel Islands (CSUCI) in Ventura County. Explore our diverse academic programs, campus life, and community engagement."        
## [3] "CSUN stands out among Southern California universities with its dedication to student success, diversity, and community involvement. Get started now!"
seo_data$h1
## [1] "Where your dreams take flight."                     
## [2] "Welcome to Cal State Channel Islands"               
## [3] "Transfer to California State University, Northridge"
seo_data$h2
## [1] "What are you looking for? | Undergraduate Majors | Graduate Programs | Bachelor's Degree for Professionals"                                                                                                                                                                                                                                         
## [2] "Apply Now | Find Degrees | See Tuition & Fees | Get to Know CI’s Unique Campus | Fast Facts | Calendars & Deadlines | Request More Information | EventsMore | NewsMore | Explore CSUCI | Food and Housing Insecurity | COVID-19 Campus Information | Information for… | Website & University Information | Proudly a part of the CSU | We’re Social"
## [3] "About Overview | Academics Overview | Admissions & Financial Aid Overview | Student Life Overview | Explore Research | Ready to find your college major? | Where You Belong: Student Life at CSUN | You Belong at CSUN | CSUN Newsroom | CSUN Events"
seo_data$h3
## [1] "Undergraduate Majors | Graduate Programs | Bachelor's Degree for Professionals"                                                                                                                                                                                                                                                   
## [2] "Following in Her Grandparents’ Footsteps | Alum Builds Cyber Security Career | First-Gen Student to Financial Literacy Advocate | From Behind the Pharmacy Counter to Pharmaceutical Leader"                                                                                                                                      
## [3] "Matadors Honoring Their Roots | An Evening of Gratitude, Inspiration, and Impact: CSUN Honors Donors at The Soraya | CSUN’s Teenage Drama Workshop Marks its 70th Anniversary with Two Productions About the Magic of ‘Family’ | Juneteenth Symposium Unites CSU Campuses in Celebration and Reflection | Jul 9 | Jul 10 | Jul 16"

Summary:

CLU seems to have more vocabulary talking to their audience the student while CSUCI seems to be more about sharing about what it has to offer. Gives more of a sales pitch than bringing the student into the story.By using words like take flight, and painting the picture for the students to attend their school.