Web Scraping imdb

library(rvest)
library(dplyr)
library(purrr)

link<-"https://www.imdb.com/search/title/?genres=adventure&explore=title_type,genres&pf_rd_m=A2FGELUUNOQJNL&pf_rd_p=fd0c0dd4-de47-4168-baa8-239e02fd9ee7&pf_rd_r=T90VBQ6SRMZPT58WHWZE&pf_rd_s=center-4&pf_rd_t=15051&pf_rd_i=genre&ref_=ft_gnr_pr4_i_2"

page<- read_html(link)

#If possible, select the node that encloses all required nodes and iterating over them to extract needed data

df<- page %>% 
  html_nodes(".mode-advanced") %>% 
  map_df(~list(title = html_nodes(.x, '.lister-item-header a') %>% 
                     html_text() %>% 
                     {if(length(.) == 0) NA else .},
               year = html_nodes(.x, '.text-muted.unbold') %>% 
                     html_text() %>% 
                     {if(length(.) == 0) NA else .},
               rating = html_nodes(.x, '.ratings-imdb-rating strong') %>% 
                     html_text() %>% 
                     {if(length(.) == 0) NA else .},
               synopsis = html_nodes(.x, '.ratings-bar+ .text-muted') %>% 
                     html_text() %>% 
                     {if(length(.) == 0) NA else .}))

#write.csv(df, "movies.csv")

head(df)

## # A tibble: 6 x 4
##   title              year  rating synopsis                                      
##   <chr>              <chr> <chr>  <chr>                                         
## 1 Dune               (202~ 8.3    "\nFeature adaptation of Frank Herbert's scie~
## 2 No Time to Die     (202~ 7.6    "\nJames Bond has left active service. His pe~
## 3 Dune               (198~ 6.4    "\nA Duke's son leads desert warriors against~
## 4 Venom: Let There ~ (202~ 6.3    "\nEddie Brock attempts to reignite his caree~
## 5 Free Guy           (202~ 7.2    "\nA bank teller discovers that he's actually~
## 6 Eternals           (202~ <NA>    <NA>

#To scrape multiple pages

#go to the second page and adjust the start number in the link to 1. Copy that new link

Create big for loop, sequential increase by 50 paste0 excludes spaces between pasted components. Alternatively, can use sep="" go to the second page and adjust the start number to 1. Copy that new link

#start with empty data frame
movies = data.frame()

for(page_result in seq(from = 1, to = 101, by = 50)){
  
  link = paste0("https://www.imdb.com/search/title/?genres=adventure&start=", page_result,    
                "&explore=title_type,genres&ref_=adv_nxt")
  
  page<- read_html(link)
  
  df<- page %>% 
  html_nodes(".mode-advanced") %>% 
  map_df(~list(title = html_nodes(.x, '.lister-item-header a') %>% 
                     html_text() %>% 
                     {if(length(.) == 0) NA else .},
               year = html_nodes(.x, '.text-muted.unbold') %>% 
                     html_text() %>% 
                     {if(length(.) == 0) NA else .},
               rating = html_nodes(.x, '.ratings-imdb-rating strong') %>% 
                     html_text() %>% 
                     {if(length(.) == 0) NA else .},
               synopsis = html_nodes(.x, '.ratings-bar+ .text-muted') %>% 
                     html_text() %>% 
                     {if(length(.) == 0) NA else .}))

movies = rbind(movies, df)

#track progress while processing
print(paste("Page:", page_result))
  
}

## [1] "Page: 1"
## [1] "Page: 51"
## [1] "Page: 101"

head (movies)

## # A tibble: 6 x 4
##   title              year  rating synopsis                                      
##   <chr>              <chr> <chr>  <chr>                                         
## 1 Dune               (202~ 8.3    "\nFeature adaptation of Frank Herbert's scie~
## 2 No Time to Die     (202~ 7.6    "\nJames Bond has left active service. His pe~
## 3 Dune               (198~ 6.4    "\nA Duke's son leads desert warriors against~
## 4 Venom: Let There ~ (202~ 6.3    "\nEddie Brock attempts to reignite his caree~
## 5 Free Guy           (202~ 7.2    "\nA bank teller discovers that he's actually~
## 6 Eternals           (202~ <NA>    <NA>