Beginner’s Guide on Web Scraping in R (using rvest) with hnads-on example by Saurav Kaushik

Website: Analytics Vidhya URL: https://www.analyticsvidhya.com/blog/2017/03/beginners-guide-on-web-scraping-in-r-using-rvest-with-hands-on-knowledge/

Install Packages

#install.packages('rvest')

Using the CSS selector that contains the rankings, you can use the simple R code to get all the rankings:

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage, '.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."

Preprocess data to convert it to numerical format:

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6

Use the corresponding CSS selector for the titles - .lister-itme-header a. Use this selector to scrape all the titles using the following code:

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage, '.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Arrival"                       "Suicide Squad"                
## [3] "Train to Busan"                "Ghostbusters: Answer the Call"
## [5] "The Conjuring 2"               "Hush"

Use the corresponding CSS selector for scraping - Description data:

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\nA linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appear around the world."                                                                           
## [2] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                          
## [3] "\nWhile a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."                                                                                             
## [4] "\nFollowing a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat."
## [5] "\nEd and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                                               
## [6] "\nA deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."
#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data
head(description_data)
## [1] "A linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appear around the world."                                                                           
## [2] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                          
## [3] "While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."                                                                                             
## [4] "Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat."
## [5] "Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                                               
## [6] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."

Use the corresponding CSS selector for scraping - Runtime data:

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look atthe runtime
head(runtime_data)
## [1] "116 min" "123 min" "118 min" "117 min" "134 min" "82 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub("min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 116 123 118 117 134  82

Use the corresponding CSS selector for scraping - Movie genre data:

#Using CSS selectors to screape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nDrama, Sci-Fi            "             
## [2] "\nAction, Adventure, Fantasy            "
## [3] "\nAction, Horror, Thriller            "  
## [4] "\nAction, Comedy, Fantasy            "   
## [5] "\nHorror, Mystery, Thriller            " 
## [6] "\nHorror, Thriller            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Drama  Action Action Action Horror Horror
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror

Use the corresponding CSS selector for scraping - IMDB rating data:

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "7.9" "5.9" "7.6" "6.5" "7.3" "6.6"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have nother look at the ratings data
head(rating_data)
## [1] 7.9 5.9 7.6 6.5 7.3 6.6

Use the corresponding CSS selector for scraping - Votes data:

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "644,825" "655,642" "194,001" "214,998" "251,992" "119,754"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-PreProcessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1] 644825 655642 194001 214998 251992 119754

Use the corresponding CSS selector for scraping - Directors data:

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Denis Villeneuve" "David Ayer"       "Sang-ho Yeon"     "Paul Feig"       
## [5] "James Wan"        "Mike Flanagan"
#Data-Preprocessing: converting direcotrs data into factors
directors_data<-as.factor(directors_data)

Use the corresponding CSS selector for scraping - Actors data:

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Amy Adams"          "Will Smith"         "Gong Yoo"          
## [4] "Melissa McCarthy"   "Vera Farmiga"       "John Gallagher Jr."
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

Use the corresponding CSS selector for scraping - Metascore data:

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore
head(metascore_data)
## [1] "81        " "40        " "72        " "60        " "65        "
## [6] "67        "
#Data-Preprocessing: removing extra space removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Let's check the length of metascore data
length(metascore_data)
## [1] 96

It is a practical situation which can arise while scraping any website. Unfortunately, if we simply add NA’s to last 4 entries, it will map NA as Metascore for movies 96 to 100 while in reality, the data is missing for some other movies.Use the following function to get around this problem.

for (i in c(34,43,80,96)){
  a<-metascore_data[1:(i-1)]
  b<-metascore_data[i:length(metascore_data)]
  metascore_data<-append(a,list("NA"))
  metascore_data<-append(metascore_data,b)
}

#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at length of the metascore data

length(metascore_data)
## [1] 100
#Let's look at summary statistics
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   18.00   47.00   62.00   60.43   74.25   99.00       4

The same thing happens with the Gross variable which represents gross earnings of that movie in millions. I have use the same solution to work my way around:

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)
## [1] "$100.55M" "$325.10M" "$2.13M"   "$128.34M" "$102.47M" "$89.22M"
#Data_Preprocessing: removing '$' and "M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)
## [1] 89
#Filling missing entries with NA
for (i in c(6,25,34,43,50,66,73,77,80,86,99)){
  a<-gross_data[1:(i-1)]
  b<-gross_data[i:length(gross_data)]
  gross_data<-append(a,list("NA"))
  gross_data<-append(gross_data,b)
}

#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at the length of gross data
length(gross_data)
## [1] 100
summary(gross_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.18   14.27   54.65   96.38  125.00  532.10      11

Analyzing scraped data from the web

library('ggplot2')

qplot(data = movies_df,Runtime,fill = Genre,bins = 30)

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

Answer: Drama and Action movies timed for the longest runtime at 160 minutes.

ggplot(movies_df,aes(x=Runtime, y=Rating))+
  geom_point(aes(size=Votes,col=Genre))

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

Answer: Action movies has the highest votes at 160 minutes.

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
  geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 11 rows containing missing values (geom_point).

#### Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120. Answer: Action,Adventure, and Animation tie with Gross Earnings at 350 million with runtimes between 100 to 120 minutes.