Week 9 Web Scrapping Tutorial
Yujie Deng
4/5/2021
This is a study assignment following the tutorial provided by Saurav Kaushik. Original post can be accessed: https://www.analyticsvidhya.com/blog/2017/03/beginners-guide-on-web-scraping-in-r-using-rvest-with-hands-on-knowledge/
#Loading the rvest package
library('rvest')
#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
#Reading the HTML code from the website
webpage <- read_html(url)#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')
#Converting the ranking data to text
rank_data <- html_text(rank_data_html)
#Let's have a look at the rankings
head(rank_data)## [1] "1." "2." "3." "4." "5." "6."#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)
#Let's have another look at the rankings
head(rank_data)## [1] 1 2 3 4 5 6#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')
#Converting the title data to text
title_data <- html_text(title_data_html)
#Let's have a look at the title
head(title_data)## [1] "Suicide Squad" "Batman v Superman: Dawn of Justice"
## [3] "Captain America: Civil War" "Captain Fantastic"
## [5] "Deadpool" "The Accountant"#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
#Let's have a look at the description data
head(description_data)## [1] "\n A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [2] "\n Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."
## [3] "\n Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."
## [4] "\n In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] "\n A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [6] "\n As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)
#Let's have another look at the description data
head(description_data)## [1] " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [2] " Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."
## [3] " Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."
## [4] " In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] " A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [6] " As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)
#Let's have a look at the runtime
head(runtime_data)## [1] "123 min" "152 min" "147 min" "118 min" "108 min" "128 min"#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
#Let's have another look at the runtime data
head(runtime_data)## [1] 123 152 147 118 108 128#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')
#Converting the genre data to text
genre_data <- html_text(genre_data_html)
#Let's have a look at the genre
head(genre_data)## [1] "\nAction, Adventure, Fantasy "
## [2] "\nAction, Adventure, Sci-Fi "
## [3] "\nAction, Adventure, Sci-Fi "
## [4] "\nComedy, Drama "
## [5] "\nAction, Adventure, Comedy "
## [6] "\nAction, Crime, Drama "#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)
#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)
#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)
#Convering each genre from text to factor
genre_data<-as.factor(genre_data)
#Let's have another look at the genre data
head(genre_data)## [1] Action Action Action Comedy Action Action
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
#Converting the ratings data to text
rating_data <- html_text(rating_data_html)
#Let's have a look at the ratings
head(rating_data)## [1] "6.0" "6.4" "7.8" "7.9" "8.0" "7.3"#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)
#Let's have another look at the ratings data
head(rating_data)## [1] 6.0 6.4 7.8 7.9 8.0 7.3#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
#Converting the votes data to text
votes_data <- html_text(votes_data_html)
#Let's have a look at the votes data
head(votes_data)## [1] "612,340" "643,308" "676,216" "194,572" "913,869" "264,406"#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)
#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)
#Let's have another look at the votes data
head(votes_data)## [1] 612340 643308 676216 194572 913869 264406#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
#Converting the directors data to text
directors_data <- html_text(directors_data_html)
#Let's have a look at the directors data
head(directors_data)## [1] "David Ayer" "Zack Snyder" "Anthony Russo" "Matt Ross"
## [5] "Tim Miller" "Gavin O'Connor"#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)
#Let's have a look at the actors data
head(actors_data)## [1] "Will Smith" "Ben Affleck" "Chris Evans" "Viggo Mortensen"
## [5] "Ryan Reynolds" "Ben Affleck"#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')
#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)
#Let's have a look at the metascore
head(metascore_data)## [1] "40 " "44 " "75 " "72 " "65 "
## [6] "51 "#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)
#Lets check the length of metascore data
length(metascore_data)## [1] 97#Filling missing entries with NA
for (i in c(18, 57, 100)){
metascore_data <- append(metascore_data, NA, i-1)
}
length(metascore_data)## [1] 100#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(unlist(metascore_data))
#Let's have another look at length of the metascore data
length(metascore_data)## [1] 100#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)
#Let's have a look at the votes data
head(gross_data)## [1] "$325.10M" "$330.36M" "$408.08M" "$5.88M" "$363.07M" "$86.26M"#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
#Let's check the length of gross data
length(gross_data)## [1] 92#Filling missing entries with NA
for (i in c(18,67,73,75,83,87,98,100)){
gross_data <- append(gross_data, NA, i-1)
}
length(gross_data)## [1] 100#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(unlist(gross_data))
#Let's have another look at the length of gross data
length(gross_data)## [1] 100#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data,
Director = directors_data, Actor = actors_data)
#Structure of the data frame
str(movies_df)## 'data.frame': 100 obs. of 11 variables:
## $ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
## $ Title : chr "Suicide Squad" "Batman v Superman: Dawn of Justice" "Captain America: Civil War" "Captain Fantastic" ...
## $ Description : chr " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defens"| __truncated__ " Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world "| __truncated__ " Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man." " In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical "| __truncated__ ...
## $ Runtime : num 123 152 147 118 108 128 120 116 107 116 ...
## $ Genre : Factor w/ 8 levels "Action","Adventure",..: 1 1 1 5 1 1 1 1 3 7 ...
## $ Rating : num 6 6.4 7.8 7.9 8 7.3 6.8 7.4 7.6 7.9 ...
## $ Metascore : num 40 44 75 72 65 51 67 70 81 81 ...
## $ Votes : num 612340 643308 676216 194572 913869 ...
## $ Gross_Earning_in_Mil: num 325.1 330.3 408 5.88 363 ...
## $ Director : Factor w/ 98 levels "Adam Wingard",..: 23 98 6 61 93 36 40 86 82 27 ...
## $ Actor : chr "Will Smith" "Ben Affleck" "Chris Evans" "Viggo Mortensen" ...Question 1. Based on the above data, which movie from which Genre had the longest runtime?
library(ggplot2)qplot(data = movies_df, Runtime, fill = Genre, bins = 30)
library(tidyverse)## -- Attaching packages --------------------------------------- tidyverse 1.3.0 --## v tibble 3.0.6 v dplyr 1.0.4
## v tidyr 1.1.2 v stringr 1.4.0
## v readr 1.4.0 v forcats 0.5.1
## v purrr 0.3.4## -- Conflicts ------------------------------------------ tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag() masks stats::lag()movies_df %>%
filter(Runtime > 160)## Rank Title
## 1 49 Silence
## 2 57 Dangal
## Description
## 1 In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism.
## 2 Former wrestler Mahavir Singh Phogat and his two wrestler daughters struggle towards glory at the Commonwealth Games in the face of societal oppression.
## Runtime Genre Rating Metascore Votes Gross_Earning_in_Mil Director
## 1 161 Drama 7.2 79 101741 7.10 Martin Scorsese
## 2 161 Action 8.4 NA 160035 12.39 Nitesh Tiwari
## Actor
## 1 Andrew Garfield
## 2 Aamir KhanSilence from Drama genre and Dangal from Action genre had the longest runtime.
Question 2. Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
Action movies has the highest votes.
movies_df %>%
filter(Runtime > 130 & Runtime < 160) %>%
group_by(Genre) %>%
summarise(sum(Votes))## # A tibble: 5 x 2
## Genre `sum(Votes)`
## * <fct> <dbl>
## 1 Action 2682907
## 2 Adventure 415329
## 3 Biography 666087
## 4 Drama 522867
## 5 Horror 282291Question 3. Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))## Warning: Removed 8 rows containing missing values (geom_point).
movies_df %>%
filter(Runtime > 100 & Runtime < 120) %>%
group_by(Genre) %>%
summarise(mean(Gross_Earning_in_Mil, na.rm = TRUE))## # A tibble: 8 x 2
## Genre `mean(Gross_Earning_in_Mil, na.rm = TRUE)`
## * <fct> <dbl>
## 1 Action 72.7
## 2 Adventure 142.
## 3 Animation 216.
## 4 Biography 35.9
## 5 Comedy 13.4
## 6 Crime 75.4
## 7 Drama 49.3
## 8 Horror 69.8Animation genre has the highest average gross earnings.