Web Scraping Homework
Michael Vasquez
4/5/2021
Introduction
I am utilizing the webscraping tutorial by Saurav Kaushik to learn webscraping.
Webscraping
#Loading the rvest package
library('rvest')## Loading required package: xml2#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
#Reading the HTML code from the website
webpage <- read_html(url)#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')
#Converting the ranking data to text
rank_data <- html_text(rank_data_html)
#Let's have a look at the rankings
head(rank_data)## [1] "1." "2." "3." "4." "5." "6."#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)
#Let's have another look at the rankings
head(rank_data)## [1] 1 2 3 4 5 6#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')
#Converting the title data to text
title_data <- html_text(title_data_html)
#Let's have a look at the title
head(title_data)## [1] "Suicide Squad" "Batman v Superman: Dawn of Justice"
## [3] "Captain America: Civil War" "Captain Fantastic"
## [5] "Deadpool" "The Accountant"#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
#Let's have a look at the description data
head(description_data)## [1] "\n A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [2] "\n Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."
## [3] "\n Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."
## [4] "\n In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] "\n A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [6] "\n As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."#Data-Preprocessing: removing '\n'
description_data<-gsub("\n ","",description_data)
#Let's have another look at the description data
head(description_data)## [1] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [2] "Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."
## [3] "Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."
## [4] "In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] "A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [6] "As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.runtime')
#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)
#Let's have a look at the runtime
head(runtime_data)## [1] "123 min" "152 min" "147 min" "118 min" "108 min" "128 min"#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
#Let's have another look at the runtime data
head(runtime_data)## [1] 123 152 147 118 108 128#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')
#Converting the genre data to text
genre_data <- html_text(genre_data_html)
#Let's have a look at the runtime
head(genre_data)## [1] "\nAction, Adventure, Fantasy "
## [2] "\nAction, Adventure, Sci-Fi "
## [3] "\nAction, Adventure, Sci-Fi "
## [4] "\nComedy, Drama "
## [5] "\nAction, Adventure, Comedy "
## [6] "\nAction, Crime, Drama "genre_data <-gsub("\n","",genre_data)
#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)
#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)
#Convering each genre from text to factor
genre_data<-as.factor(genre_data)
#Let's have another look at the genre data
head(genre_data)## [1] Action Action Action Comedy Action Action
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
#Converting the ratings data to text
rating_data <- html_text(rating_data_html)
#Let's have a look at the ratings
head(rating_data)## [1] "6.0" "6.4" "7.8" "7.9" "8.0" "7.3"#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)
#Let's have another look at the ratings data
head(rating_data)## [1] 6.0 6.4 7.8 7.9 8.0 7.3#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
#Converting the votes data to text
votes_data <- html_text(votes_data_html)
#Let's have a look at the votes data
head(votes_data)## [1] "612,320" "643,308" "676,198" "194,572" "913,869" "264,406"#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)
#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)
#Let's have another look at the votes data
head(votes_data)## [1] 612320 643308 676198 194572 913869 264406directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
#Converting the directors data to text
directors_data <- html_text(directors_data_html)
#Let's have a look at the directors data
head(directors_data)## [1] "David Ayer" "Zack Snyder" "Anthony Russo" "Matt Ross"
## [5] "Tim Miller" "Gavin O'Connor"#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)
#Let's have a look at the actors data
head(actors_data)## [1] "Will Smith" "Ben Affleck" "Chris Evans" "Viggo Mortensen"
## [5] "Ryan Reynolds" "Ben Affleck"#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')
#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)
#Let's have a look at the metascore
head(metascore_data)## [1] "40 " "44 " "75 " "72 " "65 "
## [6] "51 "#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)
#Lets check the length of metascore data
length(metascore_data)## [1] 97for (i in c(18,57,100)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
#Data-Preprocessing: converting metascore to numerical
metascore_data <-as.numeric(metascore_data[1:100])## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion#Let's have another look at length of the metascore data
length(metascore_data)## [1] 100summary(metascore_data)## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 25.00 48.00 62.00 60.44 72.00 99.00 3#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)
#Let's have a look at the votes data
head(gross_data)## [1] "$325.10M" "$330.36M" "$408.08M" "$5.88M" "$363.07M" "$86.26M"#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
#Let's check the length of gross data
length(gross_data)## [1] 92summary(gross_data)## Length Class Mode
## 92 character character#Filling missing entries with NA
for (i in c(18, 67, 73, 75, 83, 87,98,100)){
a<-gross_data[1:(i-1)]
b<-gross_data[i:length(gross_data)]
gross_data<-append(a,list("NA"))
gross_data<-append(gross_data,b)
}
#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data[1:100])## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion#Let's have another look at the length of gross data
length(gross_data)## [1] 100#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data,
Director = directors_data, Actor = actors_data)
#Structure of the data frame
str(movies_df)## 'data.frame': 100 obs. of 11 variables:
## $ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
## $ Title : chr "Suicide Squad" "Batman v Superman: Dawn of Justice" "Captain America: Civil War" "Captain Fantastic" ...
## $ Description : chr "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wres"| __truncated__ "Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man." "In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and "| __truncated__ ...
## $ Runtime : num 123 152 147 118 108 128 120 116 107 116 ...
## $ Genre : Factor w/ 8 levels "Action","Adventure",..: 1 1 1 5 1 1 1 1 3 7 ...
## $ Rating : num 6 6.4 7.8 7.9 8 7.3 6.8 7.4 7.6 7.9 ...
## $ Metascore : num 40 44 75 72 65 51 67 70 81 81 ...
## $ Votes : num 612320 643308 676198 194572 913869 ...
## $ Gross_Earning_in_Mil: num 325.1 330.3 408 5.88 363 ...
## $ Director : Factor w/ 98 levels "Adam Wingard",..: 23 98 6 61 93 36 40 86 82 27 ...
## $ Actor : Factor w/ 91 levels "Aamir Khan","Alexander Skarsgård",..: 89 8 19 88 75 8 39 73 7 3 ...library(tidyverse)## -- Attaching packages --------------------------------------- tidyverse 1.3.0 --## v ggplot2 3.3.3 v purrr 0.3.4
## v tibble 3.1.0 v dplyr 1.0.4
## v tidyr 1.1.2 v stringr 1.4.0
## v readr 1.4.0 v forcats 0.5.1## Warning: package 'stringr' was built under R version 4.0.4## -- Conflicts ------------------------------------------ tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag() masks stats::lag()
## x purrr::pluck() masks rvest::pluck()library(RColorBrewer)Question 1: Based on the above data, which movie from which Genre had the longest runtime?
movies_df %>%
#group observations by genre
group_by(Genre) %>%
#slicing the to get the top runtime form each genre
slice_max(Runtime) %>%
#graph data
ggplot(aes(x = Title, y = Runtime, fill = Genre)) +
geom_bar(stat = "identity", color = "white")+
coord_flip()+
ggtitle("Top Runtimes") +
ylab("Runtime (min)")+
theme_dark()
Answer: The movies that had the longest runtime for their respective genres were The Wailing, The Lost City of Z, The Girl on the Train, Silence, La La Land,Fantastic Beasts and Where to Find Them, Dangal, and A Silent Voice: The Movie.
Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
options(scipen = 999)
movies_df %>%
#filter for observations with run times between 130 and 160
filter(Runtime >=130 & Runtime<=160) %>%
#group observations by genre
group_by(Genre) %>%
#slicing the to get the top number of votes form each genre
slice_max(Votes)%>%
#graph data
ggplot(aes(x = Votes, y = Runtime, color = Genre)) +
geom_point()+
ggtitle("Hightest Votes between a Runtime of 130 and 160 minutes") +
ylab("Runtime (min)")+
theme_dark()
Answer: Action has the highest votes.
Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
movies_df %>%
#filter for observations with run times between 100 and 120
filter(Runtime >=100 & Runtime<=120)%>%
#group observations by genre
group_by(Genre) %>%
#summarize average gross for each genre
summarise(Avg_gross = mean(Gross_Earning_in_Mil, na.rm = TRUE)) %>%
#graph data
ggplot(aes(x = Genre, y = Avg_gross, fill = Genre)) +
geom_bar(stat = "identity", color = "white")+
coord_flip()+
ggtitle("Highest Average Gross Earnings") +
ylab("Average Gross Earnings in Mil")+
theme_dark()+
theme(legend.position = "none")
Answer: Animation has the highest average gross earnings in runtime 100 to 120.
##Source
Kaushik, S. (2019, June 24). Beginner’s guide on web scraping in r (using rvest) with example. Retrieved April 06, 2021, from https://www.analyticsvidhya.com/blog/2017/03/beginners-guide-on-web-scraping-in-r-using-rvest-with-hands-on-knowledge/