HW_WebScraping
Isabel Decolin
11/3/2020
Packages
library(rvest)## Loading required package: xml2library(tidyverse)## ── Attaching packages ────────────────────────────────────────────────────────────────────────────────────── tidyverse 1.3.0 ──## ✓ ggplot2 3.3.2 ✓ purrr 0.3.4
## ✓ tibble 3.0.3 ✓ dplyr 1.0.2
## ✓ tidyr 1.1.2 ✓ stringr 1.4.0
## ✓ readr 1.3.1 ✓ forcats 0.5.0## ── Conflicts ───────────────────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
## x dplyr::filter() masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag() masks stats::lag()
## x purrr::pluck() masks rvest::pluck()library(plotly)##
## Attaching package: 'plotly'## The following object is masked from 'package:ggplot2':
##
## last_plot## The following object is masked from 'package:stats':
##
## filter## The following object is masked from 'package:graphics':
##
## layoutScraping webpage using R
IMDb website for 100 most popular films released in 2016.
# Specifying url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
# Reading the HTML code from the website
webpage <- read_html(url)Scraping the Rank field
Select all the rankings then copy the corresponding CSS selector.
# Using CSS selectors to scrape the rankings slection
rank_data_html <- html_nodes(webpage,'.text-primary')
# Convert the ranking data to text
rank_data <- html_text(rank_data_html)
# Look at the rankings
head(rank_data)## [1] "1." "2." "3." "4." "5." "6."# Convert rankings to numerical
rank_data <- as.numeric(rank_data)
# One more look at the rankings
head(rank_data)## [1] 1 2 3 4 5 6Titles
Select all the titles using the selector.
# Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage, '.lister-item-header a')
# Convert title data to text
title_data <- html_text(title_data_html)
# Look at the titles
head(title_data)## [1] "Split" "Ghostbusters: Answer the Call"
## [3] "Suicide Squad" "Train to Busan"
## [5] "The Conjuring 2" "Hush"Scraping for Description
Select the movie descriptions.
# CSS selector to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
# Convert data to text
description_data <- html_text(description_data_html)
head(description_data)## [1] "\n Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."
## [2] "\n Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat."
## [3] "\n A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] "\n While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."
## [5] "\n Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."
## [6] "\n A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."# Removing '\n'
description_data<-gsub("\n","",description_data)
#Look at the description data again
head(description_data)## [1] " Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."
## [2] " Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat."
## [3] " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] " While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."
## [5] " Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."
## [6] " A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."Scraping for Runtime
Select moive runtimes.
# Use CSS selector to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
# Convert data to text
runtime_data <- html_text(runtime_data_html)
head(runtime_data)## [1] "117 min" "117 min" "123 min" "118 min" "134 min" "82 min"# Remove mins and convert to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
# Look at data
head(runtime_data)## [1] 117 117 123 118 134 82Movie Genres
Scraping for movie genres
# Using CSS selector to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')
# Convert to text
genre_data <- html_text(genre_data_html)
head(genre_data)## [1] "\nHorror, Thriller "
## [2] "\nAction, Comedy, Fantasy "
## [3] "\nAction, Adventure, Fantasy "
## [4] "\nAction, Horror, Thriller "
## [5] "\nHorror, Mystery, Thriller "
## [6] "\nHorror, Thriller "#Removing \n and removing excess spaces
genre_data<-gsub("\n","",genre_data)
genre_data<-gsub(" ","",genre_data)
# Taking only the first genere of each movie
genre_data<-gsub(",.*","",genre_data)
# Convert from text to factor
genre_data<-as.factor(genre_data)
#Look at data
head(genre_data)## [1] Horror Action Action Action Horror Horror
## Levels: Action Adventure Animation Biography Comedy Crime Drama HorrorIMDB ratings
Select the IMDB ratings section
# Using CSS selctor to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
# Convert ratings data to text
rating_data <- html_text(rating_data_html)
head(rating_data)## [1] "7.3" "6.5" "6.0" "7.6" "7.3" "6.6"# Convert ratings to numerical
rating_data<-as.numeric(rating_data)
#Look at data
head(rating_data)## [1] 7.3 6.5 6.0 7.6 7.3 6.6Votes
Look at IMDB votes section
# Use CSS selector to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
# Convert data to text
votes_data <- html_text(votes_data_html)
head(votes_data)## [1] "413,315" "201,700" "591,535" "158,265" "221,153" "100,248"# Remove the commas
votes_data<-gsub(",","",votes_data)
# Convert votes to numerical
votes_data<-as.numeric(votes_data)
# Look at the data
head(votes_data)## [1] 413315 201700 591535 158265 221153 100248Directors and Actors
Look at the movie directors
# Use CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
# Convert directors data to text
directors_data <- html_text(directors_data_html)
head(directors_data)## [1] "M. Night Shyamalan" "Paul Feig" "David Ayer"
## [4] "Sang-ho Yeon" "James Wan" "Mike Flanagan"# Convert directors data into factors
directors_data<-as.factor(directors_data)
# Look at data
head(directors_data)## [1] M. Night Shyamalan Paul Feig David Ayer Sang-ho Yeon
## [5] James Wan Mike Flanagan
## 96 Levels: Adam Wingard Alex Proyas André Øvredal Andrea Arnold ... Zack Snyder# Use CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
# Convert actors data into text
actors_data <- html_text(actors_data_html)
# Convert actors data into factors
actors_data<-as.factor(actors_data)
# Look at data
head(actors_data)## [1] James McAvoy Melissa McCarthy Will Smith Yoo Gong
## [5] Vera Farmiga John Gallagher Jr.
## 91 Levels: Aamir Khan Adam Sandler Alexander Skarsgård ... Yoo GongMetascore data
Scraping of metascore data
# Using CSS selector to scrape the meta score section
metascore_data_html <- html_nodes(webpage,'.metascore')
# Convert data to text
metascore_data <- html_text(metascore_data_html)
head(metascore_data)## [1] "62 " "60 " "40 " "72 " "65 "
## [6] "67 "# Remove extra space in metascore
metascore_data<-gsub(" ","",metascore_data)
# Look at length of metascore data
length(metascore_data)## [1] 964 movies don’t have corresponding Metascore fields. Will require manual manipulation.
for (i in c(33,58,74,89)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
# Converting metascore to numerical
metascore_data<-as.numeric(metascore_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion# Look at data
length(metascore_data)## [1] 100# Look at summary statistics
summary(metascore_data)## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 22.00 47.00 60.00 59.54 72.00 99.00 4Gross variable
Gross earnings of a movie in millions. Same treatment as metascore to account for movies missing information.
# Use CSS selector to scrape gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
# Convert gross revenue data to text
gross_data <- html_text(gross_data_html)
head(gross_data)## [1] "$138.29M" "$128.34M" "$325.10M" "$2.13M" "$102.47M" "$6.86M"# Removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
# Length of gross data
length(gross_data)## [1] 90#Filling missing entries with NA
for (i in c(6,11,33,53,58,80,85,86,88, 94)){
a<-gross_data[1:(i-1)]
b<-gross_data[i:length(gross_data)]
gross_data<-append(a,list("NA"))
gross_data<-append(gross_data,b)
}
#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion#Let's have another look at the length of gross data
length(gross_data)## [1] 100summary(gross_data)## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 0.02 10.71 53.75 86.42 101.92 532.10 10Combination
Combine all scraped features into a data frame.
#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data,
Director = directors_data, Actor = actors_data)
#Structure of the data frame
str(movies_df)## 'data.frame': 100 obs. of 11 variables:
## $ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
## $ Title : chr "Split" "Ghostbusters: Answer the Call" "Suicide Squad" "Train to Busan" ...
## $ Description : chr " Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape befo"| __truncated__ " Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engine"| __truncated__ " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defens"| __truncated__ " While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan." ...
## $ Runtime : num 117 117 123 118 134 82 83 108 144 107 ...
## $ Genre : Factor w/ 8 levels "Action","Adventure",..: 8 1 1 1 8 8 1 1 1 3 ...
## $ Rating : num 7.3 6.5 6 7.6 7.3 6.6 6.2 8 6.9 7.6 ...
## $ Metascore : num 62 60 40 72 65 67 44 65 52 81 ...
## $ Votes : num 413315 201700 591535 158265 221153 ...
## $ Gross_Earning_in_Mil: num 138.2 128.3 325.1 2.13 102.4 ...
## $ Director : Factor w/ 96 levels "Adam Wingard",..: 54 69 21 79 40 61 52 90 12 76 ...
## $ Actor : Factor w/ 91 levels "Aamir Khan","Adam Sandler",..: 38 60 90 91 88 42 76 74 38 7 ...head(movies_df)## Rank Title
## 1 1 Split
## 2 2 Ghostbusters: Answer the Call
## 3 3 Suicide Squad
## 4 4 Train to Busan
## 5 5 The Conjuring 2
## 6 6 Hush
## Description
## 1 Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th.
## 2 Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat.
## 3 A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse.
## 4 While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan.
## 5 Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit.
## 6 A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window.
## Runtime Genre Rating Metascore Votes Gross_Earning_in_Mil
## 1 117 Horror 7.3 62 413315 138.20
## 2 117 Action 6.5 60 201700 128.30
## 3 123 Action 6.0 40 591535 325.10
## 4 118 Action 7.6 72 158265 2.13
## 5 134 Horror 7.3 65 221153 102.40
## 6 82 Horror 6.6 67 100248 NA
## Director Actor
## 1 M. Night Shyamalan James McAvoy
## 2 Paul Feig Melissa McCarthy
## 3 David Ayer Will Smith
## 4 Sang-ho Yeon Yoo Gong
## 5 James Wan Vera Farmiga
## 6 Mike Flanagan John Gallagher Jr.Analyzing scraped data from the web
Follow the visualizations and answer the questions given below.
qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
Question 1: Based on the above data, which movie from which Genre had the longest runtime?
q1 <-movies_df %>% select(Title, Rank, Title, Runtime, Genre) %>%
filter(Runtime == max(Runtime))
q1## Title Rank Runtime Genre
## 1 American Honey 82 163 DramaAmerican Honey from the Drama genre had the longest runtime of 163 minutes.
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
Question 2: Based on the above data, in the runtime of 130-160 mins, which genre had the highest votes?
q2 <- movies_df %>% select(Title,Rank, Runtime, Votes, Genre) %>% filter(between(Runtime, 130, 160))
q2_plot <- q2 %>% ggplot(aes(x=Genre, y= Votes)) +
geom_bar(stat='identity') +
xlab("Movie Genre") +
ylab("Votes") +
ggtitle("Movie Genres by Vote") +
coord_flip()
q2_plot
q2_df <- q2 %>% filter(Votes == max(Votes))
q2_df ## Title Rank Runtime Votes Genre
## 1 Captain America: Civil War 38 147 650963 ActionAction is the genre of the most votes in the 130-160 min runtime range. Specifically from that genre, Captain America:Civil War had the most votes.
ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))## Warning: Removed 10 rows containing missing values (geom_point).
Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
q3 <- movies_df %>% select(Runtime,Genre, Gross_Earning_in_Mil) %>% drop_na() %>%
filter(between(Runtime, 100, 120)) %>%
group_by(Genre) %>%
summarize(avgGross = mean(Gross_Earning_in_Mil)) ## `summarise()` ungrouping output (override with `.groups` argument)q3## # A tibble: 8 x 2
## Genre avgGross
## <fct> <dbl>
## 1 Action 82.0
## 2 Adventure 149.
## 3 Animation 216.
## 4 Biography 35.9
## 5 Comedy 48.1
## 6 Crime 41.3
## 7 Drama 49.8
## 8 Horror 46.8Animation has the highest average gross for the runtime 100-120 minutes.