Homework Assignment Week 2 - Web Scraping
Minh Tu Bui
6/18/2020
#Loading the rvest package
library('rvest')## Loading required package: xml2## Warning: package 'xml2' was built under R version 3.6.3library(tidyverse)## Warning: package 'tidyverse' was built under R version 3.6.3## -- Attaching packages ---------------------------------------------------------------------------- tidyverse 1.3.0 --## v ggplot2 3.3.1 v purrr 0.3.4
## v tibble 3.0.1 v dplyr 1.0.0
## v tidyr 1.1.0 v stringr 1.4.0
## v readr 1.3.1 v forcats 0.5.0## Warning: package 'ggplot2' was built under R version 3.6.3## Warning: package 'tibble' was built under R version 3.6.3## Warning: package 'tidyr' was built under R version 3.6.3## Warning: package 'purrr' was built under R version 3.6.3## Warning: package 'dplyr' was built under R version 3.6.3## Warning: package 'forcats' was built under R version 3.6.3## -- Conflicts ------------------------------------------------------------------------------- tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag() masks stats::lag()
## x purrr::pluck() masks rvest::pluck()library('ggplot2')
#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
#Reading the HTML code from the website
webpage <- read_html(url)Ranking Section
#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')
#Converting the ranking data to text
rank_data <- html_text(rank_data_html)
head(rank_data)## [1] "1." "2." "3." "4." "5." "6."#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)
head(rank_data)## [1] 1 2 3 4 5 6Title Section
#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')
#Converting the title data to text
title_data <- html_text(title_data_html)
head(title_data)## [1] "Moana"
## [2] "Moonlight"
## [3] "Suicide Squad"
## [4] "Rogue One: A Star Wars Story"
## [5] "Miss Peregrine's Home for Peculiar Children"
## [6] "La La Land"Description Section
#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
head(description_data)## [1] "\n In Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches Moana's island, she answers the Ocean's call to seek out the Demigod to set things right."
## [2] "\n A young African-American man grapples with his identity and sexuality while experiencing the everyday struggles of childhood, adolescence, and burgeoning adulthood."
## [3] "\n A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] "\n The daughter of an Imperial scientist joins the Rebel Alliance in a risky move to steal the Death Star plans."
## [5] "\n When Jacob (Asa Butterfield) discovers clues to a mystery that stretches across time, he finds Miss Peregrine's Home for Peculiar Children. But the danger deepens after he gets to know the residents and learns about their special powers."
## [6] "\n While navigating their careers in Los Angeles, a pianist and an actress fall in love while attempting to reconcile their aspirations for the future."#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)
head(description_data)## [1] " In Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches Moana's island, she answers the Ocean's call to seek out the Demigod to set things right."
## [2] " A young African-American man grapples with his identity and sexuality while experiencing the everyday struggles of childhood, adolescence, and burgeoning adulthood."
## [3] " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] " The daughter of an Imperial scientist joins the Rebel Alliance in a risky move to steal the Death Star plans."
## [5] " When Jacob (Asa Butterfield) discovers clues to a mystery that stretches across time, he finds Miss Peregrine's Home for Peculiar Children. But the danger deepens after he gets to know the residents and learns about their special powers."
## [6] " While navigating their careers in Los Angeles, a pianist and an actress fall in love while attempting to reconcile their aspirations for the future."Runtime Section
#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)
head(runtime_data)## [1] "107 min" "111 min" "123 min" "133 min" "127 min" "128 min"#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
head(runtime_data)## [1] 107 111 123 133 127 128Genre Section
#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')
#Converting the genre data to text
genre_data <- html_text(genre_data_html)
head(genre_data)## [1] "\nAnimation, Adventure, Comedy "
## [2] "\nDrama "
## [3] "\nAction, Adventure, Fantasy "
## [4] "\nAction, Adventure, Sci-Fi "
## [5] "\nAdventure, Drama, Family "
## [6] "\nComedy, Drama, Music "#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)
#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)
#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)
#Convering each genre from text to factor
genre_data<-as.factor(genre_data)
head(genre_data)## [1] Animation Drama Action Action Adventure Comedy
## Levels: Action Adventure Animation Biography Comedy Crime Drama HorrorRating Section
#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
#Converting the ratings data to text
rating_data <- html_text(rating_data_html)
head(rating_data)## [1] "7.6" "7.4" "6.0" "7.8" "6.7" "8.0"#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)
head(rating_data)## [1] 7.6 7.4 6.0 7.8 6.7 8.0Votes Section
#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
#Converting the votes data to text
votes_data <- html_text(votes_data_html)
head(votes_data)## [1] "254,974" "258,640" "580,793" "532,959" "150,551" "480,751"#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)
#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)
head(votes_data)## [1] 254974 258640 580793 532959 150551 480751Directors Section
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
#Converting the directors data to text
directors_data <- html_text(directors_data_html)
head(directors_data)## [1] "Ron Clements" "Barry Jenkins" "David Ayer" "Gareth Edwards"
## [5] "Tim Burton" "Damien Chazelle"#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
head(directors_data)## [1] Ron Clements Barry Jenkins David Ayer Gareth Edwards
## [5] Tim Burton Damien Chazelle
## 98 Levels: Alex Proyas Ana Lily Amirpour André Øvredal ... Zack SnyderActors Section
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)
head(actors_data)## [1] "Auli'i Cravalho" "Mahershala Ali" "Will Smith" "Felicity Jones"
## [5] "Eva Green" "Ryan Gosling"#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
head(actors_data)## [1] Auli'i Cravalho Mahershala Ali Will Smith Felicity Jones
## [5] Eva Green Ryan Gosling
## 92 Levels: Aamir Khan Adam Driver Adam Sandler ... Zoey DeutchMetascore Section
#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')
#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)
head(metascore_data)## [1] "81 " "99 " "40 " "65 " "57 "
## [6] "94 "#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)
length(metascore_data)## [1] 98# Adding NA's to missing values
for (i in c(22, 80)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercionlength(metascore_data)## [1] 100#Summary statistics
summary(metascore_data)## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 21.00 47.25 62.00 60.19 73.50 99.00 2Gross Revenue Section
#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)
head(gross_data)## [1] "$248.76M" "$27.85M" "$325.10M" "$532.18M" "$87.24M" "$151.10M"#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
length(gross_data)## [1] 90#Filling missing entries with NA
for (i in c(22, 48, 52, 63, 72, 84, 91, 93, 94, 100)){
a<-gross_data[1:(i-1)]
b<-gross_data[i:length(gross_data)]
gross_data<-append(a,list("NA"))
gross_data<-append(gross_data,b)
}
length(gross_data)## [1] 102#Data-Preprocessing: converting gross to numerical
unlist(gross_data)## [1] "248.7" "27.85" "325.1" "532.1" "87.24" "151.1" "330.3" "341.2" "100.5"
## [10] "36.26" "67.21" "232.6" "408.0" "363.0" "5.02" "58.70" "234.0" "5.88"
## [19] "2.01" "169.6" "2.13" "NA" "93.43" "138.2" "56.25" "54.65" "10.64"
## [28] "126.6" "34.34" "158.8" "52.85" "155.4" "47.70" "1.33" "100.0" "270.4"
## [37] "86.26" "103.1" "89.22" "35.82" "97.69" "51.74" "14.43" "75.40" "26.86"
## [46] "7.70" "61.43" "NA" "162.4" "153.7" "127.4" "NA" "31.15" "65.08"
## [55] "30.08" "47.37" "4.21" "35.59" "8.58" "55.12" "72.08" "102.4" "NA"
## [64] "5.20" "7.10" "364.0" "128.3" "43.03" "46.84" "67.27" "125.0" "NA"
## [73] "30.35" "60.32" "66.18" "26.41" "40.10" "486.3" "113.2" "12.39" "12.79"
## [82] "26.83" "82.05" "NA" "0.18" "62.68" "34.92" "0.66" "8.11" "368.3"
## [91] "NA" "21.59" "NA" "NA" "31.89" "46.01" "10.91" "2.14" "57.64"
## [100] "NA" "57.64"gross_data <- gross_data[-c(101,102)]
gross_data<-as.numeric(gross_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercionlength(gross_data)## [1] 100summary(gross_data)## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 0.18 27.11 58.17 99.58 126.20 532.10 10Combine the data to create a dataframe and inspect its structure
#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data,
Gross_Earning_in_Mil = gross_data,
Director = directors_data, Actor = actors_data)
#Structure of the data frame
str(movies_df)## 'data.frame': 100 obs. of 11 variables:
## $ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
## $ Title : Factor w/ 100 levels "10 Cloverfield Lane",..: 49 50 66 58 48 40 11 100 8 88 ...
## $ Description : Factor w/ 100 levels " A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in sile"| __truncated__,..: 60 29 20 78 94 98 47 57 13 56 ...
## $ Runtime : num 107 111 123 133 127 128 151 108 116 116 ...
## $ Genre : Factor w/ 8 levels "Action","Adventure",..: 3 7 1 1 2 5 1 3 7 1 ...
## $ Rating : num 7.6 7.4 6 7.8 6.7 8 6.5 8 7.9 7.4 ...
## $ Metascore : num 81 99 40 65 57 94 44 78 81 70 ...
## $ Votes : num 254974 258640 580793 532959 150551 ...
## $ Gross_Earning_in_Mil: num 248.7 27.9 325.1 532.1 87.2 ...
## $ Director : Factor w/ 98 levels "Alex Proyas",..: 82 11 25 35 92 20 98 14 29 86 ...
## $ Actor : Factor w/ 92 levels "Aamir Khan","Adam Driver",..: 8 52 89 32 31 72 9 34 5 70 ...Analyzing scraped data from the web
Question 1: Based on the above data, which movie from which Genre had the longest runtime?
#qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
ggplot(data = movies_df) +
geom_boxplot(mapping = aes(x = Genre, y = Runtime), fill = "red")
movies_df %>% filter(Runtime == max(Runtime)) %>%
select (Title, Genre, Runtime)## Title Genre Runtime
## 1 American Honey Drama 163Answer: The movie “American Honey” of the Drama genre has the longest runtime of 163 minutes.
Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
movies_df %>% filter(Runtime >= 130 & Runtime <= 160) %>%
group_by(Genre) %>%
summarise(sum = sum(Votes)) ## `summarise()` ungrouping output (override with `.groups` argument)## # A tibble: 6 x 2
## Genre sum
## <fct> <dbl>
## 1 Action 2535200
## 2 Adventure 394319
## 3 Animation 38048
## 4 Biography 616113
## 5 Drama 510893
## 6 Horror 260153Answer: The genre Action has the highest total votes of 2,535,200.
Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120?
ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))## Warning: Removed 10 rows containing missing values (geom_point).
movies_df %>% filter(Runtime >= 100 & Runtime <= 120) %>%
group_by(Genre) %>%
summarise(mean = mean(Gross_Earning_in_Mil, na.rm = TRUE))## `summarise()` ungrouping output (override with `.groups` argument)## # A tibble: 8 x 2
## Genre mean
## <fct> <dbl>
## 1 Action 90.7
## 2 Adventure 185.
## 3 Animation 216.
## 4 Biography 35.9
## 5 Comedy 38.6
## 6 Crime 75.4
## 7 Drama 52.5
## 8 Horror 69.8Answer: Across all genres, the Animation genre has the highest average gross earnings of 216 minutes in runtime 100 to 120.