#WEB SCRAPING LAB ASSIGNMENT
# install.packages('rvest')
# Loading the rvest package
library('rvest')
## Loading required package: xml2
library(tidyverse)
## -- Attaching packages -------------------------------------------------------------------------------- tidyverse 1.3.0 --
## v ggplot2 3.3.1 v purrr 0.3.4
## v tibble 3.0.1 v dplyr 1.0.0
## v tidyr 1.1.0 v stringr 1.4.0
## v readr 1.3.1 v forcats 0.5.0
## -- Conflicts ----------------------------------------------------------------------------------- tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag() masks stats::lag()
## x purrr::pluck() masks rvest::pluck()
# graphing library makes interactive, publication-quality graphs
library(plotly)
##
## Attaching package: 'plotly'
## The following object is masked from 'package:ggplot2':
##
## last_plot
## The following object is masked from 'package:stats':
##
## filter
## The following object is masked from 'package:graphics':
##
## layout
#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
#Reading the HTML code from the website
webpage <- read_html(url)
#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')
#Converting the ranking data to text
rank_data <- html_text(rank_data_html)
#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)
#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6
#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')
#Converting the title data to text
title_data <- html_text(title_data_html)
#Let's have a look at the title
head(title_data)
## [1] "Moana"
## [2] "Moonlight"
## [3] "Suicide Squad"
## [4] "Rogue One: A Star Wars Story"
## [5] "Miss Peregrine's Home for Peculiar Children"
## [6] "La La Land"
#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
#Let's have a look at the description data
head(description_data)
## [1] "\n In Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches Moana's island, she answers the Ocean's call to seek out the Demigod to set things right."
## [2] "\n A young African-American man grapples with his identity and sexuality while experiencing the everyday struggles of childhood, adolescence, and burgeoning adulthood."
## [3] "\n A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] "\n The daughter of an Imperial scientist joins the Rebel Alliance in a risky move to steal the Death Star plans."
## [5] "\n When Jacob (Asa Butterfield) discovers clues to a mystery that stretches across time, he finds Miss Peregrine's Home for Peculiar Children. But the danger deepens after he gets to know the residents and learns about their special powers."
## [6] "\n While navigating their careers in Los Angeles, a pianist and an actress fall in love while attempting to reconcile their aspirations for the future."
#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)
#Let's have another look at the description data
head(description_data)
## [1] " In Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches Moana's island, she answers the Ocean's call to seek out the Demigod to set things right."
## [2] " A young African-American man grapples with his identity and sexuality while experiencing the everyday struggles of childhood, adolescence, and burgeoning adulthood."
## [3] " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] " The daughter of an Imperial scientist joins the Rebel Alliance in a risky move to steal the Death Star plans."
## [5] " When Jacob (Asa Butterfield) discovers clues to a mystery that stretches across time, he finds Miss Peregrine's Home for Peculiar Children. But the danger deepens after he gets to know the residents and learns about their special powers."
## [6] " While navigating their careers in Los Angeles, a pianist and an actress fall in love while attempting to reconcile their aspirations for the future."
#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)
#Let's have a look at the runtime
head(runtime_data)
## [1] "107 min" "111 min" "123 min" "133 min" "127 min" "128 min"
#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
#Let's have another look at the runtime data
head(runtime_data)
## [1] 107 111 123 133 127 128
#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')
#Converting the genre data to text
genre_data <- html_text(genre_data_html)
#Let's have a look at the runtime
head(genre_data)
## [1] "\nAnimation, Adventure, Comedy "
## [2] "\nDrama "
## [3] "\nAction, Adventure, Fantasy "
## [4] "\nAction, Adventure, Sci-Fi "
## [5] "\nAdventure, Drama, Family "
## [6] "\nComedy, Drama, Music "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)
#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)
#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)
#Convering each genre from text to factor
genre_data<-as.factor(genre_data)
#Let's have another look at the genre data
head(genre_data)
## [1] Animation Drama Action Action Adventure Comedy
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
#Converting the ratings data to text
rating_data <- html_text(rating_data_html)
#Let's have a look at the ratings
head(rating_data)
## [1] "7.6" "7.4" "6.0" "7.8" "6.7" "8.0"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)
#Let's have another look at the ratings data
head(rating_data)
## [1] 7.6 7.4 6.0 7.8 6.7 8.0
#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
#Converting the votes data to text
votes_data <- html_text(votes_data_html)
#Let's have a look at the votes data
head(votes_data)
## [1] "255,125" "258,773" "580,892" "533,067" "150,584" "480,918"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)
#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)
#Let's have another look at the votes data
head(votes_data)
## [1] 255125 258773 580892 533067 150584 480918
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
#Converting the directors data to text
directors_data <- html_text(directors_data_html)
#Let's have a look at the directors data
head(directors_data)
## [1] "Ron Clements" "Barry Jenkins" "David Ayer" "Gareth Edwards"
## [5] "Tim Burton" "Damien Chazelle"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)
#Let's have a look at the actors data
head(actors_data)
## [1] "Auli'i Cravalho" "Mahershala Ali" "Will Smith" "Felicity Jones"
## [5] "Eva Green" "Ryan Gosling"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')
#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)
#Let's have a look at the metascore
head(metascore_data)
## [1] "81 " "99 " "40 " "65 " "57 "
## [6] "94 "
#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)
#Lets check the length of metascore data
length(metascore_data)
## [1] 98
for (i in c(22,80)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
#Let's have another look at length of the metascore data
length(metascore_data)
## [1] 100
#Let's look at summary statistics
summary(metascore_data)
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 21.00 47.25 62.00 60.19 73.50 99.00 2
#Using CSS selectors to scrap the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)
#Let's have a look at the gross data
head(gross_data)
## [1] "$248.76M" "$27.85M" "$325.10M" "$532.18M" "$87.24M" "$151.10M"
#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("[^0-9]*","",gross_data)
# gross_data<-gsub("M","",gross_data)
head(gross_data)
## [1] "24876" "2785" "32510" "53218" "8724" "15110"
#Let's check the length of gross data
length(gross_data)
## [1] 90
#Filling missing entries with NA
for (i in c(22,48,52,63,72,84,91,93,94,100)){
a<-gross_data[1:(i-1)]
b<-gross_data[i:length(gross_data)]
gross_data<-append(a,list("NA"))
gross_data<-append(gross_data,b)
}
#Data-Preprocessing: converting gross to numerical
unlist(gross_data)
## [1] "24876" "2785" "32510" "53218" "8724" "15110" "33036" "34127" "10055"
## [10] "3626" "6721" "23264" "40808" "36307" "502" "5870" "23404" "588"
## [19] "201" "16961" "213" "NA" "9343" "13829" "5625" "5465" "1064"
## [28] "12664" "3434" "15885" "5285" "15544" "4770" "133" "10001" "27040"
## [37] "8626" "10314" "8922" "3582" "9769" "5174" "1443" "7540" "2686"
## [46] "770" "6143" "NA" "16243" "15371" "12744" "NA" "3115" "6508"
## [55] "3008" "4737" "421" "3559" "858" "5512" "7208" "10247" "NA"
## [64] "520" "710" "36400" "12834" "4303" "4684" "6727" "12507" "NA"
## [73] "3035" "6032" "6618" "2641" "4010" "48630" "11326" "1239" "1279"
## [82] "2683" "8205" "NA" "018" "6268" "3492" "066" "811" "36838"
## [91] "NA" "2159" "NA" "NA" "3189" "4601" "1091" "214" "5764"
## [100] "NA" "5764"
gross_data <- gross_data[-c(101,102)]
gross_data<-as.numeric(gross_data)
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
#Let's have another look at the length of gross data
length(gross_data)
## [1] 100
summary(gross_data)
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 18 2711 5817 9960 12625 53218 10
#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data,
Director = directors_data, Actor = actors_data)
#Structure of the data frame
str(movies_df)
## 'data.frame': 100 obs. of 11 variables:
## $ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
## $ Title : chr "Moana" "Moonlight" "Suicide Squad" "Rogue One: A Star Wars Story" ...
## $ Description : chr " In Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches Moana's island, she answer"| __truncated__ " A young African-American man grapples with his identity and sexuality while experiencing the everyday strug"| __truncated__ " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defens"| __truncated__ " The daughter of an Imperial scientist joins the Rebel Alliance in a risky move to steal the Death Star plans." ...
## $ Runtime : num 107 111 123 133 127 128 151 108 116 116 ...
## $ Genre : Factor w/ 8 levels "Action","Adventure",..: 3 7 1 1 2 5 1 3 7 1 ...
## $ Rating : num 7.6 7.4 6 7.8 6.7 8 6.5 8 7.9 7.4 ...
## $ Metascore : num 81 99 40 65 57 94 44 78 81 70 ...
## $ Votes : num 255125 258773 580892 533067 150584 ...
## $ Gross_Earning_in_Mil: num 24876 2785 32510 53218 8724 ...
## $ Director : Factor w/ 98 levels "Alex Proyas",..: 82 11 25 35 92 20 98 14 29 86 ...
## $ Actor : Factor w/ 92 levels "Aamir Khan","Adam Driver",..: 8 52 89 32 31 72 9 34 5 70 ...
library('ggplot2')
qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
# Question 1: Based on the above data, which movie from which Genre had the longest runtime?
movies_df %>%
rownames_to_column(var = "Name") %>%
filter(Runtime == max(Runtime))
## Name Rank Title
## 1 88 88 American Honey
## Description
## 1 A teenage girl with nothing to lose joins a traveling magazine sales crew, and gets caught up in a whirlwind of hard partying, law bending and young love as she criss-crosses the Midwest with a band of misfits.
## Runtime Genre Rating Metascore Votes Gross_Earning_in_Mil Director
## 1 163 Drama 7 79 35944 66 Andrea Arnold
## Actor
## 1 Sasha Lane
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
# Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
movies_df %>%
rownames_to_column(var = "Name") %>%
filter(Runtime == c(130,160)) %>%
filter(Votes == max(Votes))
## Name Rank Title
## 1 91 91 A Silent Voice: The Movie
## Description
## 1 A young man is ostracized by his classmates after he bullies a deaf girl to the point where she moves away. Years later, he sets off on a path for redemption.
## Runtime Genre Rating Metascore Votes Gross_Earning_in_Mil Director
## 1 130 Animation 8.2 78 38139 NA Naoko Yamada
## Actor
## 1 Miyu Irino
ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 10 rows containing missing values (geom_point).
# Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
movies_df %>%
rownames_to_column(var = "Name") %>%
filter(Runtime == c(100,120)) %>%
group_by(Genre) %>%
summarize(averageGross = mean(Gross_Earning_in_Mil)) %>%
filter(averageGross == max(averageGross))
## `summarise()` ungrouping output (override with `.groups` argument)
## # A tibble: 1 x 2
## Genre averageGross
## <fct> <dbl>
## 1 Comedy 11326