#install.packages('rvest')
library('rvest')
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
webpage <- read_html(url)
#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')
#Converting the ranking data to text
rank_data <- html_text(rank_data_html)
#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)
#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6
#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')
#Converting the title data to text
title_data <- html_text(title_data_html)
#Let's have a look at the title
head(title_data)
## [1] "Batman v Superman: Dawn of Justice Ultimate Edition"
## [2] "Fantastic Beasts and Where to Find Them"
## [3] "Suicide Squad"
## [4] "Deadpool"
## [5] "Batman v Superman: Dawn of Justice"
## [6] "Hacksaw Ridge"
#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
#Let's have a look at the description data
head(description_data)
## [1] "\nBatman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and he is framed for murder during an international crisis. The heroes clash and force the neutral Wonder Woman to reemerge."
## [2] "\nThe adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years before Harry Potter reads his book in school."
## [3] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] "\nA wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [5] "\nFearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."
## [6] "\nWorld War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."
#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)
#Let's have a look at the runtime
head(runtime_data)
## [1] "182 min" "132 min" "123 min" "108 min" "152 min" "139 min"
#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
#Let's have another look at the runtime data
head(runtime_data)
## [1] 182 132 123 108 152 139
#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')
#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)
#Let's have a look at the metascore
head(metascore_data)
## [1] "66 " "40 " "65 " "44 " "71 "
## [6] "94 "
#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')
#Converting the genre data to text
genre_data <- html_text(genre_data_html)
#Let's have a look at the runtime
head(genre_data)
## [1] "\nAction, Adventure, Sci-Fi "
## [2] "\nAdventure, Family, Fantasy "
## [3] "\nAction, Adventure, Fantasy "
## [4] "\nAction, Adventure, Comedy "
## [5] "\nAction, Adventure, Sci-Fi "
## [6] "\nBiography, Drama, History "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)
#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)
#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)
#Convering each genre from text to factor
genre_data<-as.factor(genre_data)
#Let's have another look at the genre data
head(genre_data)
## [1] Action Adventure Action Action Action Biography
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
#Converting the ratings data to text
rating_data <- html_text(rating_data_html)
#Let's have a look at the ratings
head(rating_data)
## [1] "8.4" "7.3" "5.9" "8.0" "6.5" "8.1"
#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
#Converting the votes data to text
votes_data <- html_text(votes_data_html)
#Let's have a look at the votes data
head(votes_data)
## [1] "10,719" "447,825" "669,483" "989,475" "683,416" "497,218"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)
#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)
#Let's have another look at the votes data
head(votes_data)
## [1] 10719 447825 669483 989475 683416 497218
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
#Converting the directors data to text
directors_data <- html_text(directors_data_html)
#Let's have a look at the directors data
head(directors_data)
## [1] "David Yates" "David Ayer" "Tim Miller" "Zack Snyder"
## [5] "Mel Gibson" "Damien Chazelle"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)
#Let's have a look at the actors data
head(actors_data)
## [1] "Eddie Redmayne" "Will Smith" "Ryan Reynolds" "Ben Affleck"
## [5] "Andrew Garfield" "Ryan Gosling"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)
#Let's have another look at the ratings data
head(rating_data)
## [1] 8.4 7.3 5.9 8.0 6.5 8.1
#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)
#Lets check the length of metascore data
length(metascore_data)
## [1] 96
for (i in c(39,73,80,89)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
#Let's have another look at length of the metascore data
length(metascore_data)
## [1] 100
#Let's look at summary statistics
summary(metascore_data)
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 18.00 47.00 62.00 60.15 74.00 99.00 4
#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)
#Let's have a look at the votes data
head(gross_data)
## [1] "$234.04M" "$325.10M" "$363.07M" "$330.36M" "$67.21M" "$151.10M"
#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
#Let's check the length of gross data
length(gross_data)
## [1] 89
#Filling missing entries with NA
for (i in c(17,39,49,52,57,64,66,73,76,77,80,87,88,89)){
a<-gross_data[1:(i-1)]
b<-gross_data[i:length(gross_data)]
gross_data<-append(a,list("NA"))
gross_data<-append(gross_data,b)
}
#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
#Let's have another look at the length of gross data
length(gross_data)
## [1] 103
summary(gross_data)
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 0.18 18.71 57.64 99.38 126.60 532.10 14
## Find metascore data with missing values and replace with NAs (this is an automated method)
ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>%
# scrape the ratings bar and convert to text
html_text2()
head(ratings_bar_data) # look at the ratings bar
## [1] "8.4\nRate this\n 1 2 3 4 5 6 7 8 9 10 8.4/10 X "
## [2] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n66 Metascore"
## [3] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [4] "8.0\nRate this\n 1 2 3 4 5 6 7 8 9 10 8/10 X \n65 Metascore"
## [5] "6.5\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.5/10 X \n44 Metascore"
## [6] "8.1\nRate this\n 1 2 3 4 5 6 7 8 9 10 8.1/10 X \n71 Metascore"
library(tidyverse)
## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.1 ──
## ✓ ggplot2 3.3.5 ✓ purrr 0.3.4
## ✓ tibble 3.1.6 ✓ dplyr 1.0.8
## ✓ tidyr 1.2.0 ✓ stringr 1.4.0
## ✓ readr 2.1.2 ✓ forcats 0.5.1
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## x dplyr::filter() masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag() masks stats::lag()
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%
# extract Metascore
str_match("\\d{2}") %>%
as.numeric() # convert to number
length(metascore_data)
## [1] 100
metascore_data
## [1] NA 66 40 65 44 71 94 52 72 81 59 62 78 84 99 81 54 65 64 67 96 48 75 74 88
## [26] 41 44 78 70 57 79 71 28 25 51 73 72 51 66 81 60 32 66 57 81 47 72 64 68 NA
## [51] 52 79 77 34 42 51 45 32 69 62 77 58 48 79 47 45 23 76 65 49 NA 74 42 60 76
## [76] 39 51 36 79 42 18 35 78 67 63 36 77 68 46 55 59 59 NA 60 65 40 61 58 81 26
summary(metascore_data)
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 18.00 47.00 62.00 60.15 74.00 99.00 4
## Find the missing gross earnings (automated) Earnings are part of the votes bar in the html, scrape the votes bar and extract earnings with a regular expression to get the NAs in context.
# scrape the votess bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
html_text2()
head(votes_bar_data) # look at the votes bar data
## [1] "Votes: 10,719" "Votes: 447,825 | Gross: $234.04M"
## [3] "Votes: 669,483 | Gross: $325.10M" "Votes: 989,475 | Gross: $363.07M"
## [5] "Votes: 683,416 | Gross: $330.36M" "Votes: 497,218 | Gross: $67.21M"
gross_data <- str_match(votes_bar_data, "\\$.+$") # extract the gross earnings
gross_data <- gsub("M","",gross_data) # clean data: remove 'M' sign
gross_data <- substring(gross_data,2,6) %>% # clean data: remove '$' sign
as.numeric()
length(gross_data)
## [1] 100
#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data, Description = description_data, Runtime = runtime_data, Genre = genre_data, Rating = rating_data, Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data)
#Structure of the data frame
str(movies_df)
## 'data.frame': 100 obs. of 9 variables:
## $ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
## $ Title : chr "Batman v Superman: Dawn of Justice Ultimate Edition" "Fantastic Beasts and Where to Find Them" "Suicide Squad" "Deadpool" ...
## $ Description : chr "\nBatman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world an"| __truncated__ "\nThe adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years b"| __truncated__ "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensiv"| __truncated__ "\nA wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the m"| __truncated__ ...
## $ Runtime : num 182 132 123 108 152 139 128 144 115 107 ...
## $ Genre : Factor w/ 8 levels "Action","Adventure",..: 1 2 1 1 1 4 5 1 1 3 ...
## $ Rating : num 8.4 7.3 5.9 8 6.5 8.1 8 6.9 7.5 7.6 ...
## $ Metascore : num NA 66 40 65 44 71 94 52 72 81 ...
## $ Votes : num 10719 447825 669483 989475 683416 ...
## $ Gross_Earning_in_Mil: num NA 234 325 363 330 ...
library('ggplot2')
#Question 1: Based on the above data, which movie from which Genre had the longest runtime?
qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
# Answer: Action had the longest runtime
runtime_filtered <- movies_df %>% filter(Runtime > 150)
qplot(data = runtime_filtered, Runtime, fill = Genre)
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
# Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
# Answer: Action had the highest votes
runtime130 <- movies_df %>%
filter(Runtime >= 130 & Runtime <= 160)
#options(scipen = 100)
ggplot(runtime130,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
#Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 11 rows containing missing values (geom_point).
# Answer: Adventure had the highest average gross earnings in runtime 100 - 120
runtime_gross <- movies_df %>%
filter(Runtime >= 100 & Runtime <= 120)
ggplot(runtime_gross,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 2 rows containing missing values (geom_point).
