lab2
Honore Nguessan
7/18/2019
#Loading the rvest package
library('rvest')## Warning: package 'rvest' was built under R version 3.6.1## Loading required package: xml2#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
#Reading the HTML code from the website
webpage <- read_html(url)#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')
#Converting the ranking data to text
rank_data <- html_text(rank_data_html)
#Let's have a look at the rankings
head(rank_data)## [1] "1." "2." "3." "4." "5." "6."#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)
#Let's have another look at the rankings
head(rank_data)## [1] 1 2 3 4 5 6#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')
#Converting the title data to text
title_data <- html_text(title_data_html)
#Let's have a look at the title
head(title_data)## [1] "Suicide Squad" "Captain America: Civil War"
## [3] "Deadpool" "The Neon Demon"
## [5] "The Magnificent Seven" "Split"#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
#Let's have a look at the description data
head(description_data)## [1] "\n A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [2] "\n Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."
## [3] "\n A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [4] "\n An aspiring model, Jesse, is new to Los Angeles. However, her beauty and youth, which generate intense fascination and jealousy within the fashion industry, may prove themselves sinister."
## [5] "\n Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."
## [6] "\n Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)
#Let's have another look at the description data
head(description_data)## [1] " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [2] " Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."
## [3] " A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [4] " An aspiring model, Jesse, is new to Los Angeles. However, her beauty and youth, which generate intense fascination and jealousy within the fashion industry, may prove themselves sinister."
## [5] " Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."
## [6] " Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)
#Let's have a look at the runtime
head(runtime_data)## [1] "123 min" "147 min" "108 min" "117 min" "132 min" "117 min"#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
#Let's have another look at the runtime data
head(runtime_data)## [1] 123 147 108 117 132 117#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')
#Converting the genre data to text
genre_data <- html_text(genre_data_html)
#Let's have a look at the runtime
head(genre_data)## [1] "\nAction, Adventure, Fantasy "
## [2] "\nAction, Adventure, Sci-Fi "
## [3] "\nAction, Adventure, Comedy "
## [4] "\nHorror, Thriller "
## [5] "\nAction, Adventure, Western "
## [6] "\nHorror, Thriller "#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)
#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)
#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)
#Convering each genre from text to factor
genre_data<-as.factor(genre_data)
#Let's have another look at the genre data
head(genre_data)## [1] Action Action Action Horror Action Horror
## 9 Levels: Action Adventure Animation Biography Comedy Crime ... Thriller#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
#Converting the ratings data to text
rating_data <- html_text(rating_data_html)
#Let's have a look at the ratings
head(rating_data)## [1] "6.0" "7.8" "8.0" "6.2" "6.9" "7.3"#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)
#Let's have another look at the ratings data
head(rating_data)## [1] 6.0 7.8 8.0 6.2 6.9 7.3#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
#Converting the votes data to text
votes_data <- html_text(votes_data_html)
#Let's have a look at the votes data
head(votes_data)## [1] "536,663" "590,251" "827,314" "73,355" "169,536" "364,104"#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)
#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)
#Let's have another look at the votes data
head(votes_data)## [1] 536663 590251 827314 73355 169536 364104#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
#Converting the directors data to text
directors_data <- html_text(directors_data_html)
#Let's have a look at the directors data
head(directors_data)## [1] "David Ayer" "Anthony Russo" "Tim Miller"
## [4] "Nicolas Winding Refn" "Antoine Fuqua" "M. Night Shyamalan"#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)
#Let's have a look at the actors data
head(actors_data)## [1] "Will Smith" "Chris Evans" "Ryan Reynolds"
## [4] "Elle Fanning" "Denzel Washington" "James McAvoy"#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')
#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)
#Let's have a look at the metascore data
head(metascore_data)## [1] "40 " "75 " "65 " "51 " "54 "
## [6] "62 "#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)
#Lets check the length of metascore data
length(metascore_data)## [1] 97for (i in c(38,61,71)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion#Let's have another look at length of the metascore data
length(metascore_data)## [1] 100#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)
#Let's have a look at the votes data
head(gross_data)## [1] "$325.10M" "$408.08M" "$363.07M" "$1.33M" "$93.43M" "$138.29M"#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
#Let's check the length of gross data
length(gross_data)## [1] 92#Filling missing entries with NA
for (i in c(38,47,63,65,69,71,87,94)){
a<-gross_data[1:(i-1)]
b<-gross_data[i:length(gross_data)]
gross_data<-append(a,list("NA"))
gross_data<-append(gross_data,b)
}
#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion#Let's have another look at the length of gross data
length(gross_data)## [1] 100summary(gross_data)## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 0.02 25.52 55.20 97.14 125.40 532.10 8#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data,
Director = directors_data, Actor = actors_data)
#Structure of the data frame
str(movies_df)## 'data.frame': 100 obs. of 11 variables:
## $ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
## $ Title : Factor w/ 100 levels "10 Cloverfield Lane",..: 66 16 21 89 88 64 98 23 29 85 ...
## $ Description : Factor w/ 100 levels " A chronicle of the childhood, adolescence and burgeoning adulthood of a young, African-American, gay man gr"| __truncated__,..: 18 67 24 33 70 85 61 98 100 27 ...
## $ Runtime : num 123 147 108 117 132 117 144 115 139 106 ...
## $ Genre : Factor w/ 9 levels "Action","Adventure",..: 1 1 1 8 1 8 1 1 4 2 ...
## $ Rating : num 6 7.8 8 6.2 6.9 7.3 7 7.5 8.1 7.4 ...
## $ Metascore : num 40 75 65 51 54 62 52 72 71 77 ...
## $ Votes : num 536663 590251 827314 73355 169536 ...
## $ Gross_Earning_in_Mil: num 325.1 408 363 1.33 93.43 ...
## $ Director : Factor w/ 99 levels "Alex Proyas",..: 27 7 94 71 8 58 14 87 63 51 ...
## $ Actor : Factor w/ 91 levels "Aamir Khan","Adam Sandler",..: 87 19 73 26 22 40 40 10 5 67 ...library('ggplot2')
qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))## Warning: Removed 8 rows containing missing values (geom_point).
Question 1: Based on the above data, which movie from which Genre had the longest runtime?
The genre which had the longest runtime is Adventure.
Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
In the Runtime of 130-160 mins, the genre has the highest votes is Action.
Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
Across all genres the genre has the highest average gross earnings in runtime 100 to 120 is Action.