## load rvest
library(rvest)
## Loading required package: xml2
indicater URL to be used
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
read HTML from website
webpage <- read_html(url)
## Create new object ranked_data_html
rank_data_html <- html_nodes(webpage,'.text-primary')
## Converting ranking data to text
rank_data <- html_text(rank_data_html)
## View the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
# create new object storing ranking data as nummerical values
rank_data<-as.numeric(rank_data)
#display head again
head(rank_data)
## [1] 1 2 3 4 5 6
# create new object holding title
title_data_html <- html_nodes(webpage,'.lister-item-header a')
# new object title_data to hold converted text
title_data <- html_text(title_data_html)
# view
head(title_data)
## [1] "Suicide Squad" "Batman v Superman: Dawn of Justice"
## [3] "Captain America: Civil War" "Captain Fantastic"
## [5] "Deadpool" "The Accountant"
# new object to hold description data
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
# convert to text format
description_data <- html_text(description_data_html)
# look at head of new object
head(description_data)
## [1] "\n A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [2] "\n Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."
## [3] "\n Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."
## [4] "\n In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] "\n A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [6] "\n As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."
## data pre processing to remove \n
description_data<-gsub("\n","",description_data)
# view head
head(description_data)
## [1] " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [2] " Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."
## [3] " Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."
## [4] " In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] " A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."
## [6] " As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."
# new object for runtime values
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
# converting the data
runtime_data <- html_text(runtime_data_html)
# view head of runtime
head(runtime_data)
## [1] "123 min" "152 min" "147 min" "118 min" "108 min" "128 min"
#Data-Preprocessing: removing mins
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
# view head for runtime data
head(runtime_data)
## [1] 123 152 147 118 108 128
# new object to hold genre info
genre_data_html <- html_nodes(webpage,'.genre')
#Converting data to text
genre_data <- html_text(genre_data_html)
# view genre head
head(genre_data)
## [1] "\nAction, Adventure, Fantasy "
## [2] "\nAction, Adventure, Sci-Fi "
## [3] "\nAction, Adventure, Sci-Fi "
## [4] "\nComedy, Drama "
## [5] "\nAction, Adventure, Comedy "
## [6] "\nAction, Crime, Drama "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)
#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)
#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)
#Convering each genre from text to factor
genre_data<-as.factor(genre_data)
# view the head for genre
head(genre_data)
## [1] Action Action Action Comedy Action Action
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
get the ratings
# new object getting ratings
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
# converting text to data
rating_data <- html_text(rating_data_html)
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)
# look at head of rating
head(rating_data)
## [1] 6.0 6.4 7.8 7.9 8.0 7.3
scrape for votes
# object for votess
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
#Converting the votes data to text
votes_data <- html_text(votes_data_html)
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)
#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)
#Let's have another look at the votes data
head(votes_data)
## [1] 612605 643566 676436 194667 914024 264483
scrape for directors
# Directors Object
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
#Convert data to text
directors_data <- html_text(directors_data_html)
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
# Run Direcors head
head(directors_data)
## [1] David Ayer Zack Snyder Anthony Russo Matt Ross Tim Miller
## [6] Gavin O'Connor
## 98 Levels: Adam Wingard Alex Proyas Ana Lily Amirpour ... Zack Snyder
scrape for actors
# Actors object
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
#Convert data to text
actors_data <- html_text(actors_data_html)
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
# run head for actors
head(actors_data)
## [1] Will Smith Ben Affleck Chris Evans Viggo Mortensen
## [5] Ryan Reynolds Ben Affleck
## 91 Levels: Aamir Khan Alexander Skarsgård Amy Adams ... Zach Galifianakis
step 10 - scraping for gross revenue
# creating object to hold gross revenue values
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
#Convert gross revenue data to text
gross_data <- html_text(gross_data_html)
#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
# check head
length(gross_data)
## [1] 92
correct the problem of missing valuess for gross revenue
## fill missing entries with NA
for (i in c(18,67,73,75,83,87,98,100)){
gross_data <- append(gross_data, NA, i-1)
}
gross_data <- as.numeric(gross_data)
# view gross data
length(gross_data)
## [1] 100
summary(gross_data)
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 0.04 14.39 52.30 91.32 116.15 532.10 8
step 11 - combine everything
## combining all objects to create a larger data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data,
Director = directors_data, Actor = actors_data)
# View the structure of the data frame
str(movies_df)
## 'data.frame': 100 obs. of 11 variables:
## $ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
## $ Title : chr "Suicide Squad" "Batman v Superman: Dawn of Justice" "Captain America: Civil War" "Captain Fantastic" ...
## $ Description : chr " A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defens"| __truncated__ " Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world "| __truncated__ " Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man." " In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical "| __truncated__ ...
## $ Runtime : num 123 152 147 118 108 128 120 116 107 116 ...
## $ Genre : Factor w/ 8 levels "Action","Adventure",..: 1 1 1 5 1 1 1 1 3 7 ...
## $ Rating : num 6 6.4 7.8 7.9 8 7.3 6.8 7.4 7.6 7.9 ...
## $ Metascore : num 40 44 75 72 65 51 67 70 81 81 ...
## $ Votes : num 612605 643566 676436 194667 914024 ...
## $ Gross_Earning_in_Mil: num 325.1 330.3 408 5.88 363 ...
## $ Director : Factor w/ 98 levels "Adam Wingard",..: 23 98 6 61 93 36 40 86 82 27 ...
## $ Actor : Factor w/ 91 levels "Aamir Khan","Alexander Skarsgård",..: 89 8 19 88 75 8 39 73 7 3 ...
plot 1 - Fill Histogram
library('ggplot2')
qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
Question 1 - Based on the above data, which movie from which genre had the longest runtime?
Answer: It appears there was one drama (Silence, rank 49, 161 mins) and one action movie (Dangle, rank 57, 161 mins) both competing for longest run time
Plot 2 - Run Time and Avg Rating
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
Question 2 - Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
Answer - according to the scatterplot, the biography, drama, and animation genres all apear to have similarly high votes with an average a little above 8 votes
plot 3 - Gross Earnings and Runtime
ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 8 rows containing missing values (geom_point).
Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
Answer: Based on the scatterplot, it appears that two catagories - action and adventure - are all very closely matched for highest avg gross earnings at around $375 million