Loading packages

library('rvest')
library(tidyverse)
## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.2 ──
## ✔ ggplot2 3.3.6      ✔ purrr   0.3.5 
## ✔ tibble  3.1.8      ✔ dplyr   1.0.10
## ✔ tidyr   1.2.1      ✔ stringr 1.4.1 
## ✔ readr   2.1.3      ✔ forcats 0.5.2 
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter()         masks stats::filter()
## ✖ readr::guess_encoding() masks rvest::guess_encoding()
## ✖ dplyr::lag()            masks stats::lag()
library(plotly)
## 
## Attaching package: 'plotly'
## 
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## 
## The following object is masked from 'package:stats':
## 
##     filter
## 
## The following object is masked from 'package:graphics':
## 
##     layout

Specifying the url for desired website to be scraped

url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

Reading the HTML code from the website

webpage <- read_html(url)

Scrape for Movie Rank Information

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')
#Converting the ranking data to text
rank_data <- html_text(rank_data_html)
#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)
#Let's have another look at the rankings and the length of the data
head(rank_data)
## [1] 1 2 3 4 5 6
length(rank_data)
## [1] 100

Scrape for the Title Information

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')
#Converting the title data to text
title_data <- html_text(title_data_html)
#Let's have a look at the title
head(title_data)
## [1] "Terrifier"                    "Rogue One: A Star Wars Story"
## [3] "Sing"                         "Suicide Squad"               
## [5] "Deadpool"                     "The Handmaiden"
length(title_data)
## [1] 100

Scrape for Movie Description Information

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
#Data-Preprocessing:Removing '\n'
description_data <-gsub("\n","",description_data)
head(description_data)
## [1] "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                                                                 
## [2] "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                                                              
## [3] "In a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."
## [4] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                          
## [5] "A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                        
## [6] "A woman is hired as a handmaiden to a Japanese heiress, but secretly she is involved in a plot to defraud her."
length(description_data)
## [1] 100

Scrape for Movie Run Times

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "85 min"  "133 min" "108 min" "123 min" "108 min" "145 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1]  85 133 108 123 108 145
length(runtime_data)
## [1] 100

Scrape for Movie Genre Information

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nHorror, Thriller            "          
## [2] "\nAction, Adventure, Sci-Fi            " 
## [3] "\nAnimation, Comedy, Family            " 
## [4] "\nAction, Adventure, Fantasy            "
## [5] "\nAction, Adventure, Comedy            " 
## [6] "\nDrama, Romance, Thriller            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Horror    Action    Animation Action    Action    Drama    
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
length(genre_data)
## [1] 100

Scrape for Movie Rating Information

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "5.6" "7.8" "7.1" "5.9" "8.0" "8.1"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 5.6 7.8 7.1 5.9 8.0 8.1
length(rating_data)
## [1] 100

Scrape for Voting Information

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "26,655"    "631,155"   "170,026"   "683,537"   "1,029,202" "146,577"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1]   26655  631155  170026  683537 1029202  146577
length(votes_data)
## [1] 100

Scarpe for Movie Director Information

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Damien Leone"   "Gareth Edwards" "Garth Jennings" "David Ayer"    
## [5] "Tim Miller"     "Park Chan-wook"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
length(directors_data)
## [1] 100

Scrape for Movie Actor Information

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Jenna Kanell"        "Felicity Jones"      "Matthew McConaughey"
## [4] "Will Smith"          "Ryan Reynolds"       "Kim Min-hee"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
length(actors_data)
## [1] 100

Find metascore data with minssing values and replace with NAs

ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>%
# scrape the ratings bar and convert to text
 html_text2()
head(ratings_bar_data) 
## [1] "5.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.6/10 X "              
## [2] "7.8\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.8/10 X \n65 Metascore"
## [3] "7.1\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.1/10 X \n59 Metascore"
## [4] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [5] "8.0\nRate this\n 1 2 3 4 5 6 7 8 9 10 8/10 X \n65 Metascore"  
## [6] "8.1\nRate this\n 1 2 3 4 5 6 7 8 9 10 8.1/10 X \n84 Metascore"
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%
# extract Metascore
 str_match("\\d{2}") %>%
 as.numeric() # convert to number
length(metascore_data)
## [1] 100
metascore_data
##   [1] NA 65 59 40 65 84 62 65 60 44 71 81 73 71 57 66 94 81 70 67 81 51 72 67 76
##  [26] 52 79 74 66 48 78 51 75 33 47 58 96 82 65 42 65 45 41 47 42 68 72 26 57 88
##  [51] 64 99 48 60 59 81 44 NA 62 51 35 54 35 52 32 NA 55 69 NA NA 68 21 44 25 79
##  [76] 78 47 NA 58 66 77 81 42 74 42 61 48 68 NA 34 60 67 69 32 28 23 51 58 33 76
summary(metascore_data) 
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   21.00   47.00   61.00   59.45   72.00   99.00       7

Find the Missing Gross Earnings

votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
 html_text2()
head(votes_bar_data) # look at the votes bar data
## [1] "Votes: 26,655"                      "Votes: 631,155 | Gross: $532.18M"  
## [3] "Votes: 170,026 | Gross: $270.40M"   "Votes: 683,537 | Gross: $325.10M"  
## [5] "Votes: 1,029,202 | Gross: $363.07M" "Votes: 146,577 | Gross: $2.01M"
gross_data <- str_match(votes_bar_data, "\\$.+$") # extract the gross earnings
gross_data <- gsub("M","",gross_data) # clean data: remove 'M' sign
gross_data <- substring(gross_data,2,6) %>% # clean data: remove '$' sign
 as.numeric()
length(gross_data)
## [1] 100

Combine all the list to form a data frame

movies_df <- data.frame(Rank = rank_data, Title = title_data, Description = description_data, Runtime = runtime_data, Genre = genre_data, Rating = rating_data, Director = directors_data, Actors = actors_data, Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data)
str(movies_df)
## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Terrifier" "Rogue One: A Star Wars Story" "Sing" "Suicide Squad" ...
##  $ Description         : chr  "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown." "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death St"| __truncated__ "In a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing compe"| __truncated__ "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ ...
##  $ Runtime             : num  85 133 108 123 108 145 117 134 117 152 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 8 1 3 1 1 7 8 8 1 1 ...
##  $ Rating              : num  5.6 7.8 7.1 5.9 8 8.1 7.3 7.3 6.9 6.4 ...
##  $ Director            : Factor w/ 96 levels "Adam Wingard",..: 22 34 36 26 90 67 54 44 68 96 ...
##  $ Actors              : Factor w/ 92 levels "Aamir Khan","Alexander Skarsgård",..: 43 33 63 92 76 53 42 90 64 7 ...
##  $ Metascore           : num  NA 65 59 40 65 84 62 65 60 44 ...
##  $ Votes               : num  26655 631155 170026 683537 1029202 ...
##  $ Gross_Earning_in_Mil: num  NA 532 270 325 363 ...

Based on the above data, which movie from wchic genre had the longest runtime?

library('ggplot2')
plot1 <- movies_df
  qplot(data = movies_df,Runtime,fill = Genre,bins = 30)+
  scale_fill_discrete(name ="Genre") +
  labs(title = "Top 100 Movies of 2016 Runtime by Genre") +
  geom_histogram(position="identity", alpha=0.5, binwidth = 5, color ="white")

Answer to question 1:

movies_df1 <- select(movies_df, Title, Genre, Runtime)
movies_df1[which.max(movies_df1$Runtime),]
##                                                    Title  Genre Runtime
## 58 Batman v Superman: Dawn of Justice - Ultimate Edition Action     182
Batman v Superman: Dawn of Justice  had  the longest runtime which is 182.

Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre, text = paste("Movie Title:", title_data)), alpha = 0.7) +
  labs(title = " Top 100 Movies of 2016 Runtime by Ratings")
## Warning: Ignoring unknown aesthetics: text

Answer to question 2

movie2_df2 <- select(movies_df, Title, Genre, Runtime, Votes)  # using select function to create a subset with those four variables movies_df, Title, Genre, Runtime, Votes
movie2_df2 <- filter(movie2_df2, Runtime >= 130 & Runtime <= 160)  # Using the filter function to select to runtime between 130-160 mins
movie2_df2[which.max(movie2_df2$Votes),]                       # Finding the movie that has the highest vote
##                        Title  Genre Runtime  Votes
## 9 Captain America: Civil War Action     147 781568

Captain America: Civil War has the highest votes which are in total 781553, the runtime is 147, and the genre is action.

Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre), alpha = 0.5) +
  labs(title = " Top 100 Movies of 2016 Runtime by Gross Earnings in Millions") +
  scale_y_continuous("Gross Earnings in Millions", limits = c(-10, 600))
## Warning: Removed 10 rows containing missing values (geom_point).

Answer to question 3:

I will use almost the same functions I used below to answer the question 1 and 2 which are the select and filter functions. Also, I will use the group_by function

movie2_df3 <- select(movies_df, Title, Genre, Runtime, Gross_Earning_in_Mil)
movie2_df3 <- filter(movie2_df3, Runtime>=100 & Runtime <=120)
Genre_mean <- movie2_df3 %>%
  group_by(Genre) %>%                                       
   summarise_at(vars(Gross_Earning_in_Mil), mean, na.rm = TRUE)
Genre_mean[which.max(Genre_mean$Gross_Earning_in_Mil),] # find the highest average gross earnings
## # A tibble: 1 × 2
##   Genre     Gross_Earning_in_Mil
##   <fct>                    <dbl>
## 1 Animation                 216.

Animation movies has the highest average gross earnings in runtime 100 to 120.The Gross_Earning_in_Mil is 216.33.