Installing necessary packages

library(tidyverse)
## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.0 ──
## ✓ ggplot2 3.3.3     ✓ purrr   0.3.4
## ✓ tibble  3.0.6     ✓ dplyr   1.0.3
## ✓ tidyr   1.1.2     ✓ stringr 1.4.0
## ✓ readr   1.4.0     ✓ forcats 0.5.1
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## x dplyr::filter() masks stats::filter()
## x dplyr::lag()    masks stats::lag()
library(rvest)
## Loading required package: xml2
## 
## Attaching package: 'rvest'
## The following object is masked from 'package:purrr':
## 
##     pluck
## The following object is masked from 'package:readr':
## 
##     guess_encoding

Scraping the IMDB website to create a dataframe containing information of top 100 movies from 2016

#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)
# save_url(webpage, filename="webpage.html")

Loading the following elements:

Ranking, Title, Description, Runtime, Genre, Rating, Votes, Directors, Actors, Metascore and Gross revenue

Using the gsub command to clean data and the length command to ensure that each list contains 100 elements or NAs for missing data to sum to 100 elements

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6
#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Suicide Squad"                      "Batman v Superman: Dawn of Justice"
## [3] "Captain America: Civil War"         "Captain Fantastic"                 
## [5] "Deadpool"                           "The Accountant"
#title_data
#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\n    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                                             
## [2] "\n    Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."                                                                                   
## [3] "\n    Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."                                                                                                                                              
## [4] "\n    In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] "\n    A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                                           
## [6] "\n    As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."
#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)
## [1] "    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                                             
## [2] "    Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."                                                                                   
## [3] "    Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."                                                                                                                                              
## [4] "    In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] "    A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                                           
## [6] "    As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."
#length(description_data)
#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "123 min" "152 min" "147 min" "118 min" "108 min" "128 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 123 152 147 118 108 128
#length(runtime_data)
#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nAction, Adventure, Fantasy            "
## [2] "\nAction, Adventure, Sci-Fi            " 
## [3] "\nAction, Adventure, Sci-Fi            " 
## [4] "\nComedy, Drama            "             
## [5] "\nAction, Adventure, Comedy            " 
## [6] "\nAction, Crime, Drama            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Action Action Action Comedy Action Action
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
#length(genre_data)
#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "6.0" "6.4" "7.8" "7.9" "8.0" "7.3"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 6.0 6.4 7.8 7.9 8.0 7.3
#length(rating_data)
#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "612,320" "643,283" "676,198" "194,563" "913,869" "264,395"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1] 612320 643283 676198 194563 913869 264395
#length(votes_data)
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "David Ayer"     "Zack Snyder"    "Anthony Russo"  "Matt Ross"     
## [5] "Tim Miller"     "Gavin O'Connor"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
#length(directors_data)
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Will Smith"      "Ben Affleck"     "Chris Evans"     "Viggo Mortensen"
## [5] "Ryan Reynolds"   "Ben Affleck"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
#length(actors_data)
#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore 
#data
head(metascore_data)       
## [1] "40        " "44        " "75        " "72        " "65        "
## [6] "51        "
#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)
## [1] 97

Metascore is missing from some of the listings and it changes daily

I find them manually and type them in the vector below to fill missing gross data with NAs using a for loop

for (i in c(18, 57, 100)){
  
  a<-metascore_data[1:(i-1)]
 
  b<-metascore_data[i:length(metascore_data)]
 
  metascore_data<-append(a,list("NA"))
 
  metascore_data<-append(metascore_data,b)
  
}

#Data-Preprocessing: converting metascore to numerical
#metascore_data<-as.numeric(metascore_data)

metascore_data <- metascore_data[-c(101, 102)]
metascore_data <- as.numeric(metascore_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at length of the metascore data

length(metascore_data)
## [1] 100
#Let's look at summary statistics
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   25.00   48.00   62.00   60.44   72.00   99.00       3
#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)
## [1] "$325.10M" "$330.36M" "$408.08M" "$5.88M"   "$363.07M" "$86.26M"
#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)
## [1] 92

Gross revenue is missing from some of the listings and it changes daily

I find them manually and type them in the vector below to fill missing gross data with NAs using a for loop

#Filling missing entries with NA
for (i in c(18, 67, 73, 75, 83, 87, 98, 100)){

a <- gross_data[1:(i-1)]

b <- gross_data[i:length(gross_data)]

gross_data <- append(a,list("NA")) # used -1 in place of NA's

gross_data <- append(gross_data, b)

}

gross_data <- gross_data[-c(101, 102)]
gross_data <- as.numeric(gross_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at the length of gross data
length(gross_data)
## [1] 100
summary(gross_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.04   14.39   52.30   91.32  116.15  532.10       8

Combining all the lists to form a data frame

movies_df<-data.frame(Rank = rank_data, Title = title_data,
                      Description = description_data, Runtime = runtime_data,
                      Genre = genre_data, Rating = rating_data,
                      Metascore = metascore_data, Votes = votes_data,
                      Gross_Earning_in_Mil = gross_data,
                      Director = directors_data, Actor = actors_data)

#Structure of the data frame
str(movies_df)
## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Suicide Squad" "Batman v Superman: Dawn of Justice" "Captain America: Civil War" "Captain Fantastic" ...
##  $ Description         : chr  "    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defens"| __truncated__ "    Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world "| __truncated__ "    Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man." "    In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical "| __truncated__ ...
##  $ Runtime             : num  123 152 147 118 108 128 120 116 107 116 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 1 1 1 5 1 1 1 1 3 7 ...
##  $ Rating              : num  6 6.4 7.8 7.9 8 7.3 6.8 7.4 7.6 7.9 ...
##  $ Metascore           : num  40 44 75 72 65 51 67 70 81 81 ...
##  $ Votes               : num  612320 643283 676198 194563 913869 ...
##  $ Gross_Earning_in_Mil: num  325.1 330.3 408 5.88 363 ...
##  $ Director            : Factor w/ 98 levels "Adam Wingard",..: 23 98 6 61 93 36 40 86 82 27 ...
##  $ Actor               : Factor w/ 91 levels "Aamir Khan","Alexander Skarsgård",..: 89 8 19 88 75 8 39 73 7 3 ...

When looking at the x-axis for the Runtime of movies between 130-160 minutes,

Installing necessary library

library('ggplot2')

First plot: Runtime of Top 100 Movies of 2016 by Genre

qplot(data = movies_df,Runtime,fill = Genre,bins = 30)

# Ans:  (88)    American Honey      Drama     Runtime 163 minutes

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

In the plot above, we observe that there are two movies in particular that appear to have the highest run time compared to all other movies. To find out exactly which movies they are, the genres they belong to as well as their run times we run the code below to find our answer.

q1 <- movies_df %>%
  rownames_to_column(var = "Name") %>% 
  filter(Runtime == max(Runtime))

paste("Title:", q1$Title[1], " Genre: ",q1$Genre[1], " Runtime:",q1$Runtime[1], "mins")
## [1] "Title: Silence  Genre:  Drama  Runtime: 161 mins"
paste("Title:", q1$Title[2], " Genre: ",q1$Genre[2], " Runtime:",q1$Runtime[2], "mins")
## [1] "Title: Dangal  Genre:  Action  Runtime: 161 mins"

Second plot: Runtime of Top 100 Movies of 2016 by Rating

with the color of each point based on genre and the size based on the number of votes

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

# Ans:   A Silent Voice: The Movie    Votes: 37,986

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

In the plot above, the size of the circles increase proportionally to the number of votes. When looking at the x-axis for the run time of movies between 130-160 mins, it appears that the Action genre has the highest number of votes. To test our intuition, we run the code below and check the answer.

q2 <-movies_df %>%
  rownames_to_column(var = "Name") %>% 
  filter(Runtime >= 130 & Runtime <= 160) %>%
  filter(Votes == max(Votes))

paste("In the runtime of 130-160 mins, the genre that has the highest votes is", q2$Genre)
## [1] "In the runtime of 130-160 mins, the genre that has the highest votes is Action"

Third plot: Runtime of Top Movies of 2016 by Gross Earnings in Millions

with the color of each point based on genre and the size based on ratings

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 8 rows containing missing values (geom_point).

# Ans:    Action:  $532.1 Million in Gross Earnings

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

In the plot above, the points of interest to us are the ones that are higher up toward the top of the graph. We notice that there are three observations in particular between 100-200 mins that appear to be very close to each other. They sit together in a cluster toward the top of the graph and seem to fall in the Adventure, Action and Animation genre. To get a more precise answer, we run the code below to answer the question.

q3<- movies_df %>%
  rownames_to_column(var = "Name") %>% 
  filter(Runtime >= 100 & Runtime <= 120) %>%
  group_by(Genre) %>%
  summarize(averageGross = mean(Gross_Earning_in_Mil, na.rm = TRUE)) %>%
  filter(averageGross == max(averageGross))

paste("Across all genres the genre that has the highest average gross earnings in runtime 100 to 120 is", q3$Genre, "with", q3$averageGross, "million in average gross revenue.")
## [1] "Across all genres the genre that has the highest average gross earnings in runtime 100 to 120 is Animation with 216.33 million in average gross revenue."