Install necessary packages for this project

#install.packages('rvest')
#Loading the rvest package
library(rvest)
## Loading required package: xml2
library(tidyverse)
## ── Attaching packages ──────────────────────────────────────────────────────────────────────────────────────────── tidyverse 1.3.0 ──
## ✓ ggplot2 3.3.2     ✓ purrr   0.3.4
## ✓ tibble  3.0.3     ✓ dplyr   1.0.1
## ✓ tidyr   1.1.1     ✓ stringr 1.4.0
## ✓ readr   1.3.1     ✓ forcats 0.5.0
## ── Conflicts ─────────────────────────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
## x dplyr::filter()         masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag()            masks stats::lag()
## x purrr::pluck()          masks rvest::pluck()
library(plotly)
## 
## Attaching package: 'plotly'
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## The following object is masked from 'package:stats':
## 
##     filter
## The following object is masked from 'package:graphics':
## 
##     layout

Scrape the IMDB website to create a dataframe of information from 2016 top 100 movies

http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature

#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)
# save_url(webpage, filename="webpage.html")

Load various elements and clean data using gsub.

Use the command, length, to ensure that each list contains 100 elements or NAs for missing data to sum to 100 elements.

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6
length(rank_data)
## [1] 100

Get Titles Of Movies

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Split"                         "Ghostbusters: Answer the Call"
## [3] "Suicide Squad"                 "Train to Busan"               
## [5] "The Conjuring 2"               "Hush"

Get Descriptions of Movies

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\n    Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."                                                
## [2] "\n    Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat."
## [3] "\n    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                          
## [4] "\n    While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."                                                                                             
## [5] "\n    Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                                               
## [6] "\n    A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."
#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)
## [1] "    Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."                                                
## [2] "    Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat."
## [3] "    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                          
## [4] "    While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."                                                                                             
## [5] "    Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                                               
## [6] "    A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."
length(description_data)
## [1] 100

Generate Movie Runtimes/Lengths

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "117 min" "117 min" "123 min" "118 min" "134 min" "82 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 117 117 123 118 134  82
length(runtime_data)
## [1] 100

Extract Genres and Remove Unneccessary Elements

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nHorror, Thriller            "          
## [2] "\nAction, Comedy, Fantasy            "   
## [3] "\nAction, Adventure, Fantasy            "
## [4] "\nAction, Horror, Thriller            "  
## [5] "\nHorror, Mystery, Thriller            " 
## [6] "\nHorror, Thriller            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Horror Action Action Action Horror Horror
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
length(genre_data)
## [1] 100

Extract Ratings

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "7.3" "6.5" "6.0" "7.6" "7.3" "6.6"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 7.3 6.5 6.0 7.6 7.3 6.6
length(rating_data)
## [1] 100

Extract Vote Data

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "413,257" "201,679" "591,491" "158,210" "221,118" "100,220"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1] 413257 201679 591491 158210 221118 100220
length(votes_data)
## [1] 100

Extract Director Information

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "M. Night Shyamalan" "Paul Feig"          "David Ayer"        
## [4] "Sang-ho Yeon"       "James Wan"          "Mike Flanagan"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
length(directors_data)
## [1] 100

Extract Actor Information

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "James McAvoy"       "Melissa McCarthy"   "Will Smith"        
## [4] "Yoo Gong"           "Vera Farmiga"       "John Gallagher Jr."
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
length(actors_data)
## [1] 100

Fill missing metascores with NAs using a for loop

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore 
head(metascore_data)       
## [1] "62        " "60        " "40        " "72        " "65        "
## [6] "67        "
#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
head(metascore_data)
## [1] "62" "60" "40" "72" "65" "67"
length(metascore_data)
## [1] 96

Automate identification of missing metascore values

Create a dataframe of titles and metascores and then use a for loop to identify any metascores with missing values

# Create dataframe
title <- html_nodes(webpage, '.lister-item-header a')
title <- html_text(title)
metascore <- html_node(html_nodes(webpage, '.lister-item-content'), '.ratings-metascore span')
metascore <- html_text(metascore)
df <- data.frame(Title = title, Metascore = metascore)
head(df)
##                           Title  Metascore
## 1                         Split 62        
## 2 Ghostbusters: Answer the Call 60        
## 3                 Suicide Squad 40        
## 4                Train to Busan 72        
## 5               The Conjuring 2 65        
## 6                          Hush 67
missing_rows = vector(mode = "numeric", length = nrow(df))

# Find Missing Values
for (i in 1:nrow(df)){
  if(is.na(df[i,2])){
    missing_rows[i] <- i
  }
}

head(missing_rows)
## [1] 0 0 0 0 0 0
length(missing_rows)
## [1] 100
# Add Missing Values
for (i in 1:length(missing_rows)){
  if((missing_rows[i])!=0){
      a<-metascore_data[1:(i-1)]
      b<-metascore_data[i:length(metascore_data)]
      metascore_data<-append(a,list("NA"))
      metascore_data<-append(metascore_data,b)
  }
}

#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at length of the metascore data

length(metascore_data)
## [1] 100
#Let's look at summary statistics
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   22.00   47.00   60.00   59.54   72.00   99.00       4

Fill missing gross data with NAs using a for loop

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)
## [1] "$138.29M" "$128.34M" "$325.10M" "$2.13M"   "$102.47M" "$6.86M"
#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)
## [1] 90

Automate identification of missing gross values

Create a dataframe of titles and gross earnings and then use a for loop to identify any gross earnings with missing values

# Create dataframe
gross <- html_node(html_nodes(webpage, '.lister-item-content'), '.ghost~ .text-muted+ span')
gross <- html_text(gross)
df2 <- data.frame(Title = title, Gross = gross)
head(df2)
##                           Title    Gross
## 1                         Split $138.29M
## 2 Ghostbusters: Answer the Call $128.34M
## 3                 Suicide Squad $325.10M
## 4                Train to Busan   $2.13M
## 5               The Conjuring 2 $102.47M
## 6                          Hush     <NA>
missing_rows2 = vector(mode = "numeric", length = nrow(df2))

# Find Missing Values
for (i in 1:nrow(df2)){
  if(is.na(df2[i,2])){
    missing_rows2[i] <- i
  }
}

head(missing_rows2)
## [1] 0 0 0 0 0 6
length(missing_rows2)
## [1] 100
#Filling missing entries with NA
for (i in 1:length(missing_rows2)){
  if((missing_rows2[i])!=0){
    a <- gross_data[1:(i-1)]
    b <- gross_data[i:length(gross_data)]
    gross_data <- append(a, -1) # used -1 in place of NA's
    gross_data <- append(gross_data, b)
  }
}

gross_data <- na.exclude(gross_data)
gross_data <- gross_data[-c(101)]
gross_data <- as.numeric(gross_data)

#Let's have another look at the length of gross data
length(gross_data)
## [1] 100
summary(gross_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   -1.00    5.71   46.45   77.67   98.27  532.10

Combine all the lists to form a data frame

movies_df<-data.frame(Rank = rank_data, Title = title_data,
                      
                      Description = description_data, Runtime = runtime_data,
                      
                      Genre = genre_data, Rating = rating_data,
                      
                      Metascore = metascore_data, Votes = votes_data,                                                     
                      
                      Gross_Earning_in_Mil = gross_data,
                      
                      Director = directors_data, Actor = actors_data)

#Structure of the data frame

str(movies_df)
## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Split" "Ghostbusters: Answer the Call" "Suicide Squad" "Train to Busan" ...
##  $ Description         : chr  "    Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape befo"| __truncated__ "    Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engine"| __truncated__ "    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defens"| __truncated__ "    While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan." ...
##  $ Runtime             : num  117 117 123 118 134 82 83 108 144 107 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 8 1 1 1 8 8 1 1 1 3 ...
##  $ Rating              : num  7.3 6.5 6 7.6 7.3 6.6 6.2 8 6.9 7.6 ...
##  $ Metascore           : num  62 60 40 72 65 67 44 65 52 81 ...
##  $ Votes               : num  413257 201679 591491 158210 221118 ...
##  $ Gross_Earning_in_Mil: num  138.2 128.3 325.1 2.13 102.4 ...
##  $ Director            : Factor w/ 96 levels "Adam Wingard",..: 54 69 21 79 40 61 52 90 12 76 ...
##  $ Actor               : Factor w/ 91 levels "Aamir Khan","Adam Sandler",..: 38 60 90 91 88 42 76 74 38 7 ...

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

Add plotly to get more information on each bar segment

# p1 <- qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
p1 <- movies_df %>%
  ggplot(aes(x=Runtime, fill = Genre)) +
  geom_histogram(position="identity", alpha=0.5, binwidth = 5, color = "white")+
  scale_fill_discrete(name = "Genre") +
  labs(title = "Top 100 Movies of 2016 Runtime by Genre")  
ggplotly(p1)
longest_movie <- movies_df %>% select(Genre, Title, Runtime) %>% group_by(Genre) %>% arrange(desc(Runtime))
head(longest_movie)
## # A tibble: 6 x 3
## # Groups:   Genre [3]
##   Genre  Title                              Runtime
##   <fct>  <chr>                                <dbl>
## 1 Drama  American Honey                         163
## 2 Drama  Silence                                161
## 3 Action Dangal                                 161
## 4 Horror The Wailing                            156
## 5 Action Batman v Superman: Dawn of Justice     152
## 6 Drama  Brimstone                              148
# The movie American Honey, a drama movie, had the longest runtime at 163 minutes.

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

p2 <- movies_df %>%
  ggplot(aes(x=Runtime,y=Rating))+
  geom_point(aes(size=Votes,col=Genre, text = paste("Movie Title:", title_data)), alpha = 0.7) +
  labs(title = "Top 100 Movies of 2016 Runtime by Ratings")  
## Warning: Ignoring unknown aesthetics: text
ggplotly(p2)
highest_votes <- movies_df %>% filter(Runtime >= 130 & Runtime <= 160) %>% group_by(Genre) %>% summarise(total = sum(Votes)) %>% arrange(desc(total))
## `summarise()` ungrouping output (override with `.groups` argument)
highest_votes
## # A tibble: 6 x 2
##   Genre       total
##   <fct>       <dbl>
## 1 Action    2625534
## 2 Drama      469326
## 3 Biography  424316
## 4 Adventure  402761
## 5 Horror     270265
## 6 Animation   43297
# In the runtime between 130 and 160 minutes, action movies had the highest votes, with 2625512.

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

p3 <- movies_df %>%
  ggplot(aes(x=Runtime,y=Gross_Earning_in_Mil))+
  geom_point(aes(size = Rating,col = Genre), alpha = 0.5) +
  labs(title = "Top 100 Movies of 2016 Runtime by Gross Earnings in Millions") +
  scale_y_continuous("Gross Earnings in Millions", limits =c(-10, 600))
ggplotly(p3)
highest_avg_gross <- movies_df %>% filter(Runtime >= 100 & Runtime <= 120) %>% group_by(Genre) %>% summarise(avg = mean(Gross_Earning_in_Mil)) %>% arrange(desc(avg))
## `summarise()` ungrouping output (override with `.groups` argument)
highest_avg_gross
## # A tibble: 8 x 2
##   Genre       avg
##   <fct>     <dbl>
## 1 Animation 216. 
## 2 Adventure 149. 
## 3 Action     77.7
## 4 Drama      49.8
## 5 Comedy     48.1
## 6 Horror     46.8
## 7 Biography  35.9
## 8 Crime      27.2
# Animated movies had the highest average gross earnings of all genres among movies with runtimes between 100 and 120 minutes at about 216 million.