Loading Libraries

library('rvest')
library(tidyverse)
## -- Attaching packages --------------------------------------- tidyverse 1.3.1 --
## v ggplot2 3.3.5     v purrr   0.3.4
## v tibble  3.1.5     v dplyr   1.0.7
## v tidyr   1.1.4     v stringr 1.4.0
## v readr   2.0.2     v forcats 0.5.1
## -- Conflicts ------------------------------------------ tidyverse_conflicts() --
## x dplyr::filter()         masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag()            masks stats::lag()
library(dplyr)

Specifying the url

url <- ('https://www.imdb.com/search/title/?count=100&release_date=2016,2016&title_type=feature')

webpage <- read_html(url)

Scraping the rankings

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."

Converting all the rankings to numeric values

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6

Scraping all the titles

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Arrival"                       "Ghostbusters: Answer the Call"
## [3] "Train to Busan"                "Suicide Squad"                
## [5] "Hacksaw Ridge"                 "Hush"

Scraping all movies description

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\nA linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appear around the world."                                                                                  
## [2] "\nFollowing a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat."       
## [3] "\nWhile a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."                                                                                                    
## [4] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                 
## [5] "\nWorld War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."
## [6] "\nA deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."

Reformating the description

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)
## [1] "A linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appear around the world."                                                                                  
## [2] "Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer Jillian Holtzmann, and subway worker Patty Tolan band together to stop the otherworldly threat."       
## [3] "While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."                                                                                                    
## [4] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                 
## [5] "World War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."
## [6] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."

Scraping all the movies runtime

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "116 min" "117 min" "118 min" "123 min" "139 min" "82 min"

Removing “mins” from runtime and coverting runtime to numerical

#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 116 117 118 123 139  82

Scraping all the movies genre

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nDrama, Sci-Fi            "             
## [2] "\nAction, Comedy, Fantasy            "   
## [3] "\nAction, Horror, Thriller            "  
## [4] "\nAction, Adventure, Fantasy            "
## [5] "\nBiography, Drama, History            " 
## [6] "\nHorror, Thriller            "

Reformatting genre

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Drama     Action    Action    Action    Biography Horror   
## 9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Mystery

Scraping all the IMDB ratings

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "7.9" "6.5" "7.6" "5.9" "8.1" "6.6"

Reformatting the IMDB ratings

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 7.9 6.5 7.6 5.9 8.1 6.6

Scraping all the votes

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "642,216" "214,391" "193,077" "654,847" "472,240" "119,243"

Reformatting votes and coverting them to numeric values

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1] 642216 214391 193077 654847 472240 119243

Scraping the directors

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Denis Villeneuve" "Paul Feig"        "Sang-ho Yeon"     "David Ayer"      
## [5] "Mel Gibson"       "Mike Flanagan"

Reformatting the directors

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Amy Adams"          "Melissa McCarthy"   "Gong Yoo"          
## [4] "Will Smith"         "Andrew Garfield"    "John Gallagher Jr."

Scraping directors and actors

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Amy Adams"          "Melissa McCarthy"   "Gong Yoo"          
## [4] "Will Smith"         "Andrew Garfield"    "John Gallagher Jr."

Reformatting actors

#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

Scraping metascores

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore 
head(metascore_data)
## [1] "81        " "60        " "72        " "40        " "71        "
## [6] "67        "

Removing spaces in metascore

#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)
## [1] 96
for (i in c(39,73,80,89)){

a<-metascore_data[1:(i-1)]

b<-metascore_data[i:length(metascore_data)]

metascore_data<-append(a,list("NA"))

metascore_data<-append(metascore_data,b)

}

#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at length of the metascore data
length(metascore_data)
## [1] 100

Metascore summary

#Let's look at summary statistics
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   18.00   48.75   63.50   61.02   74.25   99.00       4

Scraping gross revenue

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)
## [1] "$100.55M" "$128.34M" "$2.13M"   "$325.10M" "$67.21M"  "$89.22M"

Reformatting gross revenue

#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)
## [1] 89
#Filling missing entries with NA
for (i in c(17,39,49,52,57,64,66,73,76,77,80,87,88,89)){

a<-gross_data[1:(i-1)]

b<-gross_data[i:length(gross_data)]

gross_data<-append(a,list("NA"))

gross_data<-append(gross_data,b)

}

#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at the length of gross data
length(gross_data)
## [1] 103

Gross Revenue summary

summary(gross_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.15   14.43   52.85   87.35  102.40  532.10      14

Reformatting metascores and scapping ratings

## Find metascore data with missing values and replace with NAs (this is an automated method) 
ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>% 
# scrape the ratings bar and convert to text
  html_text2()
head(ratings_bar_data)  
## [1] "7.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.9/10 X \n81 Metascore"
## [2] "6.5\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.5/10 X \n60 Metascore"
## [3] "7.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.6/10 X \n72 Metascore"
## [4] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [5] "8.1\nRate this\n 1 2 3 4 5 6 7 8 9 10 8.1/10 X \n71 Metascore"
## [6] "6.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.6/10 X \n67 Metascore"
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%  
# extract Metascore 
  str_match("\\d{2}") %>% 
  as.numeric()                                         # convert to number  
length(metascore_data)
## [1] 100
metascore_data
##   [1] 81 60 72 40 71 67 71 67 62 65 81 65 72 81 57 75 94 48 84 65 52 44 54 51 79
##  [26] 66 70 72 51 81 78 65 82 NA 59 76 41 99 25 74 32 48 96 58 NA 57 51 88 68 68
##  [51] 62 65 49 35 NA 58 45 42 74 35 23 69 44 79 18 42 42 60 78 23 81 59 77 46 68
##  [76] 58 66 66 78 52 42 36 49 45 55 77 62 70 76 28 79 77 55 90 32 NA 72 58 51 47

Scrapping and converting votes

# scrape the votes bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>% 
  html_text2()
head(votes_bar_data)   
## [1] "Votes: 642,216 | Gross: $100.55M" "Votes: 214,391 | Gross: $128.34M"
## [3] "Votes: 193,077 | Gross: $2.13M"   "Votes: 654,847 | Gross: $325.10M"
## [5] "Votes: 472,240 | Gross: $67.21M"  "Votes: 119,243"
gross_data <- str_match(votes_bar_data, "\\$.+$")          # extract the gross earnings
gross_data <- gsub("M","",gross_data)                      # clean data: remove 'M' sign 
gross_data <- substring(gross_data,2,6) %>%               # clean data: remove '$' sign                    
  as.numeric()
length(gross_data)
## [1] 100

Creating dataframes

#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,
Gross_Earning_in_Mil = gross_data,

Director = directors_data, Actor = actors_data)

#Structure of the data frame

str(movies_df)
## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Arrival" "Ghostbusters: Answer the Call" "Train to Busan" "Suicide Squad" ...
##  $ Description         : chr  "A linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appea"| __truncated__ "Following a ghost invasion of Manhattan, paranormal enthusiasts Erin Gilbert and Abby Yates, nuclear engineer J"| __truncated__ "While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan." "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ ...
##  $ Runtime             : num  116 117 118 123 139 82 88 116 117 134 ...
##  $ Genre               : Factor w/ 9 levels "Action","Adventure",..: 7 1 1 1 4 8 6 7 8 8 ...
##  $ Rating              : num  7.9 6.5 7.6 5.9 8.1 6.6 7.1 7.5 7.3 7.3 ...
##  $ Metascore           : num  81 60 72 40 71 67 71 67 62 65 ...
##  $ Votes               : num  642216 214391 193077 654847 472240 ...
##  $ Gross_Earning_in_Mil: num  100.5 128.3 2.13 325.1 67.21 ...
##  $ Director            : Factor w/ 97 levels "Alex Proyas",..: 26 72 84 21 59 61 31 94 54 41 ...
##  $ Actor               : Factor w/ 91 levels "Aamir Khan","Aaron Poole",..: 5 68 39 91 6 48 82 5 42 89 ...

First Visualization

library('ggplot2')
p1<-qplot(data = movies_df,Runtime,fill=Genre,bins=20)
p1

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

q1 <-movies_df %>% select(Title, Rank, Title, Runtime, Genre) %>%
  filter(Runtime == max(Runtime))
q1
##     Title Rank Runtime  Genre
## 1 Silence   64     161  Drama
## 2  Dangal   96     161 Action

Silence, a drama movie, tied with the longest runtime in 2016 of 161 minutes

Dagal, an action movie, tied with the longest runtime in 2016 of 161 minutes

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

q2 <- movies_df %>%
  filter(Runtime >= 130 & Runtime <= 160) %>%
  select(Votes, Genre) %>%
  group_by(Genre) %>%
  summarise(Votes = sum(Votes)) %>%
  arrange(desc(Votes))
q2[1,1]
## # A tibble: 1 x 1
##   Genre 
##   <fct> 
## 1 Action

Action is the genre of the most votes in the 130-160 min runtime range.

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 11 rows containing missing values (geom_point).

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

q3 <- movies_df %>%
  filter(Runtime >= 100 & Runtime <= 120) %>%
  select(Gross_Earning_in_Mil, Genre) %>%
  group_by(Genre) %>%
  summarise(Gross_Earning_in_Mil = mean(Gross_Earning_in_Mil)) %>%
  arrange(desc(Gross_Earning_in_Mil))
q3[1,1]
## # A tibble: 1 x 1
##   Genre    
##   <fct>    
## 1 Animation

Animation has the highest average gross earnings for the runtime 100-120 minutes.