Load Libraries

#install.packages('rvest')
library('rvest')
library(tidyverse)
## -- Attaching packages --------------------------------------- tidyverse 1.3.1 --
## v ggplot2 3.3.5     v purrr   0.3.4
## v tibble  3.1.4     v dplyr   1.0.7
## v tidyr   1.1.3     v stringr 1.4.0
## v readr   2.0.1     v forcats 0.5.1
## -- Conflicts ------------------------------------------ tidyverse_conflicts() --
## x dplyr::filter()         masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag()            masks stats::lag()
library(dplyr)
library(plotly)
## 
## Attaching package: 'plotly'
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## The following object is masked from 'package:stats':
## 
##     filter
## The following object is masked from 'package:graphics':
## 
##     layout
library(scales)
## 
## Attaching package: 'scales'
## The following object is masked from 'package:purrr':
## 
##     discard
## The following object is masked from 'package:readr':
## 
##     col_factor
library(highcharter)
## Registered S3 method overwritten by 'quantmod':
##   method            from
##   as.zoo.data.frame zoo
library(RColorBrewer)

Specifying the url for Desired Website to be Scraped and Reading the HTML Code

#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)

Using Selector Gadget and R Code to Scrape the Movie Rankings

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."

Convert All the Rankings to Numeric and display the first rows.

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6

Using Selector Gadget and R Code to scrape all the titles

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Nocturnal Animals" "Train to Busan"    "Arrival"          
## [4] "Suicide Squad"     "Deadpool"          "Hush"

Using Selector Gadget and R Code to scrape all the Movies Descriptions

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\nA wealthy art gallery owner is haunted by her ex-husband's novel, a violent thriller she interprets as a symbolic revenge tale."                                                    
## [2] "\nWhile a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."                                                                   
## [3] "\nA linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appear around the world."                                                 
## [4] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [5] "\nA wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                              
## [6] "\nA deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."

Reformating the Description Field values

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)
## [1] "A wealthy art gallery owner is haunted by her ex-husband's novel, a violent thriller she interprets as a symbolic revenge tale."                                                    
## [2] "While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan."                                                                   
## [3] "A linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appear around the world."                                                 
## [4] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [5] "A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                              
## [6] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."

Using Selector Gadget and R Code to scrape all the Movies Runtime

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "116 min" "118 min" "116 min" "123 min" "108 min" "82 min"

Reformatting the values for the Runtime Field and Converting them to numerical

#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 116 118 116 123 108  82

Using Selector Gadget and R Code to scrape all the Movies Genre

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nDrama, Thriller            "           
## [2] "\nAction, Horror, Thriller            "  
## [3] "\nDrama, Sci-Fi            "             
## [4] "\nAction, Adventure, Fantasy            "
## [5] "\nAction, Adventure, Comedy            " 
## [6] "\nHorror, Thriller            "

Reformatting the values for the Genre Field

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Drama  Action Drama  Action Action Horror
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror

Using Selector Gadget and R Code to scrape all the IMDB Ratings

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "7.5" "7.6" "7.9" "5.9" "8.0" "6.6"

Reformatting the values for the Genre Field

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 7.5 7.6 7.9 5.9 8.0 6.6

Using Selector Gadget and R Code to Scrape all the Votes

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "254,744" "192,162" "640,355" "654,312" "954,445" "118,602"

Reformatting the values for the Votes_data Field and Convert All Values to Numeric

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1] 254744 192162 640355 654312 954445 118602

Using Selector Gadget and R Code to Scrape the Directors Section

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Tom Ford"         "Sang-ho Yeon"     "Denis Villeneuve" "David Ayer"      
## [5] "Tim Miller"       "Mike Flanagan"

Reformatting the values for the Directors Field as Factor

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Amy Adams"          "Gong Yoo"           "Amy Adams"         
## [4] "Will Smith"         "Ryan Reynolds"      "John Gallagher Jr."

Using Selector Gadget and R Code to Scrape the Metascore Section

ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>% 
# scrape the ratings bar and convert to text
  html_text2()
head(ratings_bar_data)                                # look at the ratings bar
## [1] "7.5\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.5/10 X \n67 Metascore"
## [2] "7.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.6/10 X \n72 Metascore"
## [3] "7.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.9/10 X \n81 Metascore"
## [4] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [5] "8.0\nRate this\n 1 2 3 4 5 6 7 8 9 10 8/10 X \n65 Metascore"  
## [6] "6.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.6/10 X \n67 Metascore"
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%  
# extract Metascore 
  str_match("\\d{2}") %>% 
  as.numeric()                                         # convert to number  
length(metascore_data)
## [1] 100
metascore_data
##   [1] 67 72 81 40 65 67 81 65 71 71 72 62 60 81 48 84 44 94 79 75 65 72 51 54 52
##  [26] 78 66 25 82 74 70 35 76 51 81 49 57 74 58 51 48 65 41 65 59 NA 68 99 96 44
##  [51] 57 62 42 NA 64 42 35 58 88 78 32 81 66 42 42 46 79 69 65 32 58 33 77 47 23
##  [76] NA 59 64 52 68 78 77 68 49 61 36 70 76 66 33 65 18 58 NA 40 80 60 55 44 55
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   18.00   48.75   64.00   60.78   72.50   99.00       4

Using Selector Gadget and R Code to Scrape the gross revenue Section

# scrape the votess bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>% 
  html_text2()
head(votes_bar_data)                                      # look at the votes bar data
## [1] "Votes: 254,744 | Gross: $10.64M"  "Votes: 192,162 | Gross: $2.13M"  
## [3] "Votes: 640,355 | Gross: $100.55M" "Votes: 654,312 | Gross: $325.10M"
## [5] "Votes: 954,445 | Gross: $363.07M" "Votes: 118,602"
gross_data <- str_match(votes_bar_data, "\\$.+$")          # extract the gross earnings
gross_data <- gsub("M","",gross_data)                      # clean data: remove 'M' sign 
gross_data <- substring(gross_data,2,6) %>%               # clean data: remove '$' sign                    
  as.numeric()
length(gross_data)
## [1] 100

Create data frame with all fields

#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,

Director = directors_data, Actor = actors_data)

#Structure of the data frame

str(movies_df)
## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Nocturnal Animals" "Train to Busan" "Arrival" "Suicide Squad" ...
##  $ Description         : chr  "A wealthy art gallery owner is haunted by her ex-husband's novel, a violent thriller she interprets as a symbol"| __truncated__ "While a zombie virus breaks out in South Korea, passengers struggle to survive on the train from Seoul to Busan." "A linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appea"| __truncated__ "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ ...
##  $ Runtime             : num  116 118 116 123 108 82 99 134 88 139 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 7 1 7 1 1 8 7 8 6 4 ...
##  $ Rating              : num  7.5 7.6 7.9 5.9 8 6.6 7 7.3 7.1 8.1 ...
##  $ Metascore           : num  67 72 81 40 65 67 81 65 71 71 ...
##  $ Votes               : num  254744 192162 640355 654312 954445 ...
##  $ Gross_Earning_in_Mil: num  10.64 2.13 100.5 325.1 363 ...
##  $ Director            : Factor w/ 97 levels "Aisling Walsh",..: 94 84 27 23 92 60 49 42 31 58 ...
##  $ Actor               : chr  "Amy Adams" "Gong Yoo" "Amy Adams" "Will Smith" ...

#Create First Visualization

qplot(data = movies_df,Runtime,fill = Genre,bins = 30)

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

To answer the above question lets’s filter the data.

movies_df %>%
  select(Title, Genre, Runtime)%>%
  filter(Runtime > 150)
##                                Title     Genre Runtime
## 1 Batman v Superman: Dawn of Justice    Action     152
## 2                        The Wailing    Horror     156
## 3                            Silence     Drama     161
## 4                             Dangal    Action     161
## 5                     American Honey Adventure     163

American Honey, an Adventure movie, had the longest runtime in 2016.

Create Second visualization

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

Assuming that by “highest votes”, the author meant Highest Rating -

Let’s recreate this vizualization. This time, we will add some filters and make the graph interactive.

intmoviedf <- movies_df %>%
  filter(Runtime >= 130 & Runtime <= 160) %>%
  ggplot(aes(Runtime,Rating, colour=Genre))+
  geom_point(aes(size=Votes, col=Genre))+
  scale_color_discrete(name =  " ")+
  labs(x="Runtime", y="Rating", size="Genre: Sized by Qty of Votes") 
      
intmoviedf <- ggplotly(intmoviedf)
intmoviedf

Answer: Animation has the highest rating

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 11 rows containing missing values (geom_point).

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

Looking at the Graph, Adventure and Action, seem to have the highest Gross Earnings

Let’s modify the Graph

gen100_120 <- movies_df %>%
  select(Genre, Runtime, Rating, Gross_Earning_in_Mil)%>%
  filter(Runtime>=100 & Runtime <=120)%>%
  arrange(Runtime)
cols <- brewer.pal(8, "Dark2")

highchart() %>%
  hc_add_series(data = gen100_120, type = "bubble",
                hcaes(x=Runtime,y=Gross_Earning_in_Mil, group= Genre))%>%
  hc_colors(cols)%>%
  hc_title(text="Gross Earnings")%>%
  hc_xAxis(title=list(text="Runtime" ))%>%
  hc_yAxis(title=list(text="Gross Earning in Millions"))%>%
  hc_legend(align = "right", verticalAlign="top")%>%
  hc_tooltip(shared=TRUE, borderColor = "black", pointFormat="{point.y}")

By adding the additional features to the chart, we can clearly see that

adventure has the the highest average gross earning in runtime 100 to 120