library(rvest)
library(ggthemes)
## Warning: package 'ggthemes' was built under R version 4.1.3
## Warning in register(): Can't find generic `scale_type` in package ggplot2 to
## register S3 method.
library(ggrepel)
## Warning: package 'ggrepel' was built under R version 4.1.3
## Loading required package: ggplot2
library(RColorBrewer)
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(highcharter)
## Warning: package 'highcharter' was built under R version 4.1.3
## Registered S3 method overwritten by 'quantmod':
##   method            from
##   as.zoo.data.frame zoo
## Highcharts (www.highcharts.com) is a Highsoft software product which is
## not free for commercial and Governmental use
library(ggplot2)
library(viridis)
## Warning: package 'viridis' was built under R version 4.1.3
## Loading required package: viridisLite
#Loading the rvest package
library('rvest')

#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)

“Now, we will be scraping the following data from this website.

Step 1-using selector gadget (on IMBD page)

Step 2-copying CSS selector from selector gadget- here, ‘text.primary’

Step 3-Using R code to get rankings

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."

Step 4-Reprocess data to numerical format

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6

Step 5- Select next data type of interest, movie titles, using Selector Gadget (clear it first)

#Step 6- scrape for titles using CSS selector pathway “.lister-item-header a”

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Batman v Superman: Dawn of Justice Ultimate Edition"
## [2] "Fantastic Beasts and Where to Find Them"            
## [3] "Suicide Squad"                                      
## [4] "Deadpool"                                           
## [5] "Batman v Superman: Dawn of Justice"                 
## [6] "Hacksaw Ridge"

#Step 7-Scrape for each of the remaining data categories using selector gadget and CSS code

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\nBatman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and he is framed for murder during an international crisis. The heroes clash and force the neutral Wonder Woman to reemerge."
## [2] "\nThe adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years before Harry Potter reads his book in school."                                                                              
## [3] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                                    
## [4] "\nA wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                                  
## [5] "\nFearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."                                                                          
## [6] "\nWorld War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."
#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)
## [1] "Batman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and he is framed for murder during an international crisis. The heroes clash and force the neutral Wonder Woman to reemerge."
## [2] "The adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years before Harry Potter reads his book in school."                                                                              
## [3] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                                    
## [4] "A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                                  
## [5] "Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."                                                                          
## [6] "World War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."
#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "182 min" "132 min" "123 min" "108 min" "152 min" "139 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 182 132 123 108 152 139
#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nAction, Adventure, Sci-Fi            " 
## [2] "\nAdventure, Family, Fantasy            "
## [3] "\nAction, Adventure, Fantasy            "
## [4] "\nAction, Adventure, Comedy            " 
## [5] "\nAction, Adventure, Sci-Fi            " 
## [6] "\nBiography, Drama, History            "
#Data-Preprocessing: removing '\n'
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data 
head(genre_data)
## [1] Action    Adventure Action    Action    Action    Biography
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "8.5" "7.3" "5.9" "8.0" "6.5" "8.1"
#Data-Preprocessing: converting ratings to numerical
# rating_data<-as.numeric(rating_data)
# 
# #Let's have another look at the ratings data
# head(rating_data)
#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "9,683"   "447,685" "669,354" "989,280" "683,308" "497,087"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1]   9683 447685 669354 989280 683308 497087
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "David Yates"     "David Ayer"      "Tim Miller"      "Zack Snyder"    
## [5] "Mel Gibson"      "Damien Chazelle"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

#Let's have another look at the directors data
head(directors_data)
## [1] David Yates     David Ayer      Tim Miller      Zack Snyder    
## [5] Mel Gibson      Damien Chazelle
## 98 Levels: Alex Proyas Ana Lily Amirpour André Øvredal ... Zack Snyder
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Eddie Redmayne"  "Will Smith"      "Ryan Reynolds"   "Ben Affleck"    
## [5] "Andrew Garfield" "Ryan Gosling"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore data
head(metascore_data)
## [1] "66        " "40        " "65        " "44        " "71        "
## [6] "94        "
#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)
## [1] 96

#Step 8-Here I only have Metascore data for 94 out of the 100 movies I am scraping for; this means 6 movies did not have a Metascore field filled in on IMDB

#Step 9- I don’t want it to map NA for the missing Metascore data. Instead, I want to continue with my analysis but make note of missing data. I visually locate the movies with missing Metascore data and note their rankings in the list: 17, 39, 46, 58, 97, 98. I then use the supplied function, substituing the listed values, to get around this problem.

for (i in c(32,49,81,89)){
  a<-metascore_data[1:(i-1)]
  b<-metascore_data[i:length(metascore_data)]
  metascore_data<-append(a,list("NA"))
  metascore_data<-append(metascore_data,b)
}

#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)
#Let's have another look at length of the metascore data

length(metascore_data)
## [1] 100
#Let's look at summary statistics
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   18.00   47.00   62.00   60.15   74.00   99.00       4

#Step 10-same issue with Gross variable (gross earnings)

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)
## [1] "$234.04M" "$325.10M" "$363.07M" "$330.36M" "$67.21M"  "$151.10M"
#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)
## [1] "$234.04M" "$325.10M" "$363.07M" "$330.36M" "$67.21M"  "$151.10M"
#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)
## [1] 89

#Visually inspect and substitute the appropriate missing values in the concatenate string

for (i in c(17,20,29,31,46,63,89,94,97,98,99)){

a<-gross_data[1:(i-1)]

b<-gross_data[i:length(gross_data)]

gross_data<-append(a,list("NA"))

gross_data<-append(gross_data,b)

}

#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at the length of gross data
length(gross_data)
## [1] 100
#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)

#Let's have another look at the length of gross data
length(gross_data)
## [1] 100
summary(gross_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.18   18.71   57.64   99.38  126.60  532.10      11

#Step 11- Now we have successfully scraped all the 11 features for the 100 most popular feature films released in 2016. Let’s combine them to create a dataframe and inspect its structure.

#Combining all the lists to form a data frame
movies_df <- data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data
)
str(movies_df)
## 'data.frame':    100 obs. of  9 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Batman v Superman: Dawn of Justice Ultimate Edition" "Fantastic Beasts and Where to Find Them" "Suicide Squad" "Deadpool" ...
##  $ Description         : chr  "Batman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and "| __truncated__ "The adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years bef"| __truncated__ "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man"| __truncated__ ...
##  $ Runtime             : num  182 132 123 108 152 139 128 144 115 107 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 1 2 1 1 1 4 5 1 1 3 ...
##  $ Rating              : chr  "8.5" "7.3" "5.9" "8.0" ...
##  $ Metascore           : num  66 40 65 44 71 94 52 72 81 59 ...
##  $ Votes               : num  9683 447685 669354 989280 683308 ...
##  $ Gross_Earning_in_Mil: num  234 325.1 363 330.3 67.2 ...

#Analyzing scraped data

library('ggplot2')

qplot(data = movies_df,Runtime,fill = Genre,bins = 30)

#Question 1: Based on the above data, which movie from which Genre had the longest runtime?

#Batman vs Superman

longest_RuntimeMovie <- movies_df %>%
  filter(Runtime == max(runtime_data))
longest_RuntimeMovie[1,]
##   Rank                                               Title
## 1    1 Batman v Superman: Dawn of Justice Ultimate Edition
##                                                                                                                                                                                                                               Description
## 1 Batman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and he is framed for murder during an international crisis. The heroes clash and force the neutral Wonder Woman to reemerge.
##   Runtime  Genre Rating Metascore Votes Gross_Earning_in_Mil
## 1     182 Action    8.5        66  9683                  234

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

Action

movies_df1 <-
movies_df %>%
 filter(Runtime %in% (130:160))
Highest_votes <- movies_df %>%
  filter(Runtime >= 130 & Runtime <= 160) %>%
  select(Votes, Genre) %>%
  group_by(Genre) %>%
  summarise(Votes = sum(Votes)) %>%
  arrange(desc(Votes))
Highest_votes[1,]
## # A tibble: 1 x 2
##   Genre    Votes
##   <fct>    <dbl>
## 1 Action 2972950
ggplot(movies_df1,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

Horror

gross_earn <- movies_df %>%
  filter(Runtime >= 100 & Runtime <= 120) %>%
  select(Gross_Earning_in_Mil, Genre) %>%
  group_by(Genre) %>%
  summarise(Gross_Earning_in_Mil = mean(Gross_Earning_in_Mil)) %>%
  arrange(desc(Gross_Earning_in_Mil))
gross_earn[1,1]
## # A tibble: 1 x 1
##   Genre 
##   <fct> 
## 1 Horror
movies_df2 <-
movies_df %>%
 filter(Runtime %in% (100:120))
ggplot(movies_df2,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Using size for a discrete variable is not advised.
## Warning: Removed 4 rows containing missing values (geom_point).