week_9_Web Scraping RMD

Install necessary packages for this project

#install.packages('rvest')
#Loading the rvest package
library('rvest')

## Warning: package 'rvest' was built under R version 4.0.4

library(tidyverse)

## Warning: package 'tidyverse' was built under R version 4.0.4

## -- Attaching packages --------------------------------------- tidyverse 1.3.0 --

## v ggplot2 3.3.3     v purrr   0.3.4
## v tibble  3.0.6     v dplyr   1.0.3
## v tidyr   1.1.2     v stringr 1.4.0
## v readr   1.4.0     v forcats 0.5.1

## -- Conflicts ------------------------------------------ tidyverse_conflicts() --
## x dplyr::filter()         masks stats::filter()
## x readr::guess_encoding() masks rvest::guess_encoding()
## x dplyr::lag()            masks stats::lag()

library(plotly)

## Warning: package 'plotly' was built under R version 4.0.4

## 
## Attaching package: 'plotly'

## The following object is masked from 'package:ggplot2':
## 
##     last_plot

## The following object is masked from 'package:stats':
## 
##     filter

## The following object is masked from 'package:graphics':
## 
##     layout

Scrape the IMDB website to create a dataframe of information from 2016 top 100 movies

#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)

Load various elements and clean data using gsub.

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)

## [1] "1." "2." "3." "4." "5." "6."

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)

## [1] 1 2 3 4 5 6

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)

## [1] "Suicide Squad"                      "Batman v Superman: Dawn of Justice"
## [3] "Captain America: Civil War"         "Captain Fantastic"                 
## [5] "Deadpool"                           "The Accountant"

title_data

##   [1] "Suicide Squad"                              
##   [2] "Batman v Superman: Dawn of Justice"         
##   [3] "Captain America: Civil War"                 
##   [4] "Captain Fantastic"                          
##   [5] "Deadpool"                                   
##   [6] "The Accountant"                             
##   [7] "Shin Godzilla"                              
##   [8] "The Nice Guys"                              
##   [9] "Moana"                                      
##  [10] "Arrival"                                    
##  [11] "Passengers"                                 
##  [12] "Doctor Strange"                             
##  [13] "Split"                                      
##  [14] "Hacksaw Ridge"                              
##  [15] "La La Land"                                 
##  [16] "Nocturnal Animals"                          
##  [17] "Rogue One: A Star Wars Story"               
##  [18] "The Invisible Guest"                        
##  [19] "X-Men: Apocalypse"                          
##  [20] "The Legend of Tarzan"                       
##  [21] "Zootopia"                                   
##  [22] "Me Before You"                              
##  [23] "Hunt for the Wilderpeople"                  
##  [24] "Moonlight"                                  
##  [25] "Sing"                                       
##  [26] "The Handmaiden"                             
##  [27] "Risen"                                      
##  [28] "Star Trek Beyond"                           
##  [29] "The Magnificent Seven"                      
##  [30] "Your Name."                                 
##  [31] "Miss Peregrine's Home for Peculiar Children"
##  [32] "Hidden Figures"                             
##  [33] "Gods of Egypt"                              
##  [34] "Fantastic Beasts and Where to Find Them"    
##  [35] "Don't Breathe"                              
##  [36] "Train to Busan"                             
##  [37] "The Girl on the Train"                      
##  [38] "War Dogs"                                   
##  [39] "Lion"                                       
##  [40] "Manchester by the Sea"                      
##  [41] "Mike and Dave Need Wedding Dates"           
##  [42] "13 Hours"                                   
##  [43] "Ben-Hur"                                    
##  [44] "Allied"                                     
##  [45] "Warcraft"                                   
##  [46] "The Founder"                                
##  [47] "The Conjuring 2"                            
##  [48] "Jason Bourne"                               
##  [49] "Silence"                                    
##  [50] "Ghostbusters: Answer the Call"              
##  [51] "Sausage Party"                              
##  [52] "Hell or High Water"                         
##  [53] "The Neon Demon"                             
##  [54] "Independence Day: Resurgence"               
##  [55] "Everybody Wants Some!!"                     
##  [56] "Inferno"                                    
##  [57] "Dangal"                                     
##  [58] "10 Cloverfield Lane"                        
##  [59] "The BFG"                                    
##  [60] "The 5th Wave"                               
##  [61] "The Infiltrator"                            
##  [62] "Sully"                                      
##  [63] "Bastille Day"                               
##  [64] "Now You See Me 2"                           
##  [65] "Bad Moms"                                   
##  [66] "Jack Reacher: Never Go Back"                
##  [67] "A Silent Voice: The Movie"                  
##  [68] "The Lost City of Z"                         
##  [69] "Assassin's Creed"                           
##  [70] "The Jungle Book"                            
##  [71] "Pride and Prejudice and Zombies"            
##  [72] "Live by Night"                              
##  [73] "Hush"                                       
##  [74] "Keeping Up with the Joneses"                
##  [75] "Below Her Mouth"                            
##  [76] "Colossal"                                   
##  [77] "The Bad Batch"                              
##  [78] "A Cure for Wellness"                        
##  [79] "The Love Witch"                             
##  [80] "The Huntsman: Winter's War"                 
##  [81] "Trolls"                                     
##  [82] "The Edge of Seventeen"                      
##  [83] "The Fundamentals of Caring"                 
##  [84] "Criminal"                                   
##  [85] "Central Intelligence"                       
##  [86] "Finding Dory"                               
##  [87] "Message from the King"                      
##  [88] "Snowden"                                    
##  [89] "Allegiant"                                  
##  [90] "The Exception"                              
##  [91] "The Shallows"                               
##  [92] "London Has Fallen"                          
##  [93] "Hail, Caesar!"                              
##  [94] "Deepwater Horizon"                          
##  [95] "Patriots Day"                               
##  [96] "Blair Witch"                                
##  [97] "Miss Sloane"                                
##  [98] "The Wailing"                                
##  [99] "The Boy"                                    
## [100] "Terrifier"

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)

## [1] "\n    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                                             
## [2] "\n    Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."                                                                                   
## [3] "\n    Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."                                                                                                                                              
## [4] "\n    In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] "\n    A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                                           
## [6] "\n    As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)

## [1] "    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                                             
## [2] "    Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."                                                                                   
## [3] "    Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."                                                                                                                                              
## [4] "    In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical and intellectual education is forced to leave his paradise and enter the world, challenging his idea of what it means to be a parent."
## [5] "    A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                                           
## [6] "    As a math savant uncooks the books for a new client, the Treasury Department closes in on his activities, and the body count starts to rise."

length(description_data)

## [1] 100

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)

## [1] "123 min" "152 min" "147 min" "118 min" "108 min" "128 min"

#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)

## [1] 123 152 147 118 108 128

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)

## [1] "\nAction, Adventure, Fantasy            "
## [2] "\nAction, Adventure, Sci-Fi            " 
## [3] "\nAction, Adventure, Sci-Fi            " 
## [4] "\nComedy, Drama            "             
## [5] "\nAction, Adventure, Comedy            " 
## [6] "\nAction, Crime, Drama            "

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)

## [1] Action Action Action Comedy Action Action
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror

length(runtime_data)

## [1] 100

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)

## [1] "\nAction, Adventure, Fantasy            "
## [2] "\nAction, Adventure, Sci-Fi            " 
## [3] "\nAction, Adventure, Sci-Fi            " 
## [4] "\nComedy, Drama            "             
## [5] "\nAction, Adventure, Comedy            " 
## [6] "\nAction, Crime, Drama            "

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)

## [1] Action Action Action Comedy Action Action
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror

length(genre_data)

## [1] 100

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)

## [1] "6.0" "6.4" "7.8" "7.9" "8.0" "7.3"

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)

## [1] 6.0 6.4 7.8 7.9 8.0 7.3

length(rating_data)

## [1] 100

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)

## [1] "612,605" "643,566" "676,436" "194,667" "914,024" "264,483"

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)

## [1] 612605 643566 676436 194667 914024 264483

length(votes_data)

## [1] 100

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)

## [1] "David Ayer"     "Zack Snyder"    "Anthony Russo"  "Matt Ross"     
## [5] "Tim Miller"     "Gavin O'Connor"

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
length(directors_data)

## [1] 100

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)

## [1] "Will Smith"      "Ben Affleck"     "Chris Evans"     "Viggo Mortensen"
## [5] "Ryan Reynolds"   "Ben Affleck"

#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
length(actors_data)

## [1] 100

Fill missing metascores with NAs using a for loop

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore 
head(metascore_data)  

## [1] "40        " "44        " "75        " "72        " "65        "
## [6] "51        "

#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)

## [1] 97

for (i in c(18, 57, 100)){
 
  a<-metascore_data[1:(i-1)]
 
  b<-metascore_data[i:length(metascore_data)]
 
  metascore_data<-append(a,list("NA"))
 
  metascore_data<-append(metascore_data,b)
 
}

#Data-Preprocessing: converting metascore to numerical

metascore_data <- metascore_data[-c(101, 102)]
metascore_data <- as.numeric(metascore_data)

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

#Let's have another look at length of the metascore data

length(metascore_data)

## [1] 100

#Let's look at summary statistics
summary(metascore_data)

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   25.00   48.00   62.00   60.44   72.00   99.00       3

Fill missing gross data with NAs using a for loop

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)

## [1] "$325.10M" "$330.36M" "$408.08M" "$5.88M"   "$363.07M" "$86.26M"

#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)

## [1] 92

#Filling missing entries with NA
for (i in c(18, 67, 73, 75, 83, 87, 98, 100)){

a <- gross_data[1:(i-1)]
b <- gross_data[i:length(gross_data)]
gross_data <- append(a, -1) # used -1 in place of NA's
gross_data <- append(gross_data, b)
}
gross_data <- na.exclude(gross_data)
gross_data <- gross_data[-c(101)]
gross_data <- as.numeric(gross_data)

#Let's have another look at the length of gross data
length(gross_data)

## [1] 100

summary(gross_data)

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   -1.00    9.93   46.69   83.94  102.58  532.10

Combine all the lists to form a data frame

movies_df<-data.frame(Rank = rank_data, Title = title_data,
                      
                      Description = description_data, Runtime = runtime_data,
                      
                      Genre = genre_data, Rating = rating_data,
                      
                      Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,
                      
                      Director = directors_data, Actor = actors_data)

#Structure of the data frame

str(movies_df)

## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Suicide Squad" "Batman v Superman: Dawn of Justice" "Captain America: Civil War" "Captain Fantastic" ...
##  $ Description         : chr  "    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defens"| __truncated__ "    Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world "| __truncated__ "    Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man." "    In the forests of the Pacific Northwest, a father devoted to raising his six kids with a rigorous physical "| __truncated__ ...
##  $ Runtime             : num  123 152 147 118 108 128 120 116 107 116 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 1 1 1 5 1 1 1 1 3 7 ...
##  $ Rating              : num  6 6.4 7.8 7.9 8 7.3 6.8 7.4 7.6 7.9 ...
##  $ Metascore           : num  40 44 75 72 65 51 67 70 81 81 ...
##  $ Votes               : num  612605 643566 676436 194667 914024 ...
##  $ Gross_Earning_in_Mil: num  325.1 330.3 408 5.88 363 ...
##  $ Director            : Factor w/ 98 levels "Adam Wingard",..: 23 98 6 61 93 36 40 86 82 27 ...
##  $ Actor               : Factor w/ 91 levels "Aamir Khan","Alexander Skarsgård",..: 89 8 19 88 75 8 39 73 7 3 ...

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

Answer 1: There are two movies, which have the longest runtime - the ‘Silence’, 161 min, Drama Gener and the ‘Dangal’, 161 min, Action Gener.

Add plotly to get more information on each bar segment

#p1 <- qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
p1 <- movies_df %>%
  ggplot(aes(x=Runtime, fill = Genre)) +
  geom_histogram(position="identity", alpha=0.5, binwidth = 5, color = "white")+
  scale_fill_discrete(name = "Genre") +
  labs(title = "Top 100 Movies of 2016 Runtime by Genre")  
ggplotly(p1)

movies_df %>%
  rownames_to_column(var = "Name") %>% 
  filter(Runtime == max(Runtime))

##   Name Rank   Title
## 1   49   49 Silence
## 2   57   57  Dangal
##                                                                                                                                                                             Description
## 1     In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism.
## 2                              Former wrestler Mahavir Singh Phogat and his two wrestler daughters struggle towards glory at the Commonwealth Games in the face of societal oppression.
##   Runtime  Genre Rating Metascore  Votes Gross_Earning_in_Mil        Director
## 1     161  Drama    7.2        79 101765                 7.10 Martin Scorsese
## 2     161 Action    8.4        NA 160093                12.39   Nitesh Tiwari
##             Actor
## 1 Andrew Garfield
## 2      Aamir Khan

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

Answer 2: The Action genre has the highest votes - 676436 votes.

p2 <- movies_df %>%
  ggplot(aes(x=Runtime,y=Rating))+
  geom_point(aes(size=Votes,col=Genre, text = paste("Movie Title:", title_data)), alpha = 0.7) +
  labs(title = "Top 100 Movies of 2016 Runtime by Ratings")  

## Warning: Ignoring unknown aesthetics: text

ggplotly(p2)

movies_df %>%
  rownames_to_column(var = "Name") %>% 
  filter(Runtime >= 130 & Runtime <= 160) %>%
  filter(Votes == max(Votes))

##   Name Rank                      Title
## 1    3    3 Captain America: Civil War
##                                                                                              Description
## 1     Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man.
##   Runtime  Genre Rating Metascore  Votes Gross_Earning_in_Mil      Director
## 1     147 Action    7.8        75 676436                  408 Anthony Russo
##         Actor
## 1 Chris Evans

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

Answer 3: The Comedy genre has the highest average gross earning - 113.2 millons.

p3 <- movies_df %>%
  ggplot(aes(x=Runtime,y=Gross_Earning_in_Mil))+
  geom_point(aes(size = Rating,col = Genre), alpha = 0.5) +
  labs(title = "Top 100 Movies of 2016 Runtime by Gross Earnings in Millions") +
  scale_y_continuous("Gross Earnings in Millions", limits =c(-10, 600))
ggplotly(p3)

movies_df %>%
  rownames_to_column(var = "Name") %>% 
  filter(Runtime == c(100,120)) %>%
  group_by(Genre) %>%
  summarize(averageGross = mean(Gross_Earning_in_Mil)) %>%
  filter(averageGross == max(averageGross))

## # A tibble: 1 x 2
##   Genre  averageGross
##   <fct>         <dbl>
## 1 Comedy         113.