library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.2     ✔ readr     2.1.4
## ✔ forcats   1.0.0     ✔ stringr   1.5.0
## ✔ ggplot2   3.4.2     ✔ tibble    3.2.1
## ✔ lubridate 1.9.2     ✔ tidyr     1.3.0
## ✔ purrr     1.0.1     
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(ggplot2)
library(stringr)
#Loading the rvest package
library('rvest')
## 
## Attaching package: 'rvest'
## 
## The following object is masked from 'package:readr':
## 
##     guess_encoding
#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)

Article Rankings

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6

Movie Titles

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Terrifier"       "Suicide Squad"   "Silence"         "Hush"           
## [5] "The Conjuring 2" "Split"

Scraping Descriptions

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\nOn Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                       
## [2] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [3] "\nIn the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."  
## [4] "\nA deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."                         
## [5] "\nEd and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                     
## [6] "\nThree girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."
#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data
head(description_data)
## [1] "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                       
## [2] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [3] "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."  
## [4] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."                         
## [5] "Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                     
## [6] "Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."

Scraping Movie Runtime

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "85 min"  "123 min" "161 min" "82 min"  "134 min" "117 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1]  85 123 161  82 134 117

Scraping Movie Genre

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nHorror, Thriller            "          
## [2] "\nAction, Adventure, Fantasy            "
## [3] "\nDrama, History            "            
## [4] "\nHorror, Thriller            "          
## [5] "\nHorror, Mystery, Thriller            " 
## [6] "\nHorror, Thriller            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Horror Action Drama  Horror Horror Horror
## 9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror

Scraping IMDB Movie Rating

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "5.6" "5.9" "7.2" "6.6" "7.3" "7.3"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 5.6 5.9 7.2 6.6 7.3 7.3

Votes

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "47,734"  "710,230" "119,468" "149,269" "292,311" "532,913"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1]  47734 710230 119468 149269 292311 532913

Scraping Director Data

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Damien Leone"       "David Ayer"         "Martin Scorsese"   
## [4] "Mike Flanagan"      "James Wan"          "M. Night Shyamalan"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

Scraping Actors

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Jenna Kanell"       "Will Smith"         "Andrew Garfield"   
## [4] "John Gallagher Jr." "Vera Farmiga"       "James McAvoy"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

Metascore

ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>% 
# scrape the ratings bar and convert to text
html_text2()
head(ratings_bar_data)
## [1] "5.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.6/10 X "              
## [2] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [3] "7.2\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.2/10 X \n79 Metascore"
## [4] "6.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.6/10 X \n67 Metascore"
## [5] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n65 Metascore"
## [6] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n63 Metascore"
# look at the ratings bar
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%  
# extract Metascore 
 str_match("\\d{2}") %>% as.numeric()

# convert to number  
length(metascore_data)
## [1] 100
metascore_data
##   [1] NA 40 79 67 65 63 71 85 94 59 81 81 55 65 73 70 65 88 57 65 78 81 54 67 60
##  [26] 44 51 41 65 74 71 81 66 96 68 58 76 47 66 NA 82 48 82 NA 44 51 75 42 32 25
##  [51] 66 52 51 99 72 58 77 57 81 37 48 44 72 32 NA 45 44 47 66 46 53 81 69 49 77
##  [76] 62 58 35 33 68 78 42 42 36 46 34 61 60 60 21 38 66 26 59 55 62 NA 68 79 74
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   21.00   47.00   62.00   60.36   72.50   99.00       5

Professor Code

# Professor's Code

# scrape the votes bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
  html_text2()
head(votes_bar_data)
## [1] "Votes: 47,734"                    "Votes: 710,230 | Gross: $325.10M"
## [3] "Votes: 119,468 | Gross: $7.10M"   "Votes: 149,269"                  
## [5] "Votes: 292,311 | Gross: $102.47M" "Votes: 532,913 | Gross: $138.29M"
# look at the votes bar data
gross_data <- str_match(votes_bar_data, "\\$.+$")

# extract the gross earnings
gross_data <- gsub("M","",gross_data)

# clean data: remove 'M' sign 
gross_data <- substring(gross_data,2,6) %>%
  # clean data: remove '$' sign                    
  as.numeric()
length(gross_data)
## [1] 100

Data Frame

#Combining all the lists to form a data frame
movies_df <- data.frame(Rank = rank_data,
                        Title = title_data,
                        Description = description_data,
                        Runtime = runtime_data,
                        Genre = genre_data,
                        Rating = rating_data,
                        Metascore = metascore_data,
                        Votes = votes_data,
                        Gross_Earning_in_Mil = gross_data,
                        Director = directors_data)

#Structure of the data frame

str(movies_df)
## 'data.frame':    100 obs. of  10 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Terrifier" "Suicide Squad" "Silence" "Hush" ...
##  $ Description         : chr  "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown." "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is"| __truncated__ "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence "| __truncated__ ...
##  $ Runtime             : num  85 123 161 82 134 117 139 145 128 108 ...
##  $ Genre               : Factor w/ 9 levels "Action","Adventure",..: 9 1 7 9 9 9 4 7 5 3 ...
##  $ Rating              : num  5.6 5.9 7.2 6.6 7.3 7.3 8.1 8.1 8 7.1 ...
##  $ Metascore           : num  NA 40 79 67 65 63 71 85 94 59 ...
##  $ Votes               : num  47734 710230 119468 149269 292311 ...
##  $ Gross_Earning_in_Mil: num  NA 325.1 7.1 NA 102.4 ...
##  $ Director            : Factor w/ 97 levels "Alessandro Carloni",..: 19 22 61 65 44 59 63 71 18 34 ...

Plot 1

qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
## Warning: `qplot()` was deprecated in ggplot2 3.4.0.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

Answer: Silence

filter(movies_df, Runtime >= 160 & Genre == "Drama")
##   Rank   Title
## 1    3 Silence
##                                                                                                                                                                         Description
## 1 In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism.
##   Runtime Genre Rating Metascore  Votes Gross_Earning_in_Mil        Director
## 1     161 Drama    7.2        79 119468                  7.1 Martin Scorsese

Plot 2

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

Answer: Action (Captain America: Civil War is the only movie that falls within the run time range)

movies_df %>% 
  filter(Runtime >= 130 & Runtime <= 160) %>%
  filter(Votes == max(Votes))
##   Rank                      Title
## 1   47 Captain America: Civil War
##                                                                                          Description
## 1 Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man.
##   Runtime  Genre Rating Metascore  Votes Gross_Earning_in_Mil      Director
## 1     147 Action    7.8        75 830020                  408 Anthony Russo

Plot 3

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 10 rows containing missing values (`geom_point()`).

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in run time 100 to 120.

Answer: Animation

movies_df %>% 
  filter(Runtime >= 100 & Runtime <= 120) %>%
  group_by(Genre) %>% 
  summarize(avgGross = mean(Gross_Earning_in_Mil)) %>% 
  arrange(desc(avgGross))
## # A tibble: 8 × 2
##   Genre     avgGross
##   <fct>        <dbl>
## 1 Animation    216. 
## 2 Adventure    125. 
## 3 Action        89.2
## 4 Drama         48.4
## 5 Horror        46.8
## 6 Comedy        33.9
## 7 Biography     28.7
## 8 Crime         NA