Webscraping Assignment

library(tidyverse)

## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.3     ✔ readr     2.1.4
## ✔ forcats   1.0.0     ✔ stringr   1.5.0
## ✔ ggplot2   3.4.3     ✔ tibble    3.2.1
## ✔ lubridate 1.9.2     ✔ tidyr     1.3.0
## ✔ purrr     1.0.2     
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

#Loading the rvest package
library('rvest')

## 
## Attaching package: 'rvest'
## 
## The following object is masked from 'package:readr':
## 
##     guess_encoding

#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)

## [1] "1." "2." "3." "4." "5." "6."

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)

## [1] 1 2 3 4 5 6

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)

## [1] "Terrifier"       "Suicide Squad"   "Silence"         "Hush"           
## [5] "The Conjuring 2" "Split"

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)

## [1] "\nOn Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                       
## [2] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [3] "\nIn the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."  
## [4] "\nA deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."                         
## [5] "\nEd and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                     
## [6] "\nThree girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)

## [1] "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                       
## [2] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [3] "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."  
## [4] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."                         
## [5] "Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                     
## [6] "Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)

## [1] "85 min"  "123 min" "161 min" "82 min"  "134 min" "117 min"

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)

## [1]  85 123 161  82 134 117

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)

## [1] "\nHorror, Thriller            "          
## [2] "\nAction, Adventure, Fantasy            "
## [3] "\nDrama, History            "            
## [4] "\nHorror, Thriller            "          
## [5] "\nHorror, Mystery, Thriller            " 
## [6] "\nHorror, Thriller            "

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Converting each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)

## [1] Horror Action Drama  Horror Horror Horror
## 9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)

## [1] "5.6" "5.9" "7.2" "6.6" "7.3" "7.3"

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)

## [1] 5.6 5.9 7.2 6.6 7.3 7.3

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)

## [1] "47,766"  "710,251" "119,473" "149,289" "292,347" "532,948"

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)

## [1]  47766 710251 119473 149289 292347 532948

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)

## [1] "Damien Leone"       "David Ayer"         "Martin Scorsese"   
## [4] "Mike Flanagan"      "James Wan"          "M. Night Shyamalan"

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)

## [1] "Jenna Kanell"       "Will Smith"         "Andrew Garfield"   
## [4] "John Gallagher Jr." "Vera Farmiga"       "James McAvoy"

#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore data 
head(metascore_data)

## [1] "40        " "79        " "67        " "65        " "63        "
## [6] "71        "

#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)

## [1] 95

Find metascore data with missing values and replace with NAs (this is an automated method)

ratings_bar_data <- html_nodes(webpage,'.ratings-bar') |>
# scrape the ratings bar and convert to text
 html_text2()
head(ratings_bar_data) # look at the ratings bar

## [1] "5.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.6/10 X "              
## [2] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [3] "7.2\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.2/10 X \n79 Metascore"
## [4] "6.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.6/10 X \n67 Metascore"
## [5] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n65 Metascore"
## [6] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n63 Metascore"

4

## [1] 4

metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%
# extract Metascore
 str_match("\\d{2}") |>
 as.numeric() # convert to number
length(metascore_data)

## [1] 100

metascore_data

##   [1] NA 40 79 67 65 63 71 85 94 59 81 81 55 65 73 70 65 88 57 65 78 81 54 67 60
##  [26] 44 51 41 65 74 71 81 66 96 68 58 76 47 66 NA 82 48 82 NA 44 51 75 42 32 25
##  [51] 66 52 51 99 72 58 77 57 81 37 48 44 72 32 NA 45 44 47 66 46 53 81 69 49 77
##  [76] 62 58 35 33 68 78 42 42 36 46 34 61 60 60 21 38 66 26 59 55 62 NA 68 79 74

summary(metascore_data)

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   21.00   47.00   62.00   60.36   72.50   99.00       5

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)

## [1] "$325.10M" "$7.10M"   "$102.47M" "$138.29M" "$67.21M"  "$2.01M"

#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)

## [1] 90

Find the missing gross earnings (automated) Earnings are part of the votes bar in the html, scrape the votes bar and extract earnings with a regular expression to get the NAs in context.

# scrape the votess bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
 html_text2()
head(votes_bar_data) # look at the votes bar data

## [1] "Votes: 47,766"                    "Votes: 710,251 | Gross: $325.10M"
## [3] "Votes: 119,473 | Gross: $7.10M"   "Votes: 149,289"                  
## [5] "Votes: 292,347 | Gross: $102.47M" "Votes: 532,948 | Gross: $138.29M"

gross_data <- str_match(votes_bar_data, "\\$.+$") # extract the gross earnings
gross_data <- gsub("M","",gross_data) # clean data: remove 'M' sign
gross_data <- substring(gross_data,2,6) |> # clean data: remove '$' sign
 as.numeric()
length(gross_data)

## [1] 100

summary(gross_data)

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.01   15.50   55.87   93.09  122.05  532.10      10

#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,

Director = directors_data)

#Structure of the data frame

str(movies_df)

## 'data.frame':    100 obs. of  10 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Terrifier" "Suicide Squad" "Silence" "Hush" ...
##  $ Description         : chr  "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown." "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is"| __truncated__ "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence "| __truncated__ ...
##  $ Runtime             : num  85 123 161 82 134 117 139 145 128 108 ...
##  $ Genre               : Factor w/ 9 levels "Action","Adventure",..: 9 1 7 9 9 9 4 7 5 3 ...
##  $ Rating              : num  5.6 5.9 7.2 6.6 7.3 7.3 8.1 8.1 8 7.1 ...
##  $ Metascore           : num  NA 40 79 67 65 63 71 85 94 59 ...
##  $ Votes               : num  47766 710251 119473 149289 292347 ...
##  $ Gross_Earning_in_Mil: num  NA 325.1 7.1 NA 102.4 ...
##  $ Director            : Factor w/ 97 levels "Alessandro Carloni",..: 19 22 61 65 44 59 63 71 18 34 ...

Question 1: Which movie from which genre had the longest runtime?

q1 <- movies_df |> group_by(Title) |> arrange(desc(Runtime)) |> select(Title, Genre, Runtime)
q1

## # A tibble: 100 × 3
## # Groups:   Title [100]
##    Title                              Genre     Runtime
##    <chr>                              <fct>       <dbl>
##  1 Silence                            Drama         161
##  2 The Wailing                        Drama         156
##  3 Batman v Superman: Dawn of Justice Action        151
##  4 Captain America: Civil War         Action        147
##  5 A Cure for Wellness                Drama         146
##  6 The Handmaiden                     Drama         145
##  7 13 Hours                           Action        144
##  8 X-Men: Apocalypse                  Action        144
##  9 The Lost City of Z                 Adventure     141
## 10 Hacksaw Ridge                      Biography     139
## # ℹ 90 more rows

library(plotly)

## 
## Attaching package: 'plotly'

## The following object is masked from 'package:ggplot2':
## 
##     last_plot

## The following object is masked from 'package:stats':
## 
##     filter

## The following object is masked from 'package:graphics':
## 
##     layout

ggplotly(q1 |> ggplot(aes(x = Title, y = Runtime, color = Genre)) + geom_point())

Answer: “The Silence”, which is in the Drama genre, had the longest runtime (161 minutes).

Question 2: In the Runtime of 130-160 mins, which genre has the highest votes?

q2 <- movies_df |> filter(between(Runtime, 130, 160)) |> group_by(Genre) |> summarise(total_votes = sum(Votes, na.rm = TRUE)) |> arrange(desc(total_votes))
q2

## # A tibble: 5 × 2
##   Genre     total_votes
##   <fct>           <dbl>
## 1 Action        3135104
## 2 Drama          765581
## 3 Adventure      593845
## 4 Biography      571913
## 5 Horror         292347

q2 |> ggplot(aes(x = Genre, y = total_votes, fill = Genre)) +
  geom_col()

Answer: The Action genre has the highest total votes (3,134,919 votes) in the Runtime of 130-160 minutes.

Question 3: Across all genres which genre has the highest average gross earnings in runtime 100 to 120?

q3 <- movies_df |> filter(between(Runtime, 100, 120)) |> group_by(Genre) |> summarise(avg_gross = mean(Gross_Earning_in_Mil, na.rm = TRUE)) |> arrange(desc(avg_gross))
q3

## # A tibble: 8 × 2
##   Genre     avg_gross
##   <fct>         <dbl>
## 1 Animation     216. 
## 2 Adventure     125. 
## 3 Action         89.2
## 4 Crime          51.2
## 5 Drama          48.4
## 6 Horror         46.8
## 7 Comedy         33.9
## 8 Biography      28.7

q3 |> ggplot(aes(x = Genre, y = avg_gross, fill = Genre)) +
  geom_col()

Answer: The Animation genre has the highest average gross earnings ($216.33 million) in the Runtime of 100-120 minutes.