Week 9 Webscraping HW

Load packages

library(tidyverse)
library(rvest)
library(plotly)

Specify the URL to be scraped

url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

Read the HTML code from the website above

webpage <- read_html(url)

Ranking

#Use CSS selectors to scrape rankings
rank_data_html <- html_nodes(webpage,'.text-primary')

#Convert the ranking data to text
rank_data <- html_text(rank_data_html)

#View the ranking data
head(rank_data)

## [1] "1." "2." "3." "4." "5." "6."

#Convert rankings to numerical format
rank_data<-as.numeric(rank_data)

#View rank data again
head(rank_data)

## [1] 1 2 3 4 5 6

Title

#Use CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Convert the title data to text
title_data <- html_text(title_data_html)

#View the title data
head(title_data)

## [1] "Terrifier"       "Suicide Squad"   "Silence"         "Hush"           
## [5] "The Conjuring 2" "Split"

Description

#Use CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Convert the description data to text
description_data <- html_text(description_data_html)

#View the description data
head(description_data)

## [1] "\nOn Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                       
## [2] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [3] "\nIn the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."  
## [4] "\nA deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."                         
## [5] "\nEd and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                     
## [6] "\nThree girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)

## [1] "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                       
## [2] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [3] "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."  
## [4] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."                         
## [5] "Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                     
## [6] "Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."

Runtime

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)

## [1] "85 min"  "123 min" "161 min" "82 min"  "134 min" "117 min"

#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)

## [1]  85 123 161  82 134 117

Genre

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)

## [1] "\nHorror, Thriller            "          
## [2] "\nAction, Adventure, Fantasy            "
## [3] "\nDrama, History            "            
## [4] "\nHorror, Thriller            "          
## [5] "\nHorror, Mystery, Thriller            " 
## [6] "\nHorror, Thriller            "

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Converting each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)

## [1] Horror Action Drama  Horror Horror Horror
## 9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror

Rating

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)

## [1] "5.6" "5.9" "7.2" "6.6" "7.3" "7.3"

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)

## [1] 5.6 5.9 7.2 6.6 7.3 7.3

Metascore

Find metascore data with missing values and replace with NAs (automated method)

ratings_bar_data <- html_nodes(webpage, '.ratings-bar') |>
  # scrape the ratings bar and convert to text
  html_text2()

head(ratings_bar_data) #look at the ratings bar

## [1] "5.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.6/10 X "              
## [2] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [3] "7.2\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.2/10 X \n79 Metascore"
## [4] "6.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.6/10 X \n67 Metascore"
## [5] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n65 Metascore"
## [6] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n63 Metascore"

metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") |>
  # extract metascore
  str_match("\\d{2}") |>
  as.numeric() # convert to number
length(metascore_data)

## [1] 100

metascore_data

##   [1] NA 40 79 67 65 63 71 85 94 59 81 81 55 65 73 70 65 88 57 65 78 81 54 67 60
##  [26] 44 51 41 65 74 71 81 66 96 68 58 76 47 66 NA 82 48 82 NA 44 51 75 42 32 25
##  [51] 66 52 51 99 72 58 77 57 81 37 48 44 72 32 NA 45 44 47 66 46 53 81 69 49 77
##  [76] 62 58 35 33 68 78 42 42 36 46 34 61 60 60 21 38 66 26 59 55 62 NA 68 79 74

summary(metascore_data)

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   21.00   47.00   62.00   60.36   72.50   99.00       5

Votes

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)

## [1] "47,604"  "710,171" "119,435" "149,202" "292,206" "532,800"

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)

## [1]  47604 710171 119435 149202 292206 532800

Gross_Earning_in_Mil

Find the missing gross earnings (automated). Scrape the votes bar to extract earnings with a regular expression to get NAs in context (earnings are a part of the votes bar)

# scrape the votes bar and convert to text

votes_bar_data <- html_nodes(webpage, '.sort-num_votes-visible') |>
  html_text2()
head(votes_bar_data) #look at the votes bar data

## [1] "Votes: 47,604"                    "Votes: 710,171 | Gross: $325.10M"
## [3] "Votes: 119,435 | Gross: $7.10M"   "Votes: 149,202"                  
## [5] "Votes: 292,206 | Gross: $102.47M" "Votes: 532,800 | Gross: $138.29M"

gross_data <- str_match(votes_bar_data, "\\$.+$") #extract the gross earnings
gross_data <- gsub("M", "", gross_data) #clean data- remove "M"
gross_data <- substring(gross_data,2,6) |>
  as.numeric() #clean data: remove $ sign
gross_data

##   [1]     NA 325.10   7.10     NA 102.40 138.20  67.21   2.01 151.10 270.40
##  [11] 100.50 248.70 153.70 363.00   2.13  36.26 532.10  27.01  87.24   0.01
##  [21] 341.20   5.02  93.43  10.66 128.30 126.60  56.25 100.00  35.14 169.60
##  [31]  89.22     NA 234.00  47.70     NA  67.27  72.08  58.70  97.69     NA
##  [41]   1.03  52.85   0.23     NA 330.30  86.26 408.00     NA 103.10  31.15
##  [51]  12.79 155.40   1.33  27.85 232.60 162.40 364.00  43.03   0.51  59.69
##  [61]  75.40   6.86   5.88  47.37     NA  10.91  10.16   8.11  55.48     NA
##  [71]  20.76   5.20  51.74   7.23  14.43   0.18  38.58   0.78  34.92  61.43
##  [81]   8.58  45.54  35.82  54.65  65.08  77.04 368.30 113.20  40.10  35.59
##  [91]  21.22 143.50  18.71  55.12  79.21   0.15     NA 158.80  57.68 125.00

length(gross_data)

## [1] 100

summary(gross_data)

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.01   15.50   55.87   93.09  122.05  532.10      10

Director

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)

## [1] "Damien Leone"       "David Ayer"         "Martin Scorsese"   
## [4] "Mike Flanagan"      "James Wan"          "M. Night Shyamalan"

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

Actor

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)

## [1] "Jenna Kanell"       "Will Smith"         "Andrew Garfield"   
## [4] "John Gallagher Jr." "Vera Farmiga"       "James McAvoy"

#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

Create a dataframe

#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,

Director = directors_data) #Actor = actors_data is temporarily omitted since there are only 99 observations for this variable but 100 for all the others

#Structure of the data frame

str(movies_df)

## 'data.frame':    100 obs. of  10 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Terrifier" "Suicide Squad" "Silence" "Hush" ...
##  $ Description         : chr  "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown." "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is"| __truncated__ "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence "| __truncated__ ...
##  $ Runtime             : num  85 123 161 82 134 117 139 145 128 108 ...
##  $ Genre               : Factor w/ 9 levels "Action","Adventure",..: 9 1 7 9 9 9 4 7 5 3 ...
##  $ Rating              : num  5.6 5.9 7.2 6.6 7.3 7.3 8.1 8.1 8 7.1 ...
##  $ Metascore           : num  NA 40 79 67 65 63 71 85 94 59 ...
##  $ Votes               : num  47604 710171 119435 149202 292206 ...
##  $ Gross_Earning_in_Mil: num  NA 325.1 7.1 NA 102.4 ...
##  $ Director            : Factor w/ 97 levels "Alessandro Carloni",..: 19 22 61 65 44 59 63 71 18 34 ...

Data analysis and visualizations.

movies_df |>
  ggplot(aes(Runtime, fill = Genre, bins = 30)) +
  geom_bar()

Based on the above data, which movie from which Genre had the longest runtime?

movies_df |>
  group_by(Genre) |>
  slice_max(Runtime, n = 1) |>
  relocate(Genre) |>
  arrange(desc(Runtime))

## # A tibble: 10 × 10
## # Groups:   Genre [9]
##    Genre      Rank Title             Description Runtime Rating Metascore  Votes
##    <fct>     <dbl> <chr>             <chr>         <dbl>  <dbl>     <dbl>  <dbl>
##  1 Drama         3 Silence           In the 17t…     161    7.2        79 119435
##  2 Action       45 Batman v Superma… Batman is …     151    6.5        44 741776
##  3 Adventure    81 The Lost City of… A true-lif…     141    6.6        78  97055
##  4 Biography     7 Hacksaw Ridge     World War …     139    8.1        71 571808
##  5 Horror        5 The Conjuring 2   Ed and Lor…     134    7.3        65 292206
##  6 Comedy        9 La La Land        While navi…     128    8          94 639080
##  7 Crime        61 The Girl on the … A divorcee…     112    6.5        48 195702
##  8 Animation    10 Sing              In a city …     108    7.1        59 184413
##  9 Animation    21 Zootopia          In a city …     108    8          78 529648
## 10 Fantasy      97 The Midnight Man  A girl and…      95    4.8        NA   4599
## # ℹ 2 more variables: Gross_Earning_in_Mil <dbl>, Director <fct>

Silence (from the Drama genre) has the longest runtime at 161 minutes.

movies_df |>
  ggplot(aes(x = Runtime, y = Rating)) +
  geom_point(aes(size = Votes, col = Genre))

Create the above viz with plotly

p1 <- plot_ly(movies_df, x = ~Runtime, y = ~Rating, text = ~Title, type = 'scatter', mode = 'markers', size = ~Votes, color = ~Genre, colors = 'Paired',
        marker = list(opacity = 0.8))
p1 <- p1 |> layout(title = 'Movie Ratings by Runtime and Genre',
         xaxis = list(showgrid = FALSE),
         yaxis = list(showgrid = FALSE))

p1

Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

movies_df |>
  filter(Runtime >= 130 & Runtime <= 160) |>
  group_by(Genre) |>
  relocate(Genre, Votes) |>
  arrange(desc(Votes))

## # A tibble: 16 × 10
## # Groups:   Genre [5]
##    Genre      Votes  Rank Title             Description Runtime Rating Metascore
##    <fct>      <dbl> <dbl> <chr>             <chr>         <dbl>  <dbl>     <dbl>
##  1 Action    829951    47 Captain America:… "Political…     147    7.8        75
##  2 Action    741776    45 Batman v Superma… "Batman is…     151    6.5        44
##  3 Action    671134    17 Rogue One: A Sta… "In a time…     133    7.8        65
##  4 Biography 571808     7 Hacksaw Ridge     "World War…     139    8.1        71
##  5 Adventure 496697    33 Fantastic Beasts… "The adven…     132    7.2        66
##  6 Action    452836    52 X-Men: Apocalypse "In the 19…     144    6.9        52
##  7 Drama     302016    34 Manchester by th… "A depress…     137    7.8        96
##  8 Horror    292206     5 The Conjuring 2   "Ed and Lo…     134    7.3        65
##  9 Action    223710    23 The Magnificent … "Seven gun…     132    6.9        54
## 10 Drama     164453     8 The Handmaiden    "A woman i…     145    8.1        85
## 11 Action    154409    42 13 Hours          "During an…     144    7.3        48
## 12 Drama     114826    99 Fences            "A working…     139    7.2        79
## 13 Drama     106694    68 A Cure for Welln… "An ambiti…     146    6.4        47
## 14 Adventure  97055    81 The Lost City of… "A true-li…     141    6.6        78
## 15 Drama      77370    32 The Wailing       "Soon afte…     156    7.4        81
## 16 Action     60926    71 Free State of Jo… "A disillu…     139    6.9        53
## # ℹ 2 more variables: Gross_Earning_in_Mil <dbl>, Director <fct>

Within the runtime of 130-160 minutes, the Action genre has the movie with the highest number of votes (829,951 for Captain America: Civil War). The 3 movies with the most votes are all from the Action genre.

movies_df |>
  ggplot(aes(x = Runtime, y = Gross_Earning_in_Mil)) +
  geom_point(aes(size = Rating, col = Genre))

Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120?

movies_df |>
  filter(Runtime >= 100 & Runtime <= 120) |>
  group_by(Genre) |>
  summarise(avggross = mean(Gross_Earning_in_Mil)) |>
  arrange(desc(avggross))

## # A tibble: 8 × 2
##   Genre     avggross
##   <fct>        <dbl>
## 1 Animation    216. 
## 2 Adventure    125. 
## 3 Action        89.2
## 4 Drama         48.4
## 5 Horror        46.8
## 6 Comedy        33.9
## 7 Biography     28.7
## 8 Crime         NA

Animation is the genre with the highest average gross earnings given runtime between 100 and 120 minutes.