IMDB Webscraping Assignment

Author

Adi Ve

image credit: mrpeloquin on Google Maps, Event Cinemas Queens St., Auckland, NZ

Load Libraries and Data Source

library(tidyverse)
library(stringr)
library(rvest)
library(xml2)
library(plotly)

url <- 'https://www.imdb.com/search/title/?count=100&release_date=2016,2016&title_type=feature'
webpage <- read_html(url)

Scrape Scrape

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
[1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
[1] 1 2 3 4 5 6
#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
[1] "Terrifier"       "Suicide Squad"   "Silence"         "Hush"           
[5] "The Conjuring 2" "Split"          
#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)
[1] "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                       
[2] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
[3] "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."  
[4] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."                         
[5] "Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                     
[6] "Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."                      
#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
[1] "85 min"  "123 min" "161 min" "82 min"  "134 min" "117 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
[1]  85 123 161  82 134 117
#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
[1] "\nHorror, Thriller            "          
[2] "\nAction, Adventure, Fantasy            "
[3] "\nDrama, History            "            
[4] "\nHorror, Thriller            "          
[5] "\nHorror, Mystery, Thriller            " 
[6] "\nHorror, Thriller            "          
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
[1] Horror Action Drama  Horror Horror Horror
9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror
#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
[1] "5.6" "5.9" "7.2" "6.6" "7.3" "7.3"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
[1] 5.6 5.9 7.2 6.6 7.3 7.3
#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
[1] "47,773"  "710,255" "119,475" "149,289" "292,356" "532,952"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
[1]  47773 710255 119475 149289 292356 532952
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
[1] "Damien Leone"       "David Ayer"         "Martin Scorsese"   
[4] "Mike Flanagan"      "James Wan"          "M. Night Shyamalan"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
[1] "Jenna Kanell"       "Will Smith"         "Andrew Garfield"   
[4] "John Gallagher Jr." "Vera Farmiga"       "James McAvoy"

Here, below, use the code from prof. Saidi’s notes, and not the tutorial

ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>%
# scrape the ratings bar and convert to text
 html_text2()
head(ratings_bar_data) # look at the ratings bar
[1] "5.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.6/10 X "              
[2] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
[3] "7.2\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.2/10 X \n79 Metascore"
[4] "6.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.6/10 X \n67 Metascore"
[5] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n65 Metascore"
[6] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n63 Metascore"
4
[1] 4
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%
# extract Metascore
 str_match("\\d{2}") %>%
 as.numeric() # convert to number
length(metascore_data)
[1] 100
metascore_data
  [1] NA 40 79 67 65 63 71 85 94 59 81 81 55 65 73 70 65 88 57 65 78 81 54 67 60
 [26] 44 51 41 65 74 71 81 66 96 68 58 76 47 66 NA 82 48 82 NA 44 51 75 42 32 25
 [51] 66 52 51 99 72 58 77 57 81 37 48 44 72 32 NA 45 44 47 66 46 53 81 69 49 77
 [76] 62 58 35 33 68 78 42 42 36 46 34 61 60 60 21 38 66 26 59 55 62 NA 68 79 74
summary(metascore_data)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
  21.00   47.00   62.00   60.36   72.50   99.00       5 
# scrape the votess bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
 html_text2()
head(votes_bar_data) # look at the votes bar data
[1] "Votes: 47,773"                    "Votes: 710,255 | Gross: $325.10M"
[3] "Votes: 119,475 | Gross: $7.10M"   "Votes: 149,289"                  
[5] "Votes: 292,356 | Gross: $102.47M" "Votes: 532,952 | Gross: $138.29M"
gross_data <- str_match(votes_bar_data, "\\$.+$") # extract the gross earnings
gross_data <- gsub("M","",gross_data) # clean data: remove 'M' sign
gross_data <- substring(gross_data,2,6) %>% # clean data: remove '$' sign
 as.numeric()
length(gross_data)
[1] 100

Merge into Dataframe

Note that I omit actors, as others in our class have, due to the improper list length and the fact that we do not require it for our visualizations.

#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,

Director = directors_data)

Visualization time!!!

I really don’t like this histogram because it just looks like a mess.

qplot(data=movies_df, Runtime, fill=Genre, bins=30) +
  scale_fill_brewer(palette = "Spectral") +
  theme_bw() +
  labs(title = "Histogram of Runtimes by Genre", y = "Count") +
  theme(plot.title = element_text(hjust = 0.5))
Warning: `qplot()` was deprecated in ggplot2 3.4.0.

For the scatterplots I decided to use plotly because then you could also hover

RR <- ggplot(movies_df,aes(x=Runtime,y=Rating, name=Title)) + 
  geom_point(aes(size=Votes,col=Genre)) +
  scale_color_brewer(palette = "Spectral") +
  theme_bw() +
  labs(title = "Scatterplot of Rating vs. Runtime") +
  theme(plot.title = element_text(hjust = 0.5))

ggplotly(RR)

I don’t like how the point size plays out in this below graph, it doesn’t really mean anything to me visually and just ends up cluttering the plotting space

GR <- ggplot(movies_df, aes(x=Runtime, y=Gross_Earning_in_Mil)) +
  geom_point(aes(size=Rating, col=Genre)) +
  scale_color_brewer(palette = "Spectral") +
  theme_bw() +
  labs(title = "Scatterplot of Gross Earning vs. Runtime",
       y = "Gross Earning (in $ millions)") +
  theme(plot.title = element_text(hjust = 0.5))

ggplotly(GR)

Answer Questions Using Logical Indexing!!!

# Question 1: Which movie from which genre had the longest runtime?
print("Question 1")
[1] "Question 1"
print("The movie with the longest runtime was")
[1] "The movie with the longest runtime was"
movies_df$Title[movies_df$Runtime == max(movies_df$Runtime)]
[1] "Silence"
print("which was in the genre, out of the following genres")
[1] "which was in the genre, out of the following genres"
movies_df$Genre[movies_df$Runtime == max(movies_df$Runtime)]
[1] Drama
9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror
print("with a runtime in minutes of")
[1] "with a runtime in minutes of"
movies_df$Runtime[movies_df$Runtime == max(movies_df$Runtime)]
[1] 161
print("----------")
[1] "----------"
# Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
# To answer this question we can use summarize
genre_votes <- movies_df |>
  select(Genre, Votes, Runtime) |>
  filter((Runtime>=130)&(Runtime<=160)) |>
  group_by(Genre) |>
  summarise(votes = mean(Votes))

print("Question 2")
[1] "Question 2"
print("The genre with the highest mean votes in the specified runtimes was the following, out of the below genres")
[1] "The genre with the highest mean votes in the specified runtimes was the following, out of the below genres"
genre_votes$Genre[genre_votes$votes == max(genre_votes$votes)]
[1] Biography
9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror
print("with a mean votes of")
[1] "with a mean votes of"
max(genre_votes$votes)
[1] 571922
print("----------")
[1] "----------"
# Question 3: Which genre had the highest gross earnings in runtimes between 100 and 120 minutes?
# To answer this question we can use summarize
genre_earnings <- movies_df |>
  select(Genre, Gross_Earning_in_Mil, Runtime) |>
  drop_na() |>
  filter((Runtime>=100)&(Runtime<=120)) |>
  group_by(Genre) |>
  summarise(earnings = mean(Gross_Earning_in_Mil))

print("Question 3")
[1] "Question 3"
print("The genre with the highest mean gross earnings in the specified runtimes was the following, out of the below genres")
[1] "The genre with the highest mean gross earnings in the specified runtimes was the following, out of the below genres"
genre_earnings$Genre[genre_earnings$earnings == max(genre_earnings$earnings)]
[1] Animation
9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror
print("with a mean gross earnings in millions of")
[1] "with a mean gross earnings in millions of"
max(genre_earnings$earnings)
[1] 216.33

References