library(stringr)
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.0     ✔ purrr     1.0.1
## ✔ forcats   1.0.0     ✔ readr     2.1.4
## ✔ ggplot2   3.4.1     ✔ tibble    3.1.8
## ✔ lubridate 1.9.2     ✔ tidyr     1.3.0
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ℹ Use the ]8;;http://conflicted.r-lib.org/conflicted package]8;; to force all conflicts to become errors
#Loading the rvest package
library('rvest')
## Warning: package 'rvest' was built under R version 4.2.3
## 
## Attaching package: 'rvest'
## The following object is masked from 'package:readr':
## 
##     guess_encoding
#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)

Use simple R code to get all the rankings:

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."

Once you have the data, make sure that it looks in the desired format. I am preprocessing my data to convert it to numerical format.

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6

Scrape all the titles using the following code

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "The Magnificent Seven"        "Me Before You"               
## [3] "Rogue One: A Star Wars Story" "Hidden Figures"              
## [5] "Suicide Squad"                "Sing"

Description, Runtime, Genre, Rating, Metascore, Votes, Gross_Earning_in_Mil , Director and Actor data.

Using CSS selectors to scrape the description section

description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

Converting the description data to text

description_data <- html_text(description_data_html)

Let’s have a look at the description data

head(description_data)
## [1] "\nSeven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."                                                          
## [2] "\nA girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."                                                                                                                            
## [3] "\nIn a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                                                              
## [4] "\nThe story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."                                                                              
## [5] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                          
## [6] "\nIn a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."

Data-Preprocessing: removing ‘’

description_data<-gsub("\n","",description_data)

Let’s have another look at the description data

head(description_data)
## [1] "Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."                                                          
## [2] "A girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."                                                                                                                            
## [3] "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                                                              
## [4] "The story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."                                                                              
## [5] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                          
## [6] "In a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."

Using CSS selectors to scrape the Movie runtime section

runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

Converting the runtime data to text

runtime_data <- html_text(runtime_data_html)

Let’s have a look at the runtime

head(runtime_data)
## [1] "132 min" "106 min" "133 min" "127 min" "123 min" "108 min"

Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

Let’s have another look at the runtime data

head(runtime_data)
## [1] 132 106 133 127 123 108

Using CSS selectors to scrape the Movie genre section

genre_data_html <- html_nodes(webpage,'.genre')

Converting the genre data to text

genre_data <- html_text(genre_data_html)

Let’s have a look at the runtime

head(genre_data)
## [1] "\nAction, Adventure, Western            "
## [2] "\nDrama, Romance            "            
## [3] "\nAction, Adventure, Sci-Fi            " 
## [4] "\nBiography, Drama, History            " 
## [5] "\nAction, Adventure, Fantasy            "
## [6] "\nAnimation, Comedy, Family            "

Data-Preprocessing: removing 

genre_data<-gsub("\n","",genre_data)

Data-Preprocessing: removing excess spaces

genre_data<-gsub(" ","",genre_data)

Taking only the first genre of each movie

genre_data<-gsub(",.*","",genre_data)

Convering each genre from text to factor

genre_data<-as.factor(genre_data)

Let’s have another look at the genre data

head(genre_data)
## [1] Action    Drama     Action    Biography Action    Animation
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror

Using CSS selectors to scrape the IMDB rating section

rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

Converting the ratings data to text

rating_data <- html_text(rating_data_html)

Let’s have a look at the ratings

head(rating_data)
## [1] "6.8" "7.4" "7.8" "7.8" "5.9" "7.1"

Data-Preprocessing: converting ratings to numerical

rating_data<-as.numeric(rating_data)

Let’s have another look at the ratings data

head(rating_data)
## [1] 6.8 7.4 7.8 7.8 5.9 7.1

Using CSS selectors to scrape the votes section

votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "217,177" "263,326" "652,047" "238,330" "695,550" "176,676"

Data-Preprocessing: removing commas

votes_data<-gsub(",","",votes_data)

Data-Preprocessing: converting votes to numerical

votes_data<-as.numeric(votes_data)

Let’s have another look at the votes data

head(votes_data)
## [1] 217177 263326 652047 238330 695550 176676

Using CSS selectors to scrape the directors section

directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Antoine Fuqua"  "Thea Sharrock"  "Gareth Edwards" "Theodore Melfi"
## [5] "David Ayer"     "Garth Jennings"

Data-Preprocessing: converting directors data into factors

directors_data<-as.factor(directors_data)

Using CSS selectors to scrape the actors section

actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Denzel Washington"   "Emilia Clarke"       "Felicity Jones"     
## [4] "Taraji P. Henson"    "Will Smith"          "Matthew McConaughey"

Data-Preprocessing: converting actors data into factors

actors_data<-as.factor(actors_data)

Using CSS selectors to scrape the metascore section

metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore data
head(metascore_data)
## [1] "54        " "51        " "65        " "74        " "40        "
## [6] "59        "

Data-Preprocessing: removing extra space in metascore

metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)
## [1] 96

Fixing the meta ratings

ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>%
# scrape the ratings bar and convert to text
 html_text2()
head(ratings_bar_data) # look at the ratings bar
## [1] "6.8\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.8/10 X \n54 Metascore"
## [2] "7.4\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.4/10 X \n51 Metascore"
## [3] "7.8\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.8/10 X \n65 Metascore"
## [4] "7.8\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.8/10 X \n74 Metascore"
## [5] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [6] "7.1\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.1/10 X \n59 Metascore"
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%
# extract Metascore
 str_match("\\d{2}") %>%
 as.numeric() # convert to number
length(metascore_data)
## [1] 100
metascore_data
##   [1] 54 51 65 74 40 59 94 65 71 81 81 78 84 79 62 66 70 56 NA 68 67 25 73 52 96
##  [26] 44 64 55 99 76 88 44 75 36 41 47 51 72 65 57 69 48 66 32 81 72 74 51 65 66
##  [51] 77 NA 71 42 81 33 58 65 48 57 67 62 79 80 32 42 46 21 NA 79 52 45 48 42 77
##  [76] 77 34 73 33 46 60 NA 78 61 76 66 40 58 23 44 59 22 60 58 35 39 60 34 81 49
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   21.00   46.00   60.50   59.57   73.25   99.00       4

Using CSS selectors to scrape the gross revenue section

gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)
## [1] "$93.43M"  "$56.25M"  "$532.18M" "$169.61M" "$325.10M" "$270.40M"

Data-Preprocessing: removing ‘$’ and ‘M’ signs

gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)
## [1] 89

Find the missing gross earnings (automated) Earnings are part of the votes bar in the html, scrape the votes bar and extract earnings with a regular expression to get the NAs in context.

votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
 html_text2()
head(votes_bar_data) # look at the votes bar data
## [1] "Votes: 217,177 | Gross: $93.43M"  "Votes: 263,326 | Gross: $56.25M" 
## [3] "Votes: 652,047 | Gross: $532.18M" "Votes: 238,330 | Gross: $169.61M"
## [5] "Votes: 695,550 | Gross: $325.10M" "Votes: 176,676 | Gross: $270.40M"
gross_data <- str_match(votes_bar_data, "\\$.+$") # extract the gross earnings
gross_data <- gsub("M","",gross_data) # clean data: remove 'M' sign
gross_data <- substring(gross_data,2,6) %>% # clean data: remove '$' sign
 as.numeric()
length(gross_data)
## [1] 100

Analyzing scraped data from the web

library(tidyverse)
library(dplyr)
library(ggplot2)
library(plotly)
## 
## Attaching package: 'plotly'
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## The following object is masked from 'package:stats':
## 
##     filter
## The following object is masked from 'package:graphics':
## 
##     layout

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

qplot(data = movies_df,Runtime,fill = Genre,bins = 30) %>%
ggplotly()
## Warning: `qplot()` was deprecated in ggplot2 3.4.0.

Creating a more detailed graph

graph1 <- movies_df %>%
  ggplot() +
  geom_bar(aes(x=Genre, y=Runtime, fill=Title),
      position = "dodge2", stat = "identity",  alpha = 1)
graph1 <- ggplotly(graph1)
graph1

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

Filter data for movies with run time between 130 to 160 minutes

movies_130_to_160_runtime <- movies_df %>%
  filter(between(Runtime, 130, 160))

graph2 <- ggplot(movies_130_to_160_runtime,aes(x=Genre,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
ggplotly(graph2)

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 11 rows containing missing values (`geom_point()`).

Filter data for movies with run time between 100 to 120 minutes

movies_100_to_120_runtime <- movies_df %>%
  filter(between(Runtime, 100, 120)) 

graph3 <- ggplot(movies_100_to_120_runtime,aes(x=Genre,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
ggplotly(graph3)

ANSWERS