#Loading the 'rvest' package
library('rvest')Webscraping - IMDB
library(tidyverse)── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr 1.1.3 ✔ readr 2.1.4
✔ forcats 1.0.0 ✔ stringr 1.5.0
✔ ggplot2 3.4.4 ✔ tibble 3.2.1
✔ lubridate 1.9.3 ✔ tidyr 1.3.0
✔ purrr 1.0.2
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ readr::guess_encoding() masks rvest::guess_encoding()
✖ dplyr::lag() masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errorslibrary(plotly)
Attaching package: 'plotly'
The following object is masked from 'package:ggplot2':
last_plot
The following object is masked from 'package:stats':
filter
The following object is masked from 'package:graphics':
layoutlibrary(dplyr)#Specifying the url for desired website to be scraped
url <-'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'#Reading the HTML code from the website
webpage <- read_html(url)#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')#Converting the ranking data to text
rank_data <- html_text(rank_data_html)#Let's have a look at the rankings
head(rank_data)[1] "1." "2." "3." "4." "5." "6."#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)#Let's have another look at the rankings
head(rank_data)[1] 1 2 3 4 5 6#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')#Converting the title data to text
title_data <- html_text(title_data_html)#Let's have a look at the title
head(title_data)[1] "Terrifier" "Suicide Squad" "Silence" "Hush"
[5] "The Conjuring 2" "Split" #Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')#Converting the description data to text
description_data <- html_text(description_data_html)#Let's have a look at the description data
head(description_data)[1] "\nOn Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."
[2] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
[3] "\nIn the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."
[4] "\nA deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."
[5] "\nEd and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."
[6] "\nThree girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th." #Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)#Let's have another look at the description data
head(description_data)[1] "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."
[2] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
[3] "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."
[4] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."
[5] "Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."
[6] "Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th." #Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)
#Let's have a look at the runtime
head(runtime_data)[1] "85 min" "123 min" "161 min" "82 min" "134 min" "117 min"#Data-Preprocessing: removing mins and converting it to numerical
runtime_data <-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)#Let's have another look at the runtime data
head(runtime_data)[1] 85 123 161 82 134 117#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')#Converting the genre data to text
genre_data <- html_text(genre_data_html)#Let's have a look at the runtime
head(genre_data)[1] "\nHorror, Thriller "
[2] "\nAction, Adventure, Fantasy "
[3] "\nDrama, History "
[4] "\nHorror, Thriller "
[5] "\nHorror, Mystery, Thriller "
[6] "\nHorror, Thriller " #Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)#Convering each genre from text to factor
genre_data<-as.factor(genre_data)#Let's have another look at the genre data
head(genre_data)[1] Horror Action Drama Horror Horror Horror
9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')#Converting the ratings data to text
rating_data <- html_text(rating_data_html)#Let's have a look at the ratings
head(rating_data)[1] "5.6" "5.9" "7.2" "6.6" "7.3" "7.3"#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)#Let's have another look at the ratings data
head(rating_data)[1] 5.6 5.9 7.2 6.6 7.3 7.3#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')#Converting the votes data to text
votes_data <- html_text(votes_data_html)#Let's have a look at the votes data
head(votes_data)[1] "47,668" "710,204" "119,456" "149,248" "292,263" "532,862"#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)#Let's have another look at the votes data
head(votes_data)[1] 47668 710204 119456 149248 292263 532862#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')#Converting the directors data to text
directors_data <- html_text(directors_data_html)#Let's have a look at the directors data
head(directors_data)[1] "Damien Leone" "David Ayer" "Martin Scorsese"
[4] "Mike Flanagan" "James Wan" "M. Night Shyamalan"#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)#Let's have a look at the actors data
head(actors_data)[1] "Jenna Kanell" "Will Smith" "Andrew Garfield"
[4] "John Gallagher Jr." "Vera Farmiga" "James McAvoy" #Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)#Scrape the ratings bar and convert to text
ratings_bar_data <- html_nodes(webpage,'.ratings-bar') |>
html_text2()#Look at the ratings bar
head(ratings_bar_data)[1] "5.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.6/10 X "
[2] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
[3] "7.2\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.2/10 X \n79 Metascore"
[4] "6.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.6/10 X \n67 Metascore"
[5] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n65 Metascore"
[6] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n63 Metascore"#Extract metascore and convert to numerical
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") |>
str_match("\\d{2}") |>
as.numeric()
length(metascore_data)[1] 100metascore_data [1] NA 40 79 67 65 63 71 85 94 59 81 81 55 65 73 70 65 88 57 65 78 81 54 67 60
[26] 44 51 41 65 74 71 81 66 96 68 58 76 47 66 NA 82 48 82 NA 44 51 75 42 32 25
[51] 66 52 51 99 72 58 77 57 81 37 48 44 72 32 NA 45 44 47 66 46 53 81 69 49 77
[76] 62 58 35 33 68 78 42 42 36 46 34 61 60 60 21 38 66 26 59 55 62 NA 68 79 74summary(metascore_data) Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
21.00 47.00 62.00 60.36 72.50 99.00 5 #Scrape the votess bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') |>
html_text2()
#Look at the votes bar data and extract gross earnings
head(votes_bar_data)[1] "Votes: 47,668" "Votes: 710,204 | Gross: $325.10M"
[3] "Votes: 119,456 | Gross: $7.10M" "Votes: 149,248"
[5] "Votes: 292,263 | Gross: $102.47M" "Votes: 532,862 | Gross: $138.29M"gross_data <-str_match(votes_bar_data, "\\$.+$")
#Clean data: remove 'M' sign
gross_data <-gsub("M","",gross_data)
#Clean data: remove '$'
gross_data <- substring(gross_data,2,6) %>%
as.numeric()
length(gross_data)[1] 100movies_df <-data.frame (Rank=rank_data, Title=title_data, Description=description_data, Runtime=runtime_data, Genre=genre_data, Rating=rating_data, Director=directors_data, Metascore=metascore_data, Votes=votes_data, Gross_Earning_in_Mil=gross_data)
str(movies_df)'data.frame': 100 obs. of 10 variables:
$ Rank : num 1 2 3 4 5 6 7 8 9 10 ...
$ Title : chr "Terrifier" "Suicide Squad" "Silence" "Hush" ...
$ Description : chr "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown." "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is"| __truncated__ "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence "| __truncated__ ...
$ Runtime : num 85 123 161 82 134 117 139 145 128 108 ...
$ Genre : Factor w/ 9 levels "Action","Adventure",..: 9 1 7 9 9 9 4 7 5 3 ...
$ Rating : num 5.6 5.9 7.2 6.6 7.3 7.3 8.1 8.1 8 7.1 ...
$ Director : Factor w/ 97 levels "Alessandro Carloni",..: 19 22 61 65 44 59 63 71 18 34 ...
$ Metascore : num NA 40 79 67 65 63 71 85 94 59 ...
$ Votes : num 47668 710204 119456 149248 292263 ...
$ Gross_Earning_in_Mil: num NA 325.1 7.1 NA 102.4 ...Question 1: Based on the data below, which movie from which Genre had the longest runtime?
Answer 1: The movie called Silence from the drama genre has the longest runtime at 161 minutes.
movies_df |>
select(Rank, Title, Runtime, Genre) |>
filter(Runtime == max(Runtime)) Rank Title Runtime Genre
1 3 Silence 161 Dramaggplot(movies_df, aes(x=Genre, y=Runtime, color=Genre)) +
geom_point() +
labs(x="Genre", y="Run Times", title = "Genre Run Times", caption = "Source: IMDB") +
scale_color_brewer(name = "", palette = 'Set1') +
theme_minimal()
Question 2: Based on the data below, in the Runtime of 130-160 mins, which genre has the highest votes?
Answer 2: In the runtime of 130-160 mins, Action has the highest votes of all the genres.
votes <- movies_df |>
filter(Runtime >= 130 | Runtime <= 160) |>
group_by(Genre)|>
mutate(genre_votes = sum(Votes)) |>
distinct(Genre, genre_votes) |>
arrange(desc(genre_votes))
votes# A tibble: 9 × 2
# Groups: Genre [9]
Genre genre_votes
<fct> <dbl>
1 Action 10105014
2 Drama 3584028
3 Animation 2063181
4 Biography 1835082
5 Horror 1614552
6 Adventure 1412831
7 Comedy 1321838
8 Crime 952763
9 Fantasy 4601runtime130_to_160 <- movies_df |>
filter(Runtime %in% (130:160) )votes130_to_160 <- runtime130_to_160 |>
group_by(Genre) |>
dplyr::summarize(votes1=sum(votes_data, na.rm = TRUE))
votes130_to_160# A tibble: 5 × 2
Genre votes1
<fct> <dbl>
1 Action 22893890
2 Adventure 22893890
3 Biography 22893890
4 Drama 22893890
5 Horror 22893890options(scipen=999)ggplot(runtime130_to_160, aes(x=Genre, y=Votes, fill=Genre)) +
geom_bar(stat = "identity") +
coord_flip() +
ggtitle("Genre Votes for Runtimes 130 to 160") +
labs(x = "Genre",
y = "Votes",
caption = "Source: IMDB") +
theme_classic()
Question 3: Based on the data below, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
Answer 3: Animation has the highest average gross earnings in the runtime 100 to 200 minutes.
runtime100_to_120 <- movies_df |>
filter(Runtime %in% (100:120))gross_average <- runtime100_to_120 |>
group_by(Genre) |>
dplyr::summarize(Avg = mean(Gross_Earning_in_Mil, na.rm=TRUE))
gross_average# A tibble: 8 × 2
Genre Avg
<fct> <dbl>
1 Action 89.2
2 Adventure 125.
3 Animation 216.
4 Biography 28.7
5 Comedy 33.9
6 Crime 51.2
7 Drama 48.4
8 Horror 46.8ggplot(gross_average, aes(x=Genre, y=Avg, fill=Genre)) +
geom_bar(stat = "identity") +
theme_classic() +
ggtitle("Average Gross Earnings by Genre for Runtimes 100 to 120") +
theme(plot.title = element_text(hjust = 0.5)) +
labs(x = "Genre",
y = "Earnings ($Millions)",
caption = "Source: IMDB")