Webscraping final
Amani Panduwawala
2023-04-05
library('rvest')## Warning: package 'rvest' was built under R version 4.2.3library(tidyverse)## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr 1.1.0 ✔ readr 2.1.4
## ✔ forcats 1.0.0 ✔ stringr 1.5.0
## ✔ ggplot2 3.4.1 ✔ tibble 3.1.8
## ✔ lubridate 1.9.2 ✔ tidyr 1.3.0
## ✔ purrr 1.0.1
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ readr::guess_encoding() masks rvest::guess_encoding()
## ✖ dplyr::lag() masks stats::lag()
## ℹ Use the ]8;;http://conflicted.r-lib.org/conflicted package]8;; to force all conflicts to become errorslibrary(plotly)##
## Attaching package: 'plotly'
##
## The following object is masked from 'package:ggplot2':
##
## last_plot
##
## The following object is masked from 'package:stats':
##
## filter
##
## The following object is masked from 'package:graphics':
##
## layoutlibrary(ggplot2)URL used to web scrape
url <- "https://www.imdb.com/search/title/?count=100&release_date=2016,2016&title_type=feature"webpage <- read_html(url)
rank_data_htlm <- html_nodes(webpage,'.text-primary')
rank_data <- html_text(rank_data_htlm)
head(rank_data)## [1] "1." "2." "3." "4." "5." "6."rank_data <- as.numeric(rank_data)
head(rank_data)## [1] 1 2 3 4 5 6title_data_html <- html_nodes(webpage,'.lister-item-header a')
title_data <- html_text(title_data_html)
head(title_data)## [1] "The Magnificent Seven" "Me Before You"
## [3] "Rogue One: A Star Wars Story" "Hidden Figures"
## [5] "Suicide Squad" "Sing"#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
head(description_data)## [1] "\nSeven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."
## [2] "\nA girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."
## [3] "\nIn a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."
## [4] "\nThe story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."
## [5] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [6] "\nIn a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)
#Let's have another look at the description data
head(description_data)## [1] "Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."
## [2] "A girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."
## [3] "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."
## [4] "The story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."
## [5] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [6] "In a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')
runtime_data <-html_text(runtime_data_html)
head(runtime_data)## [1] "132 min" "106 min" "133 min" "127 min" "123 min" "108 min"runtime_data<-gsub("min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
head(runtime_data)## [1] 132 106 133 127 123 108genre_data_html <- html_nodes(webpage,'.genre')
genre_data <- html_text(genre_data_html)
head(genre_data)## [1] "\nAction, Adventure, Western "
## [2] "\nDrama, Romance "
## [3] "\nAction, Adventure, Sci-Fi "
## [4] "\nBiography, Drama, History "
## [5] "\nAction, Adventure, Fantasy "
## [6] "\nAnimation, Comedy, Family "#removing \n
genre_data<-gsub("\n","",genre_data)
#removing excess space
genre_data<-gsub(" ","",genre_data)
#only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)
genre_data<-as.factor(genre_data)
head(genre_data)## [1] Action Drama Action Biography Action Animation
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horrorwebpage <- read_html(url)
rating_data_html <- html_nodes(webpage,'.ratingValue strong')
rating_data <- html_text(rating_data_html)
head(rating_data)## character(0)rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')
rating_data <- html_text(rating_data_html)
head(rating_data)## [1] "6.8" "7.4" "7.8" "7.8" "5.9" "7.1"votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
votes_data <- html_text(votes_data_html)
head(votes_data)## [1] "217,195" "263,348" "652,091" "238,338" "695,587" "176,692"description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')
#Converting the description data to text
description_data <- html_text(description_data_html)
#Let's have a look at the description data
head(description_data)## [1] "\nSeven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."
## [2] "\nA girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."
## [3] "\nIn a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."
## [4] "\nThe story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."
## [5] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [6] "\nIn a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')
votes_data <- html_text(votes_data_html)
head(votes_data)## [1] "217,195" "263,348" "652,091" "238,338" "695,587" "176,692"votes_data<-gsub(",","",votes_data)
#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)
#look at the votes data
head(votes_data)## [1] 217195 263348 652091 238338 695587 176692directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')
directors_data <- html_text(directors_data_html)
head(directors_data)## [1] "Antoine Fuqua" "Thea Sharrock" "Gareth Edwards" "Theodore Melfi"
## [5] "David Ayer" "Garth Jennings"#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)
#Let's have a look at the actors data
head(actors_data)## [1] "Denzel Washington" "Emilia Clarke" "Felicity Jones"
## [4] "Taraji P. Henson" "Will Smith" "Matthew McConaughey"#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')
#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)
#Let's have a look at the metascore
head(metascore_data)## [1] "54 " "51 " "65 " "74 " "40 "
## [6] "59 "#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)
#Lets check the length of metascore data
length(metascore_data)## [1] 96for (i in c(39,73,80,89)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion#Let's have another look at length of the metascore data
length(metascore_data)## [1] 100#Let's look at summary statistics
summary(metascore_data)## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 21.00 46.00 60.50 59.57 73.25 99.00 4#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)
#Let's have a look at the votes data
head(gross_data)## [1] "$93.43M" "$56.25M" "$532.18M" "$169.61M" "$325.10M" "$270.40M"#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
#Let's check the length of gross data
length(gross_data)## [1] 89for (i in c(33,58,74,89)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
# Converting metascore to numerical
metascore_data<-as.numeric(metascore_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercionlength(metascore_data)## [1] 104summary(metascore_data)## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 21.00 46.00 60.50 59.57 73.25 99.00 8gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')
gross_data <- html_text(gross_data_html)
head(gross_data)## [1] "$93.43M" "$56.25M" "$532.18M" "$169.61M" "$325.10M" "$270.40M"#Data-Preprocessing: removing ‘$’ and ‘M’ signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)
length(gross_data)## [1] 89for (i in c(33,58,74,89)){
a<-metascore_data[1:(i-1)]
b<-metascore_data[i:length(metascore_data)]
metascore_data<-append(a,list("NA"))
metascore_data<-append(metascore_data,b)
}
# Converting metascore to numerical
metascore_data<-as.numeric(metascore_data)## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coercion
## Warning: NAs introduced by coerciongross_data<-as.numeric(gross_data)
length(gross_data)## [1] 89summary(gross_data)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 0.01 26.86 61.71 101.28 127.40 532.10#Combining all the lists to form a data frame
movies_df<-data(Rank = rank_data, Title = title_data,
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data, Gross_Earning_in_Mil = gross_data,
Director = directors_data, Actor = actors_data)## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'rank_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'title_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'description_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'runtime_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'genre_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'rating_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'metascore_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'votes_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'gross_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'directors_data' not found## Warning in data(Rank = rank_data, Title = title_data, Description =
## description_data, : data set 'actors_data' not found#Structure of the data frame
str(movies_df)## chr [1:11] "rank_data" "title_data" "description_data" "runtime_data" ...names(movies_df) <- make.names(names(movies_df))min_length <- 100
movies_df <- data.frame(Rank = rank_data[1:min_length],
Title = title_data[1:min_length],
Description = description_data[1:min_length],
Runtime = runtime_data[1:min_length],
Genre = genre_data[1:min_length],
Rating = rating_data[1:min_length],
Metascore = metascore_data[1:min_length],
Votes = votes_data[1:min_length],
Gross_Earning_in_Mil = gross_data[1:min_length],
Director = directors_data[1:min_length],
Actor = actors_data[1:min_length])qplot(data = movies_df,Runtime,fill = Genre,bins = 30)## Warning: `qplot()` was deprecated in ggplot2 3.4.0.
Question 1: Based on the above data, which movie from which Genre had
the longest runtime?
q1 <-movies_df %>% select(Title, Rank, Title, Runtime, Genre) %>%
filter(Runtime == max(Runtime))
q1## Title Rank Runtime Genre
## 1 American Honey 64 163 AdventureIt can be seen that American Honey has the longest run time.
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))
Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
g2 <- movies_df %>% select(Title,Rank, Runtime, Votes, Genre) %>% filter(between(Runtime, 130, 160))
g2plot <- g2 %>% ggplot(aes(x=Genre, y= Votes)) +
geom_bar(stat='identity') +
xlab("Movie Genre") +
ylab("Amount of Votes") +
ggtitle("Movie genre by Amount of Votes")
g2plot
q2 <- movies_df %>% filter(Votes == max(Votes))
q2## Rank Title
## 1 8 Deadpool
## Description
## 1 \nA wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks.
## Runtime Genre Rating Metascore Votes Gross_Earning_in_Mil Director
## 1 108 Action 8.0 65 1057784 363 Tim Miller
## Actor
## 1 Ryan Reynoldsq3 <- movies_df %>% select(Runtime,Genre, Gross_Earning_in_Mil) %>% drop_na() %>%
filter(between(Runtime, 100, 120)) %>%
group_by(Genre) %>%
summarize(avgGross = mean(Gross_Earning_in_Mil))
q3## # A tibble: 8 × 2
## Genre avgGross
## <fct> <dbl>
## 1 Action 129.
## 2 Adventure 89.2
## 3 Animation 216.
## 4 Biography 66.6
## 5 Comedy 94.3
## 6 Crime 54.1
## 7 Drama 51.3
## 8 Horror 72.2Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
The Action genre has the highest average gross earnings.