#Loading the rvest package

#install.packages("rtools")
#install.packages("rvest")
library(rvest)
## Warning: package 'rvest' was built under R version 4.2.3

#Specifying the url for desired website to be scraped

url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website

webpage <- read_html(url)

#Using CSS selectors to scrape the rankings section

rank_data_html <- html_nodes(webpage,'.text-primary')

#Coverting the ranking data to text

rank_data <- html_text(rank_data_html)

#Let’s have a look at the rankings

head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."

#Data-Processing: Converting rankings to numerical

rank_data<-as.numeric(rank_data)

#Let’s have another look at the rankings

head(rank_data)
## [1] 1 2 3 4 5 6

#Using CSS selectors to scrape the title section

title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text

title_data <- html_text(title_data_html)

#Let’s have a look at the title

head(title_data)
## [1] "The Magnificent Seven"        "Me Before You"               
## [3] "Rogue One: A Star Wars Story" "Hidden Figures"              
## [5] "Suicide Squad"                "Sing"

#Using CSS selectors to scrape the description section

description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text

description_data <- html_text(description_data_html)

#Let’s have a look at the description data

head(description_data)
## [1] "\nSeven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."                                                          
## [2] "\nA girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."                                                                                                                            
## [3] "\nIn a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                                                              
## [4] "\nThe story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."                                                                              
## [5] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                          
## [6] "\nIn a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."

#Data-Preprocessing: removing ‘’

description_data<-gsub("\n","",description_data)

#Let’s have another look at the description data

head(description_data)
## [1] "Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."                                                          
## [2] "A girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."                                                                                                                            
## [3] "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                                                              
## [4] "The story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."                                                                              
## [5] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                          
## [6] "In a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."

#Using CSS selectors to scrape the Movie runtime section

runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text

runtime_data <- html_text(runtime_data_html)

#Let’s have a look at the runtime

head(runtime_data)
## [1] "132 min" "106 min" "133 min" "127 min" "123 min" "108 min"

#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let’s have another look at the runtime data

head(runtime_data)
## [1] 132 106 133 127 123 108

#Using CSS selectors to scrape the Movie genre section

genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text

genre_data <- html_text(genre_data_html)

#Let’s have a look at the runtime

head(genre_data)
## [1] "\nAction, Adventure, Western            "
## [2] "\nDrama, Romance            "            
## [3] "\nAction, Adventure, Sci-Fi            " 
## [4] "\nBiography, Drama, History            " 
## [5] "\nAction, Adventure, Fantasy            "
## [6] "\nAnimation, Comedy, Family            "

#Data-Preprocessing: removing 

genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces

genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie

genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor

genre_data<-as.factor(genre_data)

#Let’s have another look at the genre data

head(genre_data)
## [1] Action    Drama     Action    Biography Action    Animation
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror

#Using CSS selectors to scrape the IMDB rating section

rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text

rating_data <- html_text(rating_data_html)

#Let’s have a look at the ratings

head(rating_data)
## [1] "6.8" "7.4" "7.8" "7.8" "5.9" "7.1"

#Data-Preprocessing: converting ratings to numerical

rating_data<-as.numeric(rating_data)

#Let’s have another look at the ratings data

head(rating_data)
## [1] 6.8 7.4 7.8 7.8 5.9 7.1

#Using CSS selectors to scrape the votes section

votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text

votes_data <- html_text(votes_data_html)

#Let’s have a look at the votes data

head(votes_data)
## [1] "217,177" "263,326" "652,047" "238,330" "695,550" "176,676"

#Data-Preprocessing: removing commas

votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical

votes_data<-as.numeric(votes_data)

#Let’s have another look at the votes data

head(votes_data)
## [1] 217177 263326 652047 238330 695550 176676

#Using CSS selectors to scrape the directors section

directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text

directors_data <- html_text(directors_data_html)

#Let’s have a look at the directors data

head(directors_data)
## [1] "Antoine Fuqua"  "Thea Sharrock"  "Gareth Edwards" "Theodore Melfi"
## [5] "David Ayer"     "Garth Jennings"

#Data-Preprocessing: converting directors data into factors

directors_data<-as.factor(directors_data)

#Using CSS selectors to scrape the actors section

actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text

actors_data <- html_text(actors_data_html)

#Let’s have a look at the actors data

head(actors_data)
## [1] "Denzel Washington"   "Emilia Clarke"       "Felicity Jones"     
## [4] "Taraji P. Henson"    "Will Smith"          "Matthew McConaughey"

#Using CSS selectors to scrape the metascore section

metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text

metascore_data <- html_text(metascore_data_html)

#Let’s have a look at the metascore

head(metascore_data)
## [1] "54        " "51        " "65        " "74        " "40        "
## [6] "59        "

#Data-Preprocessing: removing extra space in metascore

metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data

length(metascore_data)
## [1] 96
for (i in c(19,52,69,82)){

a<-metascore_data[1:(i-1)]

b<-metascore_data[i:length(metascore_data)]

metascore_data<-append(a,list("NA"))

metascore_data<-append(metascore_data,b)

}

#Data-Preprocessing: converting metascore to numerical

metascore_data<-as.numeric(metascore_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

#Let’s have another look at length of the metascore data

length(metascore_data)
## [1] 100

#Let’s look at summary statistics

summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   21.00   46.00   60.50   59.57   73.25   99.00       4

#Using CSS selectors to scrape the gross revenue section

gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text

gross_data <- html_text(gross_data_html)

#Let’s have a look at the votes data

head(gross_data)
## [1] "$93.43M"  "$56.25M"  "$532.18M" "$169.61M" "$325.10M" "$270.40M"

#Data-Preprocessing: removing ‘$’ and ‘M’ signs

gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)

#Let’s check the length of gross data

length(gross_data)
## [1] 89

#Filling missing entries with NA

for (i in c(19,52,54,55,61,67,69,72,82,83,92)){

a<-gross_data[1:(i-1)]

b<-gross_data[i:length(gross_data)]

gross_data<-append(a,list("NA"))

gross_data<-append(gross_data,b)

}

#Data-Preprocessing: converting gross to numerical

gross_data<-as.numeric(gross_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

#Let’s have another look at the length of gross data

length(gross_data)
## [1] 100
summary(gross_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.01   26.86   61.71  101.28  127.40  532.10      11

#Combining all the lists to form a data frame

movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,

Director = directors_data, Actor = actors_data)

#Structure of the data frame

str(movies_df)
## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "The Magnificent Seven" "Me Before You" "Rogue One: A Star Wars Story" "Hidden Figures" ...
##  $ Description         : chr  "Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town f"| __truncated__ "A girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of." "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death St"| __truncated__ "The story of a team of female African-American mathematicians who served a vital role in NASA during the early "| __truncated__ ...
##  $ Runtime             : num  132 106 133 127 123 108 128 108 139 116 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 1 7 1 4 1 3 5 1 4 7 ...
##  $ Rating              : num  6.8 7.4 7.8 7.8 5.9 7.1 8 8 8.1 7.9 ...
##  $ Metascore           : num  54 51 65 74 40 59 94 65 71 81 ...
##  $ Votes               : num  217177 263326 652047 238330 695550 ...
##  $ Gross_Earning_in_Mil: num  93.4 56.2 532.1 169.6 325.1 ...
##  $ Director            : Factor w/ 99 levels "Aisling Walsh",..: 11 91 34 92 26 36 20 94 61 30 ...
##  $ Actor               : chr  "Denzel Washington" "Emilia Clarke" "Felicity Jones" "Taraji P. Henson" ...

#Analyzing scraped from the web

library('ggplot2')

qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
## Warning: `qplot()` was deprecated in ggplot2 3.4.0.

#Question 1 The movies with longest runtime are Silence from the drama category and American Honey from the Adventure category.

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

#Question 2

It appears that the action genre gets the most votes in the Runtime of 130-160mins

#Question 3

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 11 rows containing missing values (`geom_point()`).

#Question 3 In the runtimes from 100-120 minutes, on average animation films have the highest average gross earnings.