#Loading the rvest package

library(rvest)
## Warning: package 'rvest' was built under R version 3.6.1
## Loading required package: xml2
library(xml2)

#Specifying the url for desired website to be scraped

url <- 'https://www.imdb.com/search/title/?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)

#Using CSS selectors to scrape the rankings section

rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6

#Using CSS selectors to scrape the title section

title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text

title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Miss Peregrine's Home for Peculiar Children"
## [2] "Suicide Squad"                              
## [3] "London Has Fallen"                          
## [4] "Deadpool"                                   
## [5] "Me Before You"                              
## [6] "Passengers"

#Using CSS selectors to scrape the description section

description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\n    When Jacob discovers clues to a mystery that stretches across time, he finds Miss Peregrine's Home for Peculiar Children. But the danger deepens after he gets to know the residents and learns about their special powers."
## [2] "\n    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                        
## [3] "\n    In London for the Prime Minister's funeral, Mike Banning is caught up in a plot to assassinate all the attending world leaders."                                                                                            
## [4] "\n    A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                      
## [5] "\n    A girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."                                                                                                                          
## [6] "\n    A spacecraft traveling to a distant colony planet and transporting thousands of people has a malfunction in its sleep chambers. As a result, two passengers are awakened 90 years early."
#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data
head(description_data)
## [1] "    When Jacob discovers clues to a mystery that stretches across time, he finds Miss Peregrine's Home for Peculiar Children. But the danger deepens after he gets to know the residents and learns about their special powers."
## [2] "    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                        
## [3] "    In London for the Prime Minister's funeral, Mike Banning is caught up in a plot to assassinate all the attending world leaders."                                                                                            
## [4] "    A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                      
## [5] "    A girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."                                                                                                                          
## [6] "    A spacecraft traveling to a distant colony planet and transporting thousands of people has a malfunction in its sleep chambers. As a result, two passengers are awakened 90 years early."

#Using CSS selectors to scrape the Movie runtime section

runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "127 min" "123 min" "99 min"  "108 min" "106 min" "116 min"
#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 127 123  99 108 106 116

#Using CSS selectors to scrape the Movie genre section

genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nAdventure, Drama, Fantasy            " 
## [2] "\nAction, Adventure, Fantasy            "
## [3] "\nAction, Thriller            "          
## [4] "\nAction, Adventure, Comedy            " 
## [5] "\nDrama, Romance            "            
## [6] "\nDrama, Romance, Sci-Fi            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Adventure Action    Action    Action    Drama     Drama    
## 8 Levels: Action Adventure Animation Biography Comedy Crime ... Horror

#Using CSS selectors to scrape the IMDB rating section

rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "6.7" "6.0" "5.9" "8.0" "7.4" "7.0"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 6.7 6.0 5.9 8.0 7.4 7.0

#Using CSS selectors to scrape the votes section

votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "143,028" "542,331" "127,762" "834,324" "179,695" "316,172"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1] 143028 542331 127762 834324 179695 316172
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Tim Burton"    "David Ayer"    "Babak Najafi"  "Tim Miller"   
## [5] "Thea Sharrock" "Morten Tyldum"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Eva Green"         "Will Smith"        "Gerard Butler"    
## [4] "Ryan Reynolds"     "Emilia Clarke"     "Jennifer Lawrence"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore 
head(metascore_data)
## [1] "57        " "40        " "28        " "65        " "51        "
## [6] "41        "
#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)
## [1] 97
#missing Metascore
for (i in c(38, 61, 71)){

a<-metascore_data[1:(i-1)]

b<-metascore_data[i:length(metascore_data)]

metascore_data<-append(a,list("NA"))

metascore_data<-append(metascore_data,b)

}

#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at length of the metascore data
length(metascore_data)
## [1] 100
#Let's look at summary statistics
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    21.0    47.0    60.0    59.6    72.0    99.0       3
#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)
## [1] "$87.24M"  "$325.10M" "$62.68M"  "$363.07M" "$56.25M"  "$100.01M"
#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)
gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)
## [1] 92
#Filling missing entries with NA
for (i in c(38, 47, 63, 65, 69, 71, 87, 94)){

a<-gross_data[1:(i-1)]

b<-gross_data[i:length(gross_data)]

gross_data<-append(a,list("NA"))

gross_data<-append(gross_data,b)

}

#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)
## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion

## Warning: NAs introduced by coercion
#Let's have another look at the length of gross data
length(gross_data)
## [1] 100
summary(gross_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.04   14.27   54.92   91.52  116.15  532.10       8
#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,

Director = directors_data, Actor = actors_data)

#Structure of the data frame

str(movies_df)
## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : Factor w/ 100 levels "10 Cloverfield Lane",..: 52 68 46 23 49 58 17 93 64 12 ...
##  $ Description         : Factor w/ 100 levels "    A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in sile"| __truncated__,..: 93 20 65 25 10 21 72 84 60 47 ...
##  $ Runtime             : num  127 123 99 108 106 116 147 87 108 151 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 2 1 1 1 7 7 1 3 3 1 ...
##  $ Rating              : num  6.7 6 5.9 8 7.4 7 7.8 6.5 7.1 6.5 ...
##  $ Metascore           : num  57 40 28 65 51 41 75 61 59 44 ...
##  $ Votes               : num  143028 542331 127762 834324 179695 ...
##  $ Gross_Earning_in_Mil: num  87.2 325.1 62.7 363 56.2 ...
##  $ Director            : Factor w/ 99 levels "Alex Proyas",..: 93 26 10 94 91 67 6 17 39 99 ...
##  $ Actor               : Factor w/ 90 levels "Aamir Khan","Alexander Skarsgård",..: 27 87 31 70 24 40 16 51 57 7 ...
library('ggplot2')
## Registered S3 methods overwritten by 'ggplot2':
##   method         from 
##   [.quosures     rlang
##   c.quosures     rlang
##   print.quosures rlang
qplot(data = movies_df,Runtime,fill = Genre,bins = 30)

##Question 1: Based on the above data, which movie from which Genre had the longest runtime?
##Answer: Advanture
ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

##Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?
##Answer: Action
ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 8 rows containing missing values (geom_point).

##Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.
##Answer: Action