Scrapping rank from the data

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)

## [1] "1." "2." "3." "4." "5." "6."

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)

## [1] 1 2 3 4 5 6

Scrapping title from the data

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)

## [1] "Suicide Squad"              "Captain America: Civil War"
## [3] "Deadpool"                   "The Neon Demon"            
## [5] "The Magnificent Seven"      "Split"

Scrapping description from the data

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)

## [1] "\n    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."        
## [2] "\n    Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."                                                                                         
## [3] "\n    A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                      
## [4] "\n    An aspiring model, Jesse, is new to Los Angeles. However, her beauty and youth, which generate intense fascination and jealousy within the fashion industry, may prove themselves sinister."
## [5] "\n    Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."                        
## [6] "\n    Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)

## [1] "    A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."        
## [2] "    Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man."                                                                                         
## [3] "    A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                      
## [4] "    An aspiring model, Jesse, is new to Los Angeles. However, her beauty and youth, which generate intense fascination and jealousy within the fashion industry, may prove themselves sinister."
## [5] "    Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."                        
## [6] "    Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."

Scrapping runtime from the data

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)

## [1] "123 min" "147 min" "108 min" "117 min" "132 min" "117 min"

#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)

## [1] 123 147 108 117 132 117

Scrapping genre from the data

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)

## [1] "\nAction, Adventure, Fantasy            "
## [2] "\nAction, Adventure, Sci-Fi            " 
## [3] "\nAction, Adventure, Comedy            " 
## [4] "\nHorror, Thriller            "          
## [5] "\nAction, Adventure, Western            "
## [6] "\nHorror, Thriller            "

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)

## [1] Action Action Action Horror Action Horror
## 9 Levels: Action Adventure Animation Biography Comedy Crime ... Thriller

Scrapping rating from the data

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)

## [1] "6.0" "7.8" "8.0" "6.2" "6.9" "7.3"

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)

## [1] 6.0 7.8 8.0 6.2 6.9 7.3

Scrapping votes from the data

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)

## [1] "536,663" "590,251" "827,314" "73,355"  "169,536" "364,104"

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)

## [1] 536663 590251 827314  73355 169536 364104

Scrapping director from the data

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)

## [1] "David Ayer"           "Anthony Russo"        "Tim Miller"          
## [4] "Nicolas Winding Refn" "Antoine Fuqua"        "M. Night Shyamalan"

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

Scrapping actor from the data

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)

## [1] "Will Smith"        "Chris Evans"       "Ryan Reynolds"    
## [4] "Elle Fanning"      "Denzel Washington" "James McAvoy"

#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

Scrapping metascore from the data

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore 
head(metascore_data)

## [1] "40        " "75        " "65        " "51        " "54        "
## [6] "62        "

#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)

## [1] 97

### Adding missing metascore

for (i in c(38, 61, 71)){

a<-metascore_data[1:(i-1)]

b<-metascore_data[i:length(metascore_data)]

metascore_data<-append(a,list("NA"))

metascore_data<-append(metascore_data,b)

}

#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)

#Let's have another look at length of the metascore data

length(metascore_data)

## [1] 100

#Let's look at summary statistics
summary(metascore_data)

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   21.00   46.00   60.00   58.57   72.00   99.00       3

Scrapping Gross earing from the data

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)

## [1] "$325.10M" "$408.08M" "$363.07M" "$1.33M"   "$93.43M"  "$138.29M"

#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)

## [1] 92

### Missing Gross data
#Filling missing entries with NA
for (i in c(38, 47, 63, 65, 69, 71, 87, 94)){

a<-gross_data[1:(i-1)]

b<-gross_data[i:length(gross_data)]

gross_data<-append(a,list("NA"))

gross_data<-append(gross_data,b)
}
#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)

#Let's have another look at the length of gross data
length(gross_data)

## [1] 100

#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,

Director = directors_data, Actor = actors_data)

#Structure of the data frame

str(movies_df)

## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : Factor w/ 100 levels "10 Cloverfield Lane",..: 66 16 21 89 88 64 98 23 29 85 ...
##  $ Description         : Factor w/ 100 levels "    A chronicle of the childhood, adolescence and burgeoning adulthood of a young, African-American, gay man gr"| __truncated__,..: 18 67 24 33 70 85 61 98 100 27 ...
##  $ Runtime             : num  123 147 108 117 132 117 144 115 139 106 ...
##  $ Genre               : Factor w/ 9 levels "Action","Adventure",..: 1 1 1 8 1 8 1 1 4 2 ...
##  $ Rating              : num  6 7.8 8 6.2 6.9 7.3 7 7.5 8.1 7.4 ...
##  $ Metascore           : num  40 75 65 51 54 62 52 72 71 77 ...
##  $ Votes               : num  536663 590251 827314 73355 169536 ...
##  $ Gross_Earning_in_Mil: num  325.1 408 363 1.33 93.43 ...
##  $ Director            : Factor w/ 99 levels "Alex Proyas",..: 27 7 94 71 8 58 14 87 63 51 ...
##  $ Actor               : Factor w/ 91 levels "Aamir Khan","Adam Sandler",..: 87 19 73 26 22 40 40 10 5 67 ...

Question 1: Based on the above data, which movie from which Genre had the longest runtime?

Answer : Adventure

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

Answer : Action

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

Answer : Action