library(rvest) 
library(tidyverse) 
## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.1 ──
## ✔ ggplot2 3.3.6     ✔ purrr   0.3.4
## ✔ tibble  3.1.7     ✔ dplyr   1.0.9
## ✔ tidyr   1.2.0     ✔ stringr 1.4.0
## ✔ readr   2.1.2     ✔ forcats 0.5.1
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter()         masks stats::filter()
## ✖ readr::guess_encoding() masks rvest::guess_encoding()
## ✖ dplyr::lag()            masks stats::lag()
library(plotly)
## 
## Attaching package: 'plotly'
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## The following object is masked from 'package:stats':
## 
##     filter
## The following object is masked from 'package:graphics':
## 
##     layout
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
webpage <- read_html(url)
#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6
## [1] 1 2 3 4 5 6

length(rank_data)
## [1] 100
## [1] 100

title_data_html <- html_nodes(webpage,'.lister-item-header a')

title_data <- html_text(title_data_html)
#Let's have a look at the title
head(title_data)
## [1] "Doctor Strange"                         
## [2] "Rogue One: A Star Wars Story"           
## [3] "Suicide Squad"                          
## [4] "Fantastic Beasts and Where to Find Them"
## [5] "La La Land"                             
## [6] "Moana"
length(title_data)
## [1] 100
#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\nWhile on a journey of physical and spiritual healing, a brilliant neurosurgeon is drawn into the world of the mystic arts."                                                         
## [2] "\nIn a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                    
## [3] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] "\nThe adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years before Harry Potter reads his book in school."                          
## [5] "\nWhile navigating their careers in Los Angeles, a pianist and an actress fall in love while attempting to reconcile their aspirations for the future."                               
## [6] "\nIn Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches Moana's island, she answers the Ocean's call to seek out the Demigod to set things right."
description_data<-gsub("\n","",description_data)
#Let's have another look at the description data
head(description_data)
## [1] "While on a journey of physical and spiritual healing, a brilliant neurosurgeon is drawn into the world of the mystic arts."                                                         
## [2] "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                    
## [3] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
## [4] "The adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years before Harry Potter reads his book in school."                          
## [5] "While navigating their careers in Los Angeles, a pianist and an actress fall in love while attempting to reconcile their aspirations for the future."                               
## [6] "In Ancient Polynesia, when a terrible curse incurred by the Demigod Maui reaches Moana's island, she answers the Ocean's call to seek out the Demigod to set things right."
#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime') #Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)
#Let's have a look at the runtime
head(runtime_data)
## [1] "115 min" "133 min" "123 min" "132 min" "128 min" "107 min"
## [1] "132 min" "133 min" "97 min"  "108 min" "115 min" "128 min"
#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)
#Let's have another look at the runtime data
head(runtime_data)
## [1] 115 133 123 132 128 107
## [1] 132 133  97 108 115 128
length(runtime_data)
## [1] 100
## [1] 100
#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre') 

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nAction, Adventure, Fantasy            "  
## [2] "\nAction, Adventure, Sci-Fi            "   
## [3] "\nAction, Adventure, Fantasy            "  
## [4] "\nAdventure, Family, Fantasy            "  
## [5] "\nComedy, Drama, Music            "        
## [6] "\nAnimation, Adventure, Comedy            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data) 

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data) 

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data) 

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Action    Action    Action    Adventure Comedy    Animation
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
length(genre_data)
## [1] 100
#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "7.5" "7.8" "5.9" "7.2" "8.0" "7.6"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 7.5 7.8 5.9 7.2 8.0 7.6
length(rating_data)
## [1] 100
#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "716,649" "613,137" "675,128" "466,933" "569,578" "324,382"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data) #Data-Preprocessing: converting votes to numerical

votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1] 716649 613137 675128 466933 569578 324382
length(votes_data)
## [1] 100
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)') 

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Scott Derrickson" "Gareth Edwards"   "David Ayer"       "David Yates"     
## [5] "Damien Chazelle"  "Ron Clements"
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

length(directors_data)
## [1] 100
#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a') 

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Benedict Cumberbatch" "Felicity Jones"       "Will Smith"          
## [4] "Eddie Redmayne"       "Ryan Gosling"         "Auli'i Cravalho"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

length(actors_data)
## [1] 100
ratings_bar_data <- html_nodes(webpage,'.ratings_bar') %>% 
  # scrape the ratings bar and convert to text 
  html_text2()
head(ratings_bar_data) 
## character(0)
# look at the ratings bar

metascore_data <- str_match(ratings_bar_data , "\\d{2} Metascore") %>% 
# extract Metascore

  str_match("\\d{2}") %>%
  
as.numeric() 
# convert to number

length(metascore_data)
## [1] 0
## [1] 100

metascore_data
## numeric(0)
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
## 
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
  html_text2()

head(votes_bar_data) 
## [1] "Votes: 716,649 | Gross: $232.64M" "Votes: 613,137 | Gross: $532.18M"
## [3] "Votes: 675,128 | Gross: $325.10M" "Votes: 466,933 | Gross: $234.04M"
## [5] "Votes: 569,578 | Gross: $151.10M" "Votes: 324,382 | Gross: $248.76M"
# look at the votes bar data
## [1] "Votes: 466,278 | Gross: $234.04M"
## [3] "Votes: 90,295 | Gross: $35.82M"
## [5] "Votes: 714,666 | Gross: $232.64M"
"Votes: 612,013 | Gross: $532.18M"
## [1] "Votes: 612,013 | Gross: $532.18M"
"Votes: 1,003,867 | Gross: $363.07M"
## [1] "Votes: 1,003,867 | Gross: $363.07M"
"Votes: 291,324 | Gross: $86.26M"
## [1] "Votes: 291,324 | Gross: $86.26M"
gross_data <- str_match(votes_bar_data, "\\$.+$") 
gross_data <- gsub("M","",gross_data) 
gross_data <- substring(gross_data,2,6) %>% 
  as.numeric()
length(gross_data)
## [1] 100
# extract the gross
# clean data: remove
# clean data: remove
## [1] 100
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
  html_text2()
head(votes_bar_data) # look at the votes bar data
## [1] "Votes: 716,649 | Gross: $232.64M" "Votes: 613,137 | Gross: $532.18M"
## [3] "Votes: 675,128 | Gross: $325.10M" "Votes: 466,933 | Gross: $234.04M"
## [5] "Votes: 569,578 | Gross: $151.10M" "Votes: 324,382 | Gross: $248.76M"
## [1] "Votes: 466,278 | Gross: $234.04M"
## [3] "Votes: 90,295 | Gross: $35.82M"
## [5] "Votes: 714,666 | Gross: $232.64M"
"Votes: 612,013 | Gross: $532.18M"
## [1] "Votes: 612,013 | Gross: $532.18M"
"Votes: 1,003,867 | Gross: $363.07M"
## [1] "Votes: 1,003,867 | Gross: $363.07M"
"Votes: 291,324 | Gross: $86.26M"
## [1] "Votes: 291,324 | Gross: $86.26M"
gross_data <- str_match(votes_bar_data, "\\$.+$") 
gross_data <- gsub("M","",gross_data) 
gross_data <- substring(gross_data,2,6) %>% 
  as.numeric()
length(gross_data)
## [1] 100
# extract the gross
# clean data: remove
# clean data: remove
## [1] 100
movies_df<-data.frame(Rank = rank_data, Title = title_data, Description = description_data, Genre= genre_data,
                      Runtime = runtime_data, 
                      Rating = rating_data,
                      Votes = votes_data,
Gross_Earning_in_Mil = gross_data)
# I removed director and actor data from the dataframe since they currently only have 99 observations
                      #Director = directors_data, Actor = actors_data
#Structure of the data frame
str(movies_df)
## 'data.frame':    100 obs. of  8 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Doctor Strange" "Rogue One: A Star Wars Story" "Suicide Squad" "Fantastic Beasts and Where to Find Them" ...
##  $ Description         : chr  "While on a journey of physical and spiritual healing, a brilliant neurosurgeon is drawn into the world of the mystic arts." "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death St"| __truncated__ "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "The adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years bef"| __truncated__ ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 1 1 1 2 5 3 1 4 3 1 ...
##  $ Runtime             : num  115 133 123 132 128 107 108 139 108 147 ...
##  $ Rating              : num  7.5 7.8 5.9 7.2 8 7.6 8 8.1 7.1 7.8 ...
##  $ Votes               : num  716649 613137 675128 466933 569578 ...
##  $ Gross_Earning_in_Mil: num  233 532 325 234 151 ...

#Question 1

p1 <- movies_df %>%
  ggplot(aes(x=runtime_data, y = genre_data)) +
  geom_point(position="identity", alpha=0.5, binwidth = 5, text = paste("Movie Title#:", title_data, color = "black"))+
  scale_fill_discrete(name = "Genre") +
  labs(title = "Top 100 Movies of 2016 Runtime by Genre")+
  theme_classic()
## Warning: Ignoring unknown parameters: binwidth, text
ggplotly( p1)

##The Genre with the highest votes is the ACTION GENRE with the Movie: Batman V Superman: Dawn of Justice Ultimate edition

#Question 2

p2 <- movies_df %>%
  ggplot(aes(x=Runtime,y=Rating))+
  geom_point(aes(size=Votes,col=genre_data, text = paste("Movie Title#:", title_data)), alpha =
0.7) +
  labs(title = "Top 100 Movies of 2016 Runtime by Ratings")
## Warning: Ignoring unknown aesthetics: text
ggplotly(p2)

#The Genre with the highest amount of votes is the ACTION GENRE with the movie: Captain America:Cevil War

#Question 3

p3 <- movies_df %>%
  ggplot(aes(x=Runtime,y=Gross_Earning_in_Mil))+
  geom_line(aes(size = Rating,col = Genre), alpha = 0.5) +
  labs(title = "Top 100 Movies of 2016 Runtime by Gross Earnings in Millions") +
  scale_y_continuous("Gross Earnings in Millions", limits =c(-10, 600))
ggplotly(p3)

#ANIMATION is the Genre with the highest Gross earnings between runtime 100 and 120