#The Movies_DF datafrome includes 100 observations and 11 variables.

#Scraping a webpage using R

library(rvest)
library(tidyverse)
## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.2 ──
## ✔ ggplot2 3.4.1     ✔ purrr   1.0.1
## ✔ tibble  3.1.8     ✔ dplyr   1.1.0
## ✔ tidyr   1.3.0     ✔ stringr 1.5.0
## ✔ readr   2.1.3     ✔ forcats 1.0.0
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter()         masks stats::filter()
## ✖ readr::guess_encoding() masks rvest::guess_encoding()
## ✖ dplyr::lag()            masks stats::lag()
library(highcharter)
## Registered S3 method overwritten by 'quantmod':
##   method            from
##   as.zoo.data.frame zoo
library(plotly)
## 
## Attaching package: 'plotly'
## 
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## 
## The following object is masked from 'package:stats':
## 
##     filter
## 
## The following object is masked from 'package:graphics':
## 
##     layout
#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'
#Reading the HTML code from the website
webpage <- read_html(url)
#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6
length(rank_data)
## [1] 100
#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "The Magnificent Seven"        "Me Before You"               
## [3] "Rogue One: A Star Wars Story" "Hidden Figures"              
## [5] "Suicide Squad"                "Sing"
length(title_data)
## [1] 100
#STEP 7
#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\nSeven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."                                                          
## [2] "\nA girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."                                                                                                                            
## [3] "\nIn a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                                                              
## [4] "\nThe story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."                                                                              
## [5] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                          
## [6] "\nIn a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."
length(description_data)
## [1] 100
#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)
## [1] "Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town from the private army of a destructive industrialist."                                                          
## [2] "A girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of."                                                                                                                            
## [3] "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death Star, the Empire's ultimate weapon of destruction."                                                              
## [4] "The story of a team of female African-American mathematicians who served a vital role in NASA during the early years of the U.S. space program."                                                                              
## [5] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                          
## [6] "In a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists find that their lives will never be the same."
length(description_data)
## [1] 100
#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "132 min" "106 min" "133 min" "127 min" "123 min" "108 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 132 106 133 127 123 108
length(runtime_data)
## [1] 100
#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nAction, Adventure, Western            "
## [2] "\nDrama, Romance            "            
## [3] "\nAction, Adventure, Sci-Fi            " 
## [4] "\nBiography, Drama, History            " 
## [5] "\nAction, Adventure, Fantasy            "
## [6] "\nAnimation, Comedy, Family            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Action    Drama     Action    Biography Action    Animation
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror
length(genre_data)
## [1] 100
#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "6.8" "7.4" "7.8" "7.8" "5.9" "7.1"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 6.8 7.4 7.8 7.8 5.9 7.1
length(rating_data)
## [1] 100
#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "217,151" "263,304" "652,024" "238,315" "695,524" "176,666"
#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1] 217151 263304 652024 238315 695524 176666
length(votes_data)
## [1] 100
#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "Antoine Fuqua"  "Thea Sharrock"  "Gareth Edwards" "Theodore Melfi"
## [5] "David Ayer"     "Garth Jennings"
length(directors_data)
## [1] 100
#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Denzel Washington"   "Emilia Clarke"       "Felicity Jones"     
## [4] "Taraji P. Henson"    "Will Smith"          "Matthew McConaughey"
#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
length(actors_data)
## [1] 100

Find metascore data with missing values and replace with NAs (this is an automated method)

ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>%
# scrape the ratings bar and convert to text
 html_text2()
head(ratings_bar_data) 
## [1] "6.8\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.8/10 X \n54 Metascore"
## [2] "7.4\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.4/10 X \n51 Metascore"
## [3] "7.8\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.8/10 X \n65 Metascore"
## [4] "7.8\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.8/10 X \n74 Metascore"
## [5] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [6] "7.1\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.1/10 X \n59 Metascore"
# look at the ratings bar
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%
# extract Metascore
 str_match("\\d{2}") %>%
 as.numeric() 
# convert to number
length(metascore_data)
## [1] 100
metascore_data
##   [1] 54 51 65 74 40 59 94 65 71 81 81 78 84 79 62 66 70 56 NA 68 67 25 73 52 96
##  [26] 44 64 55 99 76 88 44 75 36 41 47 51 72 65 57 69 48 66 32 81 72 74 51 65 66
##  [51] 77 NA 71 42 81 33 58 65 48 57 67 62 79 80 32 42 46 21 NA 79 52 45 48 42 77
##  [76] 77 34 73 33 46 60 NA 78 61 76 66 40 58 23 44 59 22 60 58 35 39 60 34 81 49
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   21.00   46.00   60.50   59.57   73.25   99.00       4

Find the missing gross earnings (automated) Earnings are part of the votes bar in the

html, scrape the votes bar and extract earnings with a regular expression to get the NAs in context.

# scrape the votes bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
 html_text2()
head(votes_bar_data) # look at the votes bar data
## [1] "Votes: 217,151 | Gross: $93.43M"  "Votes: 263,304 | Gross: $56.25M" 
## [3] "Votes: 652,024 | Gross: $532.18M" "Votes: 238,315 | Gross: $169.61M"
## [5] "Votes: 695,524 | Gross: $325.10M" "Votes: 176,666 | Gross: $270.40M"
gross_data <- str_match(votes_bar_data, "\\$.+$") # extract the gross earnings
gross_data <- gsub("M","",gross_data) # clean data: remove 'M' sign
gross_data <- substring(gross_data,2,6) %>% # clean data: remove '$' sign
 as.numeric()
length(gross_data)
## [1] 100
#Step Final: Frame all in one
#Combining all the lists to form a data frame
movies_df <-data.frame (Rank = rank_data, Title = title_data, 
Description = description_data, Runtime = runtime_data,
Genre = genre_data, Rating = rating_data,
Metascore = metascore_data, Votes = votes_data,                                     Gross_Earning_in_Mil = gross_data, Director = directors_data, Actor = actors_data)

#Structure of the data frame
str(movies_df)
## 'data.frame':    100 obs. of  11 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "The Magnificent Seven" "Me Before You" "Rogue One: A Star Wars Story" "Hidden Figures" ...
##  $ Description         : chr  "Seven gunmen from a variety of backgrounds are brought together by a vengeful young widow to protect her town f"| __truncated__ "A girl in a small town forms an unlikely bond with a recently-paralyzed man she's taking care of." "In a time of conflict, a group of unlikely heroes band together on a mission to steal the plans to the Death St"| __truncated__ "The story of a team of female African-American mathematicians who served a vital role in NASA during the early "| __truncated__ ...
##  $ Runtime             : num  132 106 133 127 123 108 128 108 139 116 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 1 7 1 4 1 3 5 1 4 7 ...
##  $ Rating              : num  6.8 7.4 7.8 7.8 5.9 7.1 8 8 8.1 7.9 ...
##  $ Metascore           : num  54 51 65 74 40 59 94 65 71 81 ...
##  $ Votes               : num  217151 263304 652024 238315 695524 ...
##  $ Gross_Earning_in_Mil: num  93.4 56.2 532.1 169.6 325.1 ...
##  $ Director            : Factor w/ 99 levels "Aisling Walsh",..: 11 91 34 92 26 36 20 94 61 30 ...
##  $ Actor               : Factor w/ 92 levels "Adam Sandler",..: 19 25 30 85 91 59 74 75 4 3 ...
library(plotly)
#Analyzing scraped data from the web - Tutorial

qplot(data = movies_df,Runtime,fill = Genre,bins = 30)
## Warning: `qplot()` was deprecated in ggplot2 3.4.0.

#Tutorial 
#Question 1: Based on the above data, which movie from which Genre had the longest runtime?

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

#Tutorial 
#Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))
## Warning: Removed 11 rows containing missing values (`geom_point()`).

#My Plots and Answers to Three (3) Questions based on the Tutorial

#QUESTION 1: Based on the above data, which movie from which Genre had the longest runtime?

ggplot(data=movies_df, aes(x= Genre, y= Runtime, color = Genre)) +
ggtitle("Runtime by Movie Genre") + 
geom_point() + geom_count(size=0.4) +
#theme(plot.title = element_text(angle=15, family = "Georgia", face = "bold")) 
theme(axis.text.x = element_text(angle=20, family = "Georgia")) 

  #theme(axis.text.x = element_text(angle=25, family = "Georgia", face = "italics"))
longest_RuntimeMovie <- movies_df %>%
  filter(Runtime == max(runtime_data))
longest_RuntimeMovie
##   Rank          Title
## 1   64 American Honey
##                                                                                                                                                                                                          Description
## 1 A teenage girl with nothing to lose joins a traveling magazine sales crew, and gets caught up in a whirlwind of hard partying, law bending and young love as she criss-crosses the Midwest with a band of misfits.
##   Runtime     Genre Rating Metascore Votes Gross_Earning_in_Mil      Director
## 1     163 Adventure      7        80 44372                 0.66 Andrea Arnold
##        Actor
## 1 Sasha Lane

#QUESTION 1 Answer - The movie “American Honey” has the longest runtime (163 minutes). The genre is Adventure.

#QUESTION 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

runtime130_160 <- movies_df %>% 
select(Genre, Runtime, Votes) %>% 
filter(between(Runtime, 130, 160)) 
runtime130_160_plot1 <- runtime130_160 %>% 
ggplot(aes(fill = Genre, x=Genre, y= Votes)) +
geom_bar(stat='identity')+
theme(axis.text.x = element_text(angle=15)) +
xlab("Movie Genres") +  
ylab("Votes") +
ggtitle("2016 Movies (130-160 min) with Highest Votes, by Genre") +
coord_trans() +
theme(plot.title = element_text(family = "Georgia", face = "bold")) 
runtime130_160_plot1

#QUESTION 2 Answer - The genre with the highest votes is Action. The only two categories not captured in this data frame are comedy and crime. The data frame for this question included only 17 observations. Vote totals = Action: 2.973 million; Biography: 581,274; Drama: 775,062; Horror: 278,844; Animation: 87,230.

library(dplyr)

#QUESTION 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

runtime100_120 <- movies_df %>% filter(Runtime %in% (100:120))
highest_gross_average <- runtime100_120 %>%
  group_by(Genre) %>%
  dplyr::summarize(Mean = mean(Gross_Earning_in_Mil, na.rm=TRUE))
highest_gross_average
## # A tibble: 8 × 2
##   Genre      Mean
##   <fct>     <dbl>
## 1 Action     92.4
## 2 Adventure 149. 
## 3 Animation 216. 
## 4 Biography  35.1
## 5 Comedy     45.9
## 6 Crime      51.1
## 7 Drama      61.2
## 8 Horror     69.8

#QUESTION 3 Answer: the genre with the highest gross average earnings (runtime 100-120 minutes) is Animation. The average (mean) is 216.33000.

Q3 <- ggplot(highest_gross_average, aes(fill=Genre,x=Genre, y=Mean)) + 
  geom_bar(stat = "identity")+ 
   geom_point(size = 1) +
    ggtitle("Highest Gross Avg. Earnings ($)Runtime 100 - 120 Min")+
    coord_flip() +
  theme(axis.text.x = element_text(angle = 0))
theme(plot.title = element_text(family = "NewCenturySchoolbook", face = "bold"))
## List of 1
##  $ plot.title:List of 11
##   ..$ family       : chr "NewCenturySchoolbook"
##   ..$ face         : chr "bold"
##   ..$ colour       : NULL
##   ..$ size         : NULL
##   ..$ hjust        : NULL
##   ..$ vjust        : NULL
##   ..$ angle        : NULL
##   ..$ lineheight   : NULL
##   ..$ margin       : NULL
##   ..$ debug        : NULL
##   ..$ inherit.blank: logi FALSE
##   ..- attr(*, "class")= chr [1:2] "element_text" "element"
##  - attr(*, "class")= chr [1:2] "theme" "gg"
##  - attr(*, "complete")= logi FALSE
##  - attr(*, "validate")= logi TRUE
ggplotly(Q3)