Install Packages

#install.packages('rvest')
library(tidyverse)
## -- Attaching packages --------------------------------------- tidyverse 1.3.1 --
## v ggplot2 3.3.5     v purrr   0.3.4
## v tibble  3.1.6     v dplyr   1.0.8
## v tidyr   1.2.0     v stringr 1.4.0
## v readr   2.1.2     v forcats 0.5.1
## -- Conflicts ------------------------------------------ tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x dplyr::lag()    masks stats::lag()
library(dplyr)
library(ggplot2)

Load the rVest package

library('rvest')
## Warning: package 'rvest' was built under R version 4.1.3
## 
## Attaching package: 'rvest'
## The following object is masked from 'package:readr':
## 
##     guess_encoding
#Specifying the url for desired website to be scraped
url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

#Reading the HTML code from the website
webpage <- read_html(url)

Using CSS selectors to scrape the rankings section

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)
## [1] "1." "2." "3." "4." "5." "6."
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)
## [1] 1 2 3 4 5 6
length(rank_data)
## [1] 100

Using CSS selectors to scrape the title section

title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)
## [1] "Batman v Superman: Dawn of Justice Ultimate Edition"
## [2] "Fantastic Beasts and Where to Find Them"            
## [3] "Suicide Squad"                                      
## [4] "Deadpool"                                           
## [5] "Batman v Superman: Dawn of Justice"                 
## [6] "Hacksaw Ridge"
length(title_data)
## [1] 100

Using CSS selectors to scrape the description section

description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)
## [1] "\nBatman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and he is framed for murder during an international crisis. The heroes clash and force the neutral Wonder Woman to reemerge."
## [2] "\nThe adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years before Harry Potter reads his book in school."                                                                              
## [3] "\nA secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                                    
## [4] "\nA wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                                  
## [5] "\nFearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."                                                                          
## [6] "\nWorld War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."

Cleaning up those Movie descriptions

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)
## [1] "Batman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and he is framed for murder during an international crisis. The heroes clash and force the neutral Wonder Woman to reemerge."
## [2] "The adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years before Harry Potter reads his book in school."                                                                              
## [3] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."                                                    
## [4] "A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                                                  
## [5] "Fearing that the actions of Superman are left unchecked, Batman takes on the Man of Steel, while the world wrestles with what kind of a hero it really needs."                                                                          
## [6] "World War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."

Using CSS selector to scrape the Movie runtime section

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)
## [1] "182 min" "132 min" "123 min" "108 min" "152 min" "139 min"
#Data-Preprocessing: removing mins and converting it to numerical

runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)
## [1] 182 132 123 108 152 139

Using CSS selector to scrape the Movie Genre Section AND some more data cleaning

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)
## [1] "\nAction, Adventure, Sci-Fi            " 
## [2] "\nAdventure, Family, Fantasy            "
## [3] "\nAction, Adventure, Fantasy            "
## [4] "\nAction, Adventure, Comedy            " 
## [5] "\nAction, Adventure, Sci-Fi            " 
## [6] "\nBiography, Drama, History            "
#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)
## [1] Action    Adventure Action    Action    Action    Biography
## Levels: Action Adventure Animation Biography Comedy Crime Drama Horror

Using CSS selectors to scrape the IMDB rating section

rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)
## [1] "8.1" "7.3" "5.9" "8.0" "6.5" "8.1"
#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)
## [1] 8.1 7.3 5.9 8.0 6.5 8.1

Using CSS selectors to scrape the votes section

votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)
## [1] "13,351"  "448,577" "669,711" "990,091" "683,673" "497,626"

Lets remove those commas

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)
## [1]  13351 448577 669711 990091 683673 497626

Using CSS selectors to scrape the directors section

directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)
## [1] "David Yates"     "David Ayer"      "Tim Miller"      "Zack Snyder"    
## [5] "Mel Gibson"      "Damien Chazelle"

Converting Director data into FACTORS

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

Using CSS selectors to scrape Actors Data

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)
## [1] "Eddie Redmayne"  "Will Smith"      "Ryan Reynolds"   "Ben Affleck"    
## [5] "Andrew Garfield" "Ryan Gosling"

Covert actors data into FACTORS

#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

Using CSS selectors to scrape the metascore section and more data cleaning.

metascore_data_html <- html_nodes(webpage,'.ratings-metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore 
head(metascore_data)
## [1] "\n66        \n        Metascore\n            "
## [2] "\n40        \n        Metascore\n            "
## [3] "\n65        \n        Metascore\n            "
## [4] "\n44        \n        Metascore\n            "
## [5] "\n71        \n        Metascore\n            "
## [6] "\n94        \n        Metascore\n            "
#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)
## [1] 96
ratings_bar_data <- html_nodes(webpage,'.ratings-metascore') %>%
# scrape the ratings bar and convert to text
 html_text2()
head(ratings_bar_data) 
## [1] "66 Metascore" "40 Metascore" "65 Metascore" "44 Metascore" "71 Metascore"
## [6] "94 Metascore"

Lets look at the ratings bar

#metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%
   #str_match("\\d{2}") %>%
 #as.numeric()
#length(metascore_data)
#metascore_data
#summary(metascore_data)

#head(metascore_data)
ratings_bar_data <- html_nodes(webpage,'.ratings-bar') %>%
# scrape the ratings bar and convert to text
 html_text2()
head(ratings_bar_data) # look at the ratings bar
## [1] "8.1\nRate this\n 1 2 3 4 5 6 7 8 9 10 8.1/10 X "              
## [2] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n66 Metascore"
## [3] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
## [4] "8.0\nRate this\n 1 2 3 4 5 6 7 8 9 10 8/10 X \n65 Metascore"  
## [5] "6.5\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.5/10 X \n44 Metascore"
## [6] "8.1\nRate this\n 1 2 3 4 5 6 7 8 9 10 8.1/10 X \n71 Metascore"
metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") %>%
# extract Metascore
 str_match("\\d{2}") %>%
 as.numeric() # convert to number
length(metascore_data)
## [1] 100
metascore_data
##   [1] NA 66 40 65 44 71 94 52 72 81 59 62 78 84 99 81 54 65 64 67 96 48 75 74 88
##  [26] 41 44 78 70 57 79 71 28 25 51 73 72 51 66 81 60 32 66 57 81 47 72 64 68 NA
##  [51] 52 79 77 34 42 51 45 32 69 62 77 58 48 79 47 45 23 76 65 49 NA 74 42 60 76
##  [76] 39 51 36 79 42 18 35 78 67 63 36 77 68 46 55 59 59 NA 60 65 40 61 58 81 26
summary(metascore_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   18.00   47.00   62.00   60.15   74.00   99.00       4

#Gross Varibale

votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') %>%
 html_text2()
head(votes_bar_data)
## [1] "Votes: 13,351"                    "Votes: 448,577 | Gross: $234.04M"
## [3] "Votes: 669,711 | Gross: $325.10M" "Votes: 990,091 | Gross: $363.07M"
## [5] "Votes: 683,673 | Gross: $330.36M" "Votes: 497,626 | Gross: $67.21M"
gross_data <- str_match(votes_bar_data, "\\$.+$") # extract the gross earnings
gross_data <- gsub("M","",gross_data) # clean data: remove 'M' sign
gross_data <- substring(gross_data,2,6) %>% # clean data: remove '$' sign
 as.numeric()
length(gross_data)
## [1] 100
summary(gross_data)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##    0.18   18.71   57.64   99.38  126.60  532.10      11

Combine all the list to form a dataframe

topmovies_df <-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data)

Structure of the data frame

str(topmovies_df)
## 'data.frame':    100 obs. of  9 variables:
##  $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
##  $ Title               : chr  "Batman v Superman: Dawn of Justice Ultimate Edition" "Fantastic Beasts and Where to Find Them" "Suicide Squad" "Deadpool" ...
##  $ Description         : chr  "Batman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and "| __truncated__ "The adventures of writer Newt Scamander in New York's secret community of witches and wizards seventy years bef"| __truncated__ "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man"| __truncated__ ...
##  $ Runtime             : num  182 132 123 108 152 139 128 144 115 107 ...
##  $ Genre               : Factor w/ 8 levels "Action","Adventure",..: 1 2 1 1 1 4 5 1 1 3 ...
##  $ Rating              : num  8.1 7.3 5.9 8 6.5 8.1 8 6.9 7.5 7.6 ...
##  $ Metascore           : num  NA 66 40 65 44 71 94 52 72 81 ...
##  $ Votes               : num  13351 448577 669711 990091 683673 ...
##  $ Gross_Earning_in_Mil: num  NA 234 325 363 330 ...

Analyzing Data, Visualization, Questions.

Plot1 <- qplot(data = topmovies_df,Runtime,fill = Genre,bins = 30)

Plot1

Question 1: Based on the above data, which movie from which Genre had the longest run-time?

topmovies_df %>%
rownames_to_column(var= "Name") %>%
  filter(runtime_data == max(runtime_data))
##   Name Rank                                               Title
## 1    1    1 Batman v Superman: Dawn of Justice Ultimate Edition
##                                                                                                                                                                                                                               Description
## 1 Batman is manipulated by Lex Luthor to fear Superman. Superman´s existence is meanwhile dividing the world and he is framed for murder during an international crisis. The heroes clash and force the neutral Wonder Woman to reemerge.
##   Runtime  Genre Rating Metascore Votes Gross_Earning_in_Mil
## 1     182 Action    8.1        NA 13351                   NA
Plot2 <- ggplot(topmovies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

Plot2

Question 2: Based on the above data, in the Runtime of 130-160 mins, which genre has the highest votes?

topmovies_df %>%
  rownames_to_column(var = "Name") %>%
  filter(Runtime >= 130 & Runtime <= 160) %>%
  filter(Votes == max(Votes))
##   Name Rank                      Title
## 1   23   23 Captain America: Civil War
##                                                                                          Description
## 1 Political involvement in the Avengers' affairs causes a rift between Captain America and Iron Man.
##   Runtime  Genre Rating Metascore  Votes Gross_Earning_in_Mil
## 1     147 Action    7.8        75 747630                  408
Plot3 <- ggplot(topmovies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))

Plot3
## Warning: Removed 11 rows containing missing values (geom_point).

Question 3: Based on the above data, across all genres which genre has the highest average gross earnings in runtime 100 to 120.

topmovies_df %>%
  filter(Runtime >= 100 & Runtime <= 120 & !is.na(Gross_Earning_in_Mil)) %>%
  group_by(Genre) %>%
  summarise(averageGross = mean(Gross_Earning_in_Mil)) %>%
  filter(averageGross == max(averageGross))
## # A tibble: 1 x 2
##   Genre     averageGross
##   <fct>            <dbl>
## 1 Animation         216.

I was able to take the questions to tutoring and breakdown the questions so that I could recognize which functions to use on my own.

#END