Movies

library('rvest')
library(tidyverse)

Webscraping

url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

webpage <- read_html(url)

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

# #Using CSS selectors to scrape the actors section
# actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')
# #Converting the gross actors data to text
# actors_data <- html_text(actors_data_html)
# 
# #Data-Preprocessing: converting actors data into factors
# actors_data<-as.factor(actors_data)

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

for (i in c(1,40,44,65,97)){

a<-metascore_data[0:(i-1)]

b<-metascore_data[i:length(metascore_data)]

metascore_data<-append(a,list("NA"))

metascore_data<-append(metascore_data,b)

}

#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Filling missing entries with NA
for (i in c(3,31,34,39,43,47,64,69,96)){

a<-gross_data[1:(i-1)]

b<-gross_data[i:length(gross_data)]

gross_data<-append(a,list("NA"))

gross_data<-append(gross_data,b)
}

gross_data<-append(list("NA"),gross_data)

#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)

#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, 

Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                                                             Gross_Earning_in_Mil = gross_data,

Director = directors_data)

The Plots

qplot(data = movies_df,Runtime,fill = Genre,bins = 30)

Warning: `qplot()` was deprecated in ggplot2 3.4.0.

ggplot(movies_df,aes(x=Runtime,y=Rating))+
geom_point(aes(size=Votes,col=Genre))

ggplot(movies_df,aes(x=Runtime,y=Gross_Earning_in_Mil))+
geom_point(aes(size=Rating,col=Genre))

Warning: Removed 10 rows containing missing values (`geom_point()`).

Questions

Question 1: Which movie from which Genre had the longest runtime?

maxlen<-max(movies_df$Runtime)

for (i in 1:nrow(movies_df)) {
  if (movies_df$Runtime[i] == maxlen) {
    print(movies_df$Title[i], max.levels = 0)
    print(movies_df$Genre[i], max.levels = 0)
  }
}

[1] "Silence"
[1] Drama

The movie with the longest runtime in the dataset is “Silence” and a drama.

Question 2: In the Runtime of 130-160 mins, which genre has the highest votes?

The way I interpreted this question was which genre has the highest number of total votes.

#Filter for those runtimes
movies130 <- filter(movies_df, Runtime>=130)
movies130 <- group_by(filter(movies130, Runtime<=160), Genre)

# Average gross by genre
Votes <- summarise(movies130, votes=sum(Votes, na.rm=TRUE))

#Find row with max votes and print the genre
maxvotes<-max(Votes$votes)

for (i in 1:nrow(Votes)) {
  if (Votes$votes[i] == maxvotes) {
    print(Votes$Genre[i], max.levels = 0)
  }
}

[1] Action

The genre with the most votes with those runtimes is action.

Question 3: Across all genres which genre has the highest average gross earnings in runtime 100 to 120.

#Filter for those runtimes
movies100 <- filter(movies_df, Runtime>=100)
movies100 <- group_by(filter(movies100, Runtime<=120), Genre)

# Average gross by genre
Avgross <- summarise(movies100, grossav=mean(Gross_Earning_in_Mil, na.rm=TRUE))

#Find row with max gross and print the genre
maxGross<-max(Avgross$grossav)

for (i in 1:nrow(Avgross)) {
  if (Avgross$grossav[i] == maxGross) {
    print(Avgross$Genre[i], max.levels = 0)
  }
}

[1] Animation

The genre with the highest average gross earnings in that subset is animation.