WebScraping week 9

library(tidyverse)

── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.1.3     ✔ readr     2.1.4
✔ forcats   1.0.0     ✔ stringr   1.5.0
✔ ggplot2   3.4.3     ✔ tibble    3.2.1
✔ lubridate 1.9.2     ✔ tidyr     1.3.0
✔ purrr     1.0.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

library(rvest)

Attaching package: 'rvest'

The following object is masked from 'package:readr':

    guess_encoding

url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature' #defineing the url

webpage <- read_html(url) #read url

Rank

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage, '.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)
#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)
#Let's have another look at the rankings
head(rank_data)

[1] 1 2 3 4 5 6

length(rank_data)

[1] 100

Movie Title

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)

[1] "Terrifier"       "Suicide Squad"   "Silence"         "Hush"           
[5] "The Conjuring 2" "Split"          

length(title_data)

[1] 100

Description

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Data-Preprocessing: removing '\n' data cleaning
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)

[1] "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown."                                                                       
[2] "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse."
[3] "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is rumored to have committed apostasy, and to propagate Catholicism."  
[4] "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence when a masked killer appears at her window."                         
[5] "Ed and Lorraine Warren travel to North London to help a single mother raising four children alone in a house plagued by a supernatural spirit."                                     
[6] "Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th."                      

length(description_data)

[1] 100

Movie Run Times

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have a look at the runtime
head(runtime_data)

[1]  85 123 161  82 134 117

length(runtime_data)

[1] 100

Genre

genre_data_html <- html_nodes(webpage,'.genre') #Using CSS selectors to scrape the Movie genre section

genre_data <- html_text(genre_data_html) #Converting the genre data to text
head(genre_data)

[1] "\nHorror, Thriller            "          
[2] "\nAction, Adventure, Fantasy            "
[3] "\nDrama, History            "            
[4] "\nHorror, Thriller            "          
[5] "\nHorror, Mystery, Thriller            " 
[6] "\nHorror, Thriller            "          

length(genre_data)

[1] 100

genre_data<-gsub("\n","",genre_data) #removing \n

genre_data<-gsub(" ","",genre_data) #removing excess spaces

genre_data<-gsub(",.*","",genre_data) #taking only the first genre of each movie

genre_data<-as.factor(genre_data) #Convering each genre from text to factor

head(genre_data)

[1] Horror Action Drama  Horror Horror Horror
9 Levels: Action Adventure Animation Biography Comedy Crime Drama ... Horror

length(genre_data)

[1] 100

IMDB Ratings

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)

[1] "5.6" "5.9" "7.2" "6.6" "7.3" "7.3"

length(rating_data)

[1] 100

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)

[1] 5.6 5.9 7.2 6.6 7.3 7.3

length(rating_data)

[1] 100

Number of votes

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)

[1] "47,604"  "710,171" "119,435" "149,202" "292,206" "532,800"

length(votes_data)

[1] 100

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)

[1]  47604 710171 119435 149202 292206 532800

length(votes_data)

[1] 100

Director

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)

[1] "Damien Leone"       "David Ayer"         "Martin Scorsese"   
[4] "Mike Flanagan"      "James Wan"          "M. Night Shyamalan"

length(directors_data)

[1] 100

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

Actors ! 99??

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)

[1] "Jenna Kanell"       "Will Smith"         "Andrew Garfield"   
[4] "John Gallagher Jr." "Vera Farmiga"       "James McAvoy"      

length(actors_data)

[1] 99

#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)

Meta Score

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore 
head(metascore_data)

[1] "40        " "79        " "67        " "65        " "63        "
[6] "71        "

#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)

[1] 95

MetaScore

ratings_bar_data <- html_nodes(webpage,'.ratings-bar') |>
# scrape the ratings bar and convert to text
 html_text2()
head(ratings_bar_data) # look at the ratings bar

[1] "5.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.6/10 X "              
[2] "5.9\nRate this\n 1 2 3 4 5 6 7 8 9 10 5.9/10 X \n40 Metascore"
[3] "7.2\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.2/10 X \n79 Metascore"
[4] "6.6\nRate this\n 1 2 3 4 5 6 7 8 9 10 6.6/10 X \n67 Metascore"
[5] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n65 Metascore"
[6] "7.3\nRate this\n 1 2 3 4 5 6 7 8 9 10 7.3/10 X \n63 Metascore"

metascore_data <- str_match(ratings_bar_data, "\\d{2} Metascore") |>

  # extract Metascore
 str_match("\\d{2}") |>
 as.numeric() # convert to number

metascore_data

  [1] NA 40 79 67 65 63 71 85 94 59 81 81 55 65 73 70 65 88 57 65 78 81 54 67 60
 [26] 44 51 41 65 74 71 81 66 96 68 58 76 47 66 NA 82 48 82 NA 44 51 75 42 32 25
 [51] 66 52 51 99 72 58 77 57 81 37 48 44 72 32 NA 45 44 47 66 46 53 81 69 49 77
 [76] 62 58 35 33 68 78 42 42 36 46 34 61 60 60 21 38 66 26 59 55 62 NA 68 79 74

summary(metascore_data)

   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
  21.00   47.00   62.00   60.36   72.50   99.00       5 

length(metascore_data)

[1] 100

Gross Earning

# scrape the votess bar and convert to text
votes_bar_data <- html_nodes(webpage,'.sort-num_votes-visible') |>
 html_text2()
head(votes_bar_data) # look at the votes bar data

[1] "Votes: 47,604"                    "Votes: 710,171 | Gross: $325.10M"
[3] "Votes: 119,435 | Gross: $7.10M"   "Votes: 149,202"                  
[5] "Votes: 292,206 | Gross: $102.47M" "Votes: 532,800 | Gross: $138.29M"

gross_data <- str_match(votes_bar_data, "\\$.+$") 

# extract the gross earnings
gross_data <- gsub("M","",gross_data) # clean data: remove 'M' sign
gross_data <- substring(gross_data,2,6) |> # clean data: remove '$' sign
 as.numeric()

length(gross_data)

[1] 100

The data frame

#Combining all the lists to form a data frame
movies_df<-data.frame(Rank = rank_data, Title = title_data,

Description = description_data, Runtime = runtime_data,

Genre = genre_data, Rating = rating_data,

Metascore = metascore_data, Votes = votes_data,                           Gross_Earning_in_Mil = gross_data,

Director = directors_data) #, Actor = actors_data

#Structure of the data frame

str(movies_df)

'data.frame':   100 obs. of  10 variables:
 $ Rank                : num  1 2 3 4 5 6 7 8 9 10 ...
 $ Title               : chr  "Terrifier" "Suicide Squad" "Silence" "Hush" ...
 $ Description         : chr  "On Halloween night, Tara Heyes finds herself as the obsession of a sadistic murderer known as Art the Clown." "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive "| __truncated__ "In the 17th century, two Portuguese Jesuit priests travel to Japan in an attempt to locate their mentor, who is"| __truncated__ "A deaf and mute writer who retreated into the woods to live a solitary life must fight for her life in silence "| __truncated__ ...
 $ Runtime             : num  85 123 161 82 134 117 139 145 128 108 ...
 $ Genre               : Factor w/ 9 levels "Action","Adventure",..: 9 1 7 9 9 9 4 7 5 3 ...
 $ Rating              : num  5.6 5.9 7.2 6.6 7.3 7.3 8.1 8.1 8 7.1 ...
 $ Metascore           : num  NA 40 79 67 65 63 71 85 94 59 ...
 $ Votes               : num  47604 710171 119435 149202 292206 ...
 $ Gross_Earning_in_Mil: num  NA 325.1 7.1 NA 102.4 ...
 $ Director            : Factor w/ 97 levels "Alessandro Carloni",..: 19 22 61 65 44 59 63 71 18 34 ...

Analyzing scraped data from the web

Question 1: which Movie from which Genre had the longest run time?

The movie Silence has the longest run time and it belongs to the Drama genre.

longrun <- movies_df |>   
  group_by(Genre) |>
  top_n(1, wt = Runtime)  

ggplot(longrun, aes(x = Genre, y = Runtime, fill = Title)) +   
  geom_bar(stat = "identity", position = "dodge", alpha = .65) +  
  labs(title = "Runtime and Genre",
       x = "Genre",        
       y = "Runtime( Minutes)",        
       fill = "Movie Title") +     
  theme_minimal() +   
  theme(axis.text.x = element_text(angle = 35, hjust = 1,),
        plot.title =  element_text(hjust = .5) )

Question 2: In the Runtime of 130-160 mins, which genre has the highest votes?

The genre Biography has the overall average highest votes

votes3060 <-movies_df |> 
  filter(Runtime >= 130, Runtime <=160)  

genrevotes <- votes3060 |>
  group_by(Genre) |>
  summarize(AverageVotes = mean(Votes)) 

ggplot(genrevotes, aes(y = Genre, x = AverageVotes, fill = Genre)) +
  
  geom_bar(stat = "identity", alpha = .65) +
  labs(title = "Highest Vote by Genre",
       subtitle = "Runtime 130-160 min",
       y = "Genre",
       x = "Average Votes (in Thousands)") +
  scale_x_continuous(labels = scales::number_format(scale = 1e-3,      scale_suffix = "k")) +
  theme_minimal()+
  theme(plot.title =  element_text(hjust = .5),
        plot.subtitle = element_text(hjust = .5))

Question 3: Across all genres which genre has the highest average gross earnings in run time 100 to 120.

Animation tops the charts at average 216 Mil

genrearnings <- movies_df |>
  filter(Runtime >= 100, Runtime <= 120) |>
  group_by(Genre) |>
  summarize(AverageEarnings = mean(Gross_Earning_in_Mil, na.rm = TRUE)) |>
  
arrange(desc(AverageEarnings))
print(genrearnings)

# A tibble: 8 × 2
  Genre     AverageEarnings
  <fct>               <dbl>
1 Animation           216. 
2 Adventure           125. 
3 Action               89.2
4 Crime                51.2
5 Drama                48.4
6 Horror               46.8
7 Comedy               33.9
8 Biography            28.7