HW 10, Michael Simms

library(rvest)

url <- 'http://www.imdb.com/search/title?count=100&release_date=2016,2016&title_type=feature'

webpage <- read_html(url)

#Using CSS selectors to scrape the rankings section
rank_data_html <- html_nodes(webpage,'.text-primary')

#Converting the ranking data to text
rank_data <- html_text(rank_data_html)

#Let's have a look at the rankings
head(rank_data)

[1] "1." "2." "3." "4." "5." "6."

#Data-Preprocessing: Converting rankings to numerical
rank_data<-as.numeric(rank_data)

#Let's have another look at the rankings
head(rank_data)

[1] 1 2 3 4 5 6

#Using CSS selectors to scrape the title section
title_data_html <- html_nodes(webpage,'.lister-item-header a')

#Converting the title data to text
title_data <- html_text(title_data_html)

#Let's have a look at the title
head(title_data)

[1] "Independence Day: Resurgence" "The Huntsman: Winter's War"  
[3] "The Nice Guys"                "Deadpool"                    
[5] "Hacksaw Ridge"                "Arrival"

#Using CSS selectors to scrape the description section
description_data_html <- html_nodes(webpage,'.ratings-bar+ .text-muted')

#Converting the description data to text
description_data <- html_text(description_data_html)

#Let's have a look at the description data
head(description_data)

[1] "\nTwo decades after the first Independence Day invasion, Earth is faced with a new extra-Solar threat. But will mankind's new space defenses be enough?"                                                               
[2] "\nEric and fellow warrior Sara, raised as members of ice Queen Freya's army, try to conceal their forbidden love as they fight to survive the wicked intentions of both Freya and her sister Ravenna."                 
[3] "\nIn 1970s Los Angeles, a mismatched pair of private eyes investigate a missing girl and the mysterious death of a porn star."                                                                                         
[4] "\nA wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                               
[5] "\nWorld War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."
[6] "\nA linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appear around the world."

#Data-Preprocessing: removing '\n'
description_data<-gsub("\n","",description_data)

#Let's have another look at the description data 
head(description_data)

[1] "Two decades after the first Independence Day invasion, Earth is faced with a new extra-Solar threat. But will mankind's new space defenses be enough?"                                                               
[2] "Eric and fellow warrior Sara, raised as members of ice Queen Freya's army, try to conceal their forbidden love as they fight to survive the wicked intentions of both Freya and her sister Ravenna."                 
[3] "In 1970s Los Angeles, a mismatched pair of private eyes investigate a missing girl and the mysterious death of a porn star."                                                                                         
[4] "A wisecracking mercenary gets experimented on and becomes immortal but ugly, and sets out to track down the man who ruined his looks."                                                                               
[5] "World War II American Army Medic Desmond T. Doss, who served during the Battle of Okinawa, refuses to kill people and becomes the first man in American history to receive the Medal of Honor without firing a shot."
[6] "A linguist works with the military to communicate with alien lifeforms after twelve mysterious spacecraft appear around the world."

#Using CSS selectors to scrape the Movie runtime section
runtime_data_html <- html_nodes(webpage,'.text-muted .runtime')

#Converting the runtime data to text
runtime_data <- html_text(runtime_data_html)

#Let's have a look at the runtime
head(runtime_data)

[1] "120 min" "114 min" "116 min" "108 min" "139 min" "116 min"

#Data-Preprocessing: removing mins and converting it to numerical
runtime_data<-gsub(" min","",runtime_data)
runtime_data<-as.numeric(runtime_data)

#Let's have another look at the runtime data
head(runtime_data)

[1] 120 114 116 108 139 116

#Using CSS selectors to scrape the Movie genre section
genre_data_html <- html_nodes(webpage,'.genre')

#Converting the genre data to text
genre_data <- html_text(genre_data_html)

#Let's have a look at the runtime
head(genre_data)

[1] "\nAction, Adventure, Sci-Fi            "
[2] "\nAction, Adventure, Drama            " 
[3] "\nAction, Comedy, Crime            "    
[4] "\nAction, Comedy            "           
[5] "\nBiography, Drama, History            "
[6] "\nDrama, Mystery, Sci-Fi            "

#Data-Preprocessing: removing \n
genre_data<-gsub("\n","",genre_data)

#Data-Preprocessing: removing excess spaces
genre_data<-gsub(" ","",genre_data)

#taking only the first genre of each movie
genre_data<-gsub(",.*","",genre_data)

#Convering each genre from text to factor
genre_data<-as.factor(genre_data)

#Let's have another look at the genre data
head(genre_data)

[1] Action    Action    Action    Action    Biography Drama    
Levels: Action Adventure Animation Biography Comedy Crime Drama Horror

#Using CSS selectors to scrape the IMDB rating section
rating_data_html <- html_nodes(webpage,'.ratings-imdb-rating strong')

#Converting the ratings data to text
rating_data <- html_text(rating_data_html)

#Let's have a look at the ratings
head(rating_data)

[1] "5.2" "6.1" "7.3" "8.0" "8.1" "7.9"

#Data-Preprocessing: converting ratings to numerical
rating_data<-as.numeric(rating_data)

#Let's have another look at the ratings data
head(rating_data)

[1] 5.2 6.1 7.3 8.0 8.1 7.9

#Using CSS selectors to scrape the votes section
votes_data_html <- html_nodes(webpage,'.sort-num_votes-visible span:nth-child(2)')

#Converting the votes data to text
votes_data <- html_text(votes_data_html)

#Let's have a look at the votes data
head(votes_data)

[1] "184,427"   "114,942"   "345,730"   "1,074,690" "554,968"   "724,506"

#Data-Preprocessing: removing commas
votes_data<-gsub(",","",votes_data)

#Data-Preprocessing: converting votes to numerical
votes_data<-as.numeric(votes_data)

#Let's have another look at the votes data
head(votes_data)

[1]  184427  114942  345730 1074690  554968  724506

#Using CSS selectors to scrape the directors section
directors_data_html <- html_nodes(webpage,'.text-muted+ p a:nth-child(1)')

#Converting the directors data to text
directors_data <- html_text(directors_data_html)

#Let's have a look at the directors data
head(directors_data)

[1] "Roland Emmerich"       "Cedric Nicolas-Troyan" "Shane Black"          
[4] "Tim Miller"            "Mel Gibson"            "Denis Villeneuve"

#Data-Preprocessing: converting directors data into factors
directors_data<-as.factor(directors_data)

#Using CSS selectors to scrape the actors section
actors_data_html <- html_nodes(webpage,'.lister-item-content .ghost+ a')

#Converting the gross actors data to text
actors_data <- html_text(actors_data_html)

#Let's have a look at the actors data
head(actors_data)

[1] "Liam Hemsworth"  "Chris Hemsworth" "Russell Crowe"   "Ryan Reynolds"  
[5] "Andrew Garfield" "Amy Adams"

#Data-Preprocessing: converting actors data into factors
actors_data<-as.factor(actors_data)
#But, I want you to closely follow what happens when I do the same thing for Metascore data.

#Using CSS selectors to scrape the metascore section
metascore_data_html <- html_nodes(webpage,'.metascore')

#Converting the runtime data to text
metascore_data <- html_text(metascore_data_html)

#Let's have a look at the metascore data
head(metascore_data)

[1] "32        " "35        " "70        " "65        " "71        "
[6] "81        "

#Data-Preprocessing: removing extra space in metascore
metascore_data<-gsub(" ","",metascore_data)

#Lets check the length of metascore data
length(metascore_data)

[1] 95

for (i in c(39,73,80,89)){

a<-metascore_data[1:(i-1)]

b<-metascore_data[i:length(metascore_data)]

metascore_data<-append(a,list("NA"))

metascore_data<-append(metascore_data,b)

}

#Data-Preprocessing: converting metascore to numerical
metascore_data<-as.numeric(metascore_data)

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

#Let's have another look at length of the metascore data

length(metascore_data)

[1] 99

#Let's look at summary statistics
summary(metascore_data)

   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
  21.00   47.00   61.00   60.27   73.50   99.00       4

#Using CSS selectors to scrape the gross revenue section
gross_data_html <- html_nodes(webpage,'.ghost~ .text-muted+ span')

#Converting the gross revenue data to text
gross_data <- html_text(gross_data_html)

#Let's have a look at the votes data
head(gross_data)

[1] "$103.14M" "$48.39M"  "$36.26M"  "$363.07M" "$67.21M"  "$100.55M"

#Data-Preprocessing: removing '$' and 'M' signs
gross_data<-gsub("M","",gross_data)

gross_data<-substring(gross_data,2,6)

#Let's check the length of gross data
length(gross_data)

[1] 89

#Filling missing entries with NA
for (i in c(17,39,49,52,57,64,66,73,76,77,80,87,88,89)){

a<-gross_data[1:(i-1)]

b<-gross_data[i:length(gross_data)]

gross_data<-append(a,list("NA"))

gross_data<-append(gross_data,b)

}

#Data-Preprocessing: converting gross to numerical
gross_data<-as.numeric(gross_data)

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

Warning: NAs introduced by coercion

#Let's have another look at the length of gross data
length(gross_data)

[1] 103

summary(gross_data)

   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
   0.01   21.22   58.70  100.97  127.40  532.10      14

#Combining all the lists to form a data frame
#movies_df<-data.frame(Rank = rank_data, Title = title_data,
#Description = description_data, Runtime = runtime_data,
#Genre = genre_data, Rating = rating_data,
#Metascore = metascore_data, Votes = votes_data,                                                             #Gross_Earning_in_Mil = gross_data,
#Director = directors_data, Actor = actors_data)

#Structure of the data frame

#str(movies_df)