MultiPage Scrape

### Install Packages

install.packages("rvest")     #package for harvesting data
install.packages("tidyverse") #packages for cleansing data
install.packages("tm")        # package for text mining
install.packages("wordcloud") #package for wordcloud creation

### Load Packages
library(rvest)
library(tidyverse)
library(tm)
library(wordcloud)

url_base <- "https://www.winemag.com/ratings/?s=&drink_type=wine&page=%d"

### Scrape and Extract

map_df(1:50, function(i) {                                                      #returns a data frame with individual elements 1 to 50

cat(".")                                                                        # simple progress indicator 

  pg <- read_html(sprintf(url_base, i))                                         # read and iterate through the base url and store in a variable called pg
  
  
#  read nodes, clean data, place into data frame

 data.frame(
            wine=html_text(html_nodes(pg, "h3.title")),                
            excerpt=html_text(html_nodes(pg, "div.excerpt")),
            rating=gsub(" Points", "", html_text(html_nodes(pg, "span.rating"))),
            appellation=html_text(html_nodes(pg, "span.appellation")),
            price=gsub("\\$", "", html_text(html_nodes(pg, "span.price"))),
             
            stringsAsFactors=FALSE)
}) -> wines

..................................................

## View the dataset
wines

## Examine the data types
sapply(wines, class)

       wine     excerpt      rating appellation       price 
"character" "character" "character" "character" "character" 

## Convert the price and rating columns to numbers and store in a dataframe
wine_price_rating<-data.frame(wines,priceNum=as.numeric(wines$price),ratingNum=as.numeric(wines$rating)) 

## View the dataset
wine_price_rating

#Generate a bar plot that shows the number of wines at each rating level
ggplot(data=wine_price_rating) + 
  geom_bar(aes(x=ratingNum), fill="orange")

#Generate a density plot that shows the distribution of wines at each rating level
ggplot(data=wine_price_rating) + 
  geom_density(aes(x=priceNum), fill="purple")

#Generate box plots that show median price at each rating level
ggplot(wine_price_rating, aes(y=priceNum, x=rating,))+geom_boxplot()

### Create a wordcloud

##Place the exerpts into a corpus
wineCorpus <- VCorpus(VectorSource(wine_price_rating$excerpt))

##Clean the corpus
wineCorpus <- tm_map(wineCorpus, content_transformer(tolower))
wineCorpus <- tm_map(wineCorpus, removePunctuation)
wineCorpus <- tm_map(wineCorpus, PlainTextDocument)
wineCorpus <- tm_map(wineCorpus, removeWords, stopwords('english'))

## Generate the wordcloud
wordcloud(wineCorpus, colors=brewer.pal(8, "Dark2"),  random.color= TRUE, random.order = FALSE, max.words = 20)

NA
NA