Download data

# download the book : 'Moby Dick'
moby <- gutenberg_download(gutenberg_id = 2701)
moby

# A tibble: 21,712 x 2
   gutenberg_id text                                                      
          <int> <chr>                                                     
 1         2701 MOBY DICK; OR THE WHALE                                   
 2         2701 ""                                                        
 3         2701 By Herman Melville                                        
 4         2701 ""                                                        
 5         2701 ""                                                        
 6         2701 ""                                                        
 7         2701 ""                                                        
 8         2701 Original Transcriber's Notes:                             
 9         2701 ""                                                        
10         2701 This text is a combination of etexts, one from the now-de…
# ... with 21,702 more rows

First, we can remove all blank lines (having only "" as text).

# remove blank lines
moby <- moby %>% filter(!text == "")

We have to transform the dataframe, to respect the conditions of import a dataframe into a source (package tm).

moby_df <- moby

# rename the columns as necesary
colnames(moby_df) <- c("doc_id", "text")

# provide a unique identifier to the column 'doc_id'
moby_df[, "doc_id"] <- rownames(moby_df)

head(moby_df)

# A tibble: 6 x 2
  doc_id text                                                             
  <chr>  <chr>                                                            
1 1      MOBY DICK; OR THE WHALE                                          
2 2      By Herman Melville                                               
3 3      Original Transcriber's Notes:                                    
4 4      This text is a combination of etexts, one from the now-defunct E…
5 5      project at Virginia Tech and one from Project Gutenberg's archiv…
6 6      proofreaders of this version are indebted to The University of A…

Corpus

# 1- create a source form a dataframe
moby_source <- DataframeSource(moby_df)

# 2- create a corpus
moby_corpus <- VCorpus(moby_source)
moby_corpus

<<VCorpus>>
Metadata:  corpus specific: 0, document level (indexed): 0
Content:  documents: 18563

# Look at a sample
moby_corpus[[10]]$content

[1] "for the British pound, a unit of currency."

moby_corpus[[10]]$meta

  author       : character(0)
  datetimestamp: 2018-09-25 16:01:31
  description  : character(0)
  heading      : character(0)
  id           : 10
  language     : en
  origin       : character(0)

We will create a function that clean the corpus.

# clean the corpus, by removin punctuation, white spaces, numbers, and
# typical english words (the, if, ...)
tm_clean <- function(corpus) {
    corpus <- tm_map(corpus, content_transformer(tolower))  # convert to lower case
    corpus <- tm_map(corpus, removePunctuation)  # remove punctuation
    corpus <- tm_map(corpus, removeWords, c(stopwords("en")))  # remove typical english words
    corpus <- tm_map(corpus, removeNumbers)  # remove numbers
    corpus <- tm_map(corpus, stemDocument)  # aggregate seem words (like 'whale' and 'whales')
    corpus <- tm_map(corpus, stripWhitespace)  # remove extra spaces
    return(corpus)
}

# apply the function to our corpus
moby_clean <- tm_clean(moby_corpus)

# Compare the previous version and the new version modified
moby_corpus[[10]]$content

[1] "for the British pound, a unit of currency."

moby_clean[[10]]$content

[1] "british pound unit currenc"

We now have a cleaned corpus.

Term Document Matrix

# create a TDM (Term Document Matrix), to count words
moby_tdm <- TermDocumentMatrix(moby_clean)

For visualization, we need to tweak a little this TDM.

# transform the result into a matrix
moby_tdm_m <- as.matrix(moby_tdm)

# sum all the lines to have the frequency, and sort the results
term_frequency <- rowSums(moby_tdm_m)
term_frequency <- sort(term_frequency, decreasing = TRUE)

# and create a dataframe of it
term_frequency <- data.frame(term_frequency)
term_frequency$word <- rownames(term_frequency)