## # A tibble: 5 × 3
##   book                chapter short_text          
##   <fct>                 <int> <chr>               
## 1 Sense & Sensibility       1 The family of Dashwo
## 2 Sense & Sensibility       1 was large, and their
## 3 Sense & Sensibility       1 their property, wher
## 4 Sense & Sensibility       1 respectable a manner
## 5 Sense & Sensibility       1 surrounding acquaint

## # A tibble: 10 × 4
##    book                word          n  tf_idf
##    <fct>               <chr>     <int>   <dbl>
##  1 Sense & Sensibility elinor      623 0.00931
##  2 Sense & Sensibility marianne    492 0.00735
##  3 Mansfield Park      crawford    493 0.00551
##  4 Pride & Prejudice   darcy       373 0.00547
##  5 Persuasion          elliot      254 0.00544
##  6 Emma                emma        786 0.00536
##  7 Northanger Abbey    tilney      196 0.00452
##  8 Emma                weston      389 0.00433
##  9 Pride & Prejudice   bennet      294 0.00431
## 10 Persuasion          wentworth   191 0.00409

austen_num_k <- searchK(docs, vocab, K = c(5,10,11,12,13,14,15,20),
                prevalence =~ book + chapter,
                data = meta)

Optimal k was 10

austen_stm <- stm(documents=docs, 
                  data=meta,
                  vocab=vocab, 
                  prevalence =~ book + chapter,
                  K=10,
                  max.em.its=25,
                  verbose = FALSE)

## 
##  Topic 2: 
##       the town--Mrs. Otway, I protest!--and good Mr. Otway, and Miss Otway
##      teachers, Miss Nash, and Miss Prince, and Miss Richardson, to sup with
##      "The lady then--Miss Grey I think you called her--is very rich?"
##      comparing herself with Miss Larolles, the inimitable Miss Larolles; but
##      _I_ think seriously of Miss Smith!--Miss Smith is a very good sort of

• Making topic modeling work requires preprocessing
• Inspecting topics is key to understanding them
• In the future, try creating more stop words/ POS tagging