library(here)## here() starts at /home/munira.odigie/topic modelslibrary(dplyr)##
## Attaching package: 'dplyr'## The following objects are masked from 'package:stats':
##
## filter, lag## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, unionlibrary(tidytext)
library(tidyr)
library(knitr)
library(ggplot2)
library(tm)## Loading required package: NLP##
## Attaching package: 'NLP'## The following object is masked from 'package:ggplot2':
##
## annotatelibrary(topicmodels)Use this file to analyze your own data. Follow along with the same pattern as the Quaker Saints analysis.
load("~/Preparingdata/data/Thissideofparadise_long.rda")
load("~/Preparingdata/data/Thissideofparadise_trimmed.rda")First add your own data to the repository in the form of an R data set. This should be a long data set and should have a variable indicating how the data are divided up (chapters, books, stories etc). Commit and push.
What steps do you normally do at the top? Do that.
Thissideofparadise_long |> group_by(chapter_titles, word) |> summarize(n = n()) |>
cast_dtm( term = word, document=chapter_titles, n) -> Thissideofparadise_dtm## `summarise()` has grouped output by 'chapter_titles'. You can override using
## the `.groups` argument.Create an LDA object Try running this several times. Do you get the same result each time?
tsop_lda_2 <-
LDA( Thissideofparadise_dtm, k = 2, control = list(seed = 1234))
tsop_lda_2 ## A LDA_VEM topic model with 2 topics.tsop_topics_2 <- tidy(tsop_lda_2, matrix = "beta")
head(tsop_topics_2)## # A tibble: 6 × 3
## topic term beta
## <int> <chr> <dbl>
## 1 1 abroad 0.000150
## 2 2 abroad 0.000116
## 3 1 abrupt 0.000150
## 4 2 abrupt 0.0000582
## 5 1 absolutely 0.000374
## 6 2 absolutely 0.000408What happens if you change the value of the seed? Do you get the same results?
tsop_lda_6 <-
LDA( Thissideofparadise_dtm, k = 6, control = list(seed = 1234))
tsop_topics_6 <- tidy(tsop_lda_6, matrix = "beta")
head(tsop_topics_6)## # A tibble: 6 × 3
## topic term beta
## <int> <chr> <dbl>
## 1 1 abroad 4.47e-75
## 2 2 abroad 1.47e- 4
## 3 3 abroad 1.34e-43
## 4 4 abroad 3.63e-84
## 5 5 abroad 1.32e- 4
## 6 6 abroad 4.87e- 4Tidy the data and then try running for two topics.
tsop_top_terms_2 <- tsop_topics_2 |>
group_by(topic) |>
slice_max(beta, n = 10) |>
ungroup() %>%
arrange(topic, -beta)
tsop_top_terms_2 |>
mutate(term = reorder_within(term, beta, topic)) |>
ggplot(aes(beta, term, fill = factor(topic))) +
geom_col(show.legend = FALSE) +
facet_wrap(~ topic, scales = "free") +
scale_y_reordered()
tsop_top_terms_6 <- tsop_topics_6 |>
group_by(topic) |>
slice_max(beta, n = 10) |>
ungroup() %>%
arrange(topic, -beta)
tsop_top_terms_6 |>
mutate(term = reorder_within(term, beta, topic)) |>
ggplot(aes(beta, term, fill = factor(topic))) +
geom_col(show.legend = FALSE) +
facet_wrap(~ topic, scales = "free") +
scale_y_reordered()
beta_wide_2 <- tsop_topics_2 |>
mutate(topic = paste0("topic", topic)) |>
pivot_wider(names_from = topic, values_from = beta) |>
filter(topic1 > .001 | topic2 > .001) |>
mutate(log_ratio = log2(topic2 / topic1))
beta_wide_2 |> slice_max(order_by = log_ratio, n = 10)## # A tibble: 11 × 4
## term topic1 topic2 log_ratio
## <chr> <dbl> <dbl> <dbl>
## 1 gillespie 9.16e-135 0.00134 436.
## 2 cecelia 3.51e-134 0.00233 435.
## 3 connage 2.54e- 61 0.00285 193.
## 4 burne 1.50e- 4 0.00431 4.85
## 5 alec 3.74e- 4 0.00495 3.73
## 6 rosalind 1.20e- 3 0.0101 3.07
## 7 kerry 3.74e- 4 0.00303 3.02
## 8 freshman 1.50e- 4 0.00105 2.81
## 9 eleanor 3.74e- 4 0.00221 2.56
## 10 meet 2.99e- 4 0.00151 2.34
## 11 clara 2.99e- 4 0.00151 2.34beta_wide_2 |> slice_min(order_by = log_ratio, n = 10)## # A tibble: 10 × 4
## term topic1 topic2 log_ratio
## <chr> <dbl> <dbl> <dbl>
## 1 myra 0.00180 0.0000582 -4.95
## 2 beatrice 0.00292 0.000116 -4.65
## 3 regis 0.00120 0.000116 -3.36
## 4 human 0.00105 0.000116 -3.17
## 5 monsignor 0.00299 0.000466 -2.68
## 6 thousand 0.00120 0.000233 -2.36
## 7 american 0.00127 0.000291 -2.13
## 8 sloane 0.00150 0.000408 -1.88
## 9 chair 0.00105 0.000291 -1.85
## 10 em 0.00120 0.000349 -1.78Follow the analysis pattern from the example and then describe the results.
Make sure to do both the beta and the gamma analyses.
tsop_documents_2 <- tidy(tsop_lda_2, matrix = "gamma")
tsop_documents_2## # A tibble: 20 × 3
## document topic gamma
## <chr> <int> <dbl>
## 1 CHAPTER 1. AMORY, SON OF BEATRICE 1 1.00
## 2 CHAPTER 1. THE DEBUTANTE 1 0.00000819
## 3 CHAPTER 2. EXPERIMENTS IN CONVALESCENCE 1 1.00
## 4 CHAPTER 2. SPIRES AND GARGOYLES 1 0.00000353
## 5 CHAPTER 3. THE EGOTIST CONSIDERS 1 1.00
## 6 CHAPTER 3. YOUNG IRONY 1 0.00000957
## 7 CHAPTER 4. NARCISSUS OFF DUTY 1 0.00000511
## 8 CHAPTER 4. THE SUPERCILIOUS SACRIFICE 1 0.0000195
## 9 CHAPTER 5. THE EGOTIST BECOMES A PERSONAGE 1 1.00
## 10 <NA> 1 0.993
## 11 CHAPTER 1. AMORY, SON OF BEATRICE 2 0.00000553
## 12 CHAPTER 1. THE DEBUTANTE 2 1.00
## 13 CHAPTER 2. EXPERIMENTS IN CONVALESCENCE 2 0.00000866
## 14 CHAPTER 2. SPIRES AND GARGOYLES 2 1.00
## 15 CHAPTER 3. THE EGOTIST CONSIDERS 2 0.00000716
## 16 CHAPTER 3. YOUNG IRONY 2 1.00
## 17 CHAPTER 4. NARCISSUS OFF DUTY 2 1.00
## 18 CHAPTER 4. THE SUPERCILIOUS SACRIFICE 2 1.00
## 19 CHAPTER 5. THE EGOTIST BECOMES A PERSONAGE 2 0.00000654
## 20 <NA> 2 0.00745Now try with 3 topics, how do the results differ?
tsop_documents_6 <- tidy(tsop_lda_6, matrix = "gamma")
tsop_documents_6## # A tibble: 60 × 3
## document topic gamma
## <chr> <int> <dbl>
## 1 CHAPTER 1. AMORY, SON OF BEATRICE 1 0.00000550
## 2 CHAPTER 1. THE DEBUTANTE 1 0.00000815
## 3 CHAPTER 2. EXPERIMENTS IN CONVALESCENCE 1 1.00
## 4 CHAPTER 2. SPIRES AND GARGOYLES 1 0.00000351
## 5 CHAPTER 3. THE EGOTIST CONSIDERS 1 1.00
## 6 CHAPTER 3. YOUNG IRONY 1 0.00000952
## 7 CHAPTER 4. NARCISSUS OFF DUTY 1 0.00000508
## 8 CHAPTER 4. THE SUPERCILIOUS SACRIFICE 1 0.0000194
## 9 CHAPTER 5. THE EGOTIST BECOMES A PERSONAGE 1 0.00000651
## 10 <NA> 1 0.00720
## # ℹ 50 more rowstop_terms_6 <- tsop_topics_6 |>
group_by(topic) |>
slice_max(beta, n = 5) |>
ungroup() |>
arrange(topic, -beta)
top_terms_6## # A tibble: 31 × 3
## topic term beta
## <int> <chr> <dbl>
## 1 1 amory 0.0283
## 2 1 tom 0.00639
## 3 1 life 0.00432
## 4 1 people 0.00397
## 5 1 half 0.00363
## 6 2 amory 0.0235
## 7 2 burne 0.00998
## 8 2 night 0.00675
## 9 2 eleanor 0.00558
## 10 2 time 0.00470
## # ℹ 21 more rowstop_terms_6 %>%
mutate(term = reorder_within(term, beta, topic)) |>
ggplot(aes(beta, term, fill = factor(topic))) +
geom_col(show.legend = FALSE) +
facet_wrap(~ topic, scales = "free") +
scale_y_reordered()
For your analysis try different numbers of topics (these can include
large values of k, don’t be afraid to choose 10 or maybe even more.
)m
Write up a paragraph explaining what you learned about your documents.
What I about my documents are how different words show on different topics and what the most used words are. I am learning how to use beta, topics, and gamma. The topics for words show you what the difference of what certain titles are about. Also have visual representations of the words helps alot with seeing how much of the words they are.