library(tidytext)
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.4     ✔ readr     2.1.5
## ✔ forcats   1.0.0     ✔ stringr   1.5.1
## ✔ ggplot2   3.4.4     ✔ tibble    3.2.1
## ✔ lubridate 1.9.3     ✔ tidyr     1.3.1
## ✔ purrr     1.0.2     
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(MASS)
## 
## Attaching package: 'MASS'
## 
## The following object is masked from 'package:dplyr':
## 
##     select
grateful_full <- read_csv("Middlebury Parents Posts Comments GRATEFUL.csv", col_names = FALSE) |> 
  rename("grateful" = "X1") |> 
  mutate(post_num = seq(from = 1, to = 105))
## Rows: 105 Columns: 1
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (1): X1
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
regular_full <- read_csv("Middlebury Parents Posts Comments.csv", col_names = FALSE) |> 
  rename("regular" = "X1") |> 
  mutate(post_num = seq(from = 106, to = 226))
## Rows: 121 Columns: 1
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (1): X1
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
grateful_tokenized <- grateful_full |> 
  unnest_tokens(input = "grateful",
                output = "word") |> 
  mutate(category = "grateful")

regular_tokenized <- regular_full |> 
  unnest_tokens(input = "regular",
                output = "word") |> 
  mutate(category = "regular")

tokens_combined <- rbind(regular_tokenized, grateful_tokenized) |> 
  mutate(word = str_to_lower(word))
docm <- tokens_combined |> 
  filter(!(word %in% stop_words$word)) |> 
  pivot_wider(id_cols = c(post_num, category),
              names_from = "word",
              values_from = "word",
              values_fn = length,
              values_fill = 0)

docm_simp <- docm |> dplyr::select(-c(post_num, category))

pca <- prcomp(docm_simp)
  
pca$rotation[,1] |> 
  sort(decreasing = TRUE) %>%
  .[1:5]
##    parents    college       midd experience       page 
##  0.5126932  0.4446306  0.2428585  0.1431086  0.1413285
pca$rotation[,2] |> 
  sort(decreasing = TRUE) %>%
  .[1:5]
##       midd       kids      happy   grateful       page 
## 0.41734116 0.16078025 0.12350822 0.07237844 0.06077601
hc <- hclust(docm_simp |> 
                dist())
plot(hc)

#sad post
grateful_full |> 
  filter(post_num == 218) %>%
  .[,1]
## # A tibble: 0 × 1
## # ℹ 1 variable: grateful <chr>
regular_full |> 
  filter(post_num == 218) %>%
  .[,1]
## # A tibble: 1 × 1
##   regular                                                                       
##   <chr>                                                                         
## 1 So heartbreaking. My son’s girlfriend. She is pure love. We are so lucky to h…
regular_full |> 
  filter(post_num == 220) %>%
  .[,1]
## # A tibble: 1 × 1
##   regular                                                                       
##   <chr>                                                                         
## 1 I agree I wish they would have more experienced trauma and grief counselors a…
grateful_full |> 
  filter(post_num == 40) %>%
  .[,1]
## # A tibble: 1 × 1
##   grateful                                                                      
##   <chr>                                                                         
## 1 Some Midd kids wake up late during J Term, hungover and groggy. I’ve heard th…
set.seed(1)
rows_to_keep <- sample(1:nrow(docm), size = nrow(docm)/2)
train_data <- docm[rows_to_keep, ]
test_data <- docm[-rows_to_keep, ]

library(MASS)
lda1 <- lda(category ~. -category -post_num, data = docm)
## Warning in lda.default(x, grouping, ...): variables are collinear
#lda_pca <- lda(category ~. -category -post_num, data = )

preds <- predict(lda1, test_data)

table(test_data$category, preds$class)
##           
##            grateful regular
##   grateful       48       1
##   regular         0      64

What was that 1 incorrect prediction?

df <- data.frame(test_data$category, preds$class)

df |> 
  arrange(test_data.category) |> 
  mutate(post_num = seq(from = 1, to = nrow(df))) |> 
  filter(test_data.category != preds.class)
##   test_data.category preds.class post_num
## 1           grateful     regular        2
grateful_full |> 
  filter(post_num == 2)
## # A tibble: 1 × 2
##   grateful                                                              post_num
##   <chr>                                                                    <int>
## 1 "If you have a pic of your happy grateful Midd kid, please share! We…        2
#library(psych)
#cohen.kappa(table(docm$category, preds$class))