Converting to and from non-tidy formats

library(dplyr)
library(data.table)
library(ggplot2)

5.1 Tidying a document-term matrix

5.1.1 Tidying DocumentTermMatrix objects

library(tm)
data("AssociatedPress", package = "topicmodels")
AssociatedPress

<<DocumentTermMatrix (documents: 2246, terms: 10473)>>
Non-/sparse entries: 302031/23220327
Sparsity           : 99%
Maximal term length: 18
Weighting          : term frequency (tf)

terms <- Terms(AssociatedPress)
head(terms)

[1] "aaron"      "abandon"    "abandoned"  "abandoning" "abbott"     "abboud"    

library(ggplot2)
ap_sentiments %>%
  count(sentiment, term, wt = count) %>%
  ungroup() %>%
  filter(n >= 200) %>%
  mutate(n = ifelse(sentiment == "negative", -n, n)) %>%
  mutate(term = reorder(term, n)) %>%
  ggplot(aes(term, n, fill = sentiment)) +
  geom_bar(stat = "identity") +
  ylab("Contribution to sentiment") +
  coord_flip()

5.1.2 Tidying dfm objects

data("data_corpus_inaugural", package = "quanteda")
inaug_dfm <- quanteda::dfm(data_corpus_inaugural, verbose = FALSE)
inaug_dfm

Document-feature matrix of: 58 documents, 9,357 features (91.8% sparse).

inaug_td <- tidy(inaug_dfm)
inaug_td

inaug_tf_idf <- inaug_td %>%
  bind_tf_idf(term, document, count) %>%
  arrange(desc(tf_idf))
inaug_tf_idf

library(tidyr)
year_term_counts <- inaug_td %>%
  extract(document, "year", "(\\d+)", convert = TRUE) %>%
  complete(year, term, fill = list(count = 0)) %>%
  group_by(year) %>%
  mutate(year_total = sum(count))

year_term_counts %>%
  filter(term %in% c("god", "america", "foreign", "union", "constitution", "freedom")) %>%
  ggplot(aes(year, count / year_total)) +
  geom_point() +
  geom_smooth() +
  facet_wrap(~ term, scales = "free_y") +
  scale_y_continuous(labels = scales::percent_format()) +
  ylab("% frequency of word in inaugural address")

5.2 Casting tidy text data into a matrix

lyric_data=fread('../data/prince_raw_data.csv')

lyric_data %>% 
  group_by(album) %>%
  summarise(
    count=n()
  ) %>%
  as.data.frame %>%
  .[order(.$count,decreasing=T),]

albums=c('Other Songs','Crystal Ball','Emancipation','Rave Un2 the Joy Fantastic')

library(stringr)
# divide into documents, each representing one chapter
by_album <- lyric_data[,1:6] %>%
  filter(album %in% albums) %>%
  group_by(album) %>%
  mutate(linenumber=row_number()) %>%
  ungroup() %>%
  unite(document, album)
# split into words
by_album_word <- by_album %>%
  unnest_tokens(word, text)
# find document-word counts
word_counts <- by_album_word %>%
  anti_join(stop_words) %>%
  count(document, word, sort = TRUE) %>%
  ungroup()

Joining, by = "word"

word_counts

word_counts %>%
 cast_dtm(document, word, n)

Trying to compute distinct() for variables not found in the data:
- `row_col`, `column_col`
This is an error, but only a warning is raised for compatibility reasons.
The operation will return the input unchanged.

<<DocumentTermMatrix (documents: 4, terms: 3503)>>
Non-/sparse entries: 5081/8931
Sparsity           : 64%
Maximal term length: 17
Weighting          : term frequency (tf)

word_counts %>%
  cast_dfm(document,word,n)

Trying to compute distinct() for variables not found in the data:
- `row_col`, `column_col`
This is an error, but only a warning is raised for compatibility reasons.
The operation will return the input unchanged.

Document-feature matrix of: 4 documents, 3,503 features (63.7% sparse).

library(Matrix)

package 愼㸱愼㸵Matrix愼㸱愼㸶 was built under R version 3.4.4
Attaching package: 愼㸱愼㸵Matrix愼㸱愼㸶

The following object is masked from 愼㸱愼㸵package:tidyr愼㸱愼㸶:

    expand

# cast into a Matrix object
m <- word_counts %>%
  cast_sparse(document, word, n)

Trying to compute distinct() for variables not found in the data:
- `row_col`, `column_col`
This is an error, but only a warning is raised for compatibility reasons.
The operation will return the input unchanged.

class(m)

[1] "dgCMatrix"
attr(,"package")
[1] "Matrix"

dim(m)

[1]    4 3503

5.3 Tidying corpus objects with metadata

data("acq")
acq

<<VCorpus>>
Metadata:  corpus specific: 0, document level (indexed): 0
Content:  documents: 50

#first document
acq[[1]]

<<PlainTextDocument>>
Metadata:  15
Content:  chars: 1287

acq_td <- tidy(acq)
acq_td

acq_tokens <- acq_td %>%
  select(-places) %>%
  unnest_tokens(word, text) %>%
  anti_join(stop_words, by = "word")
# most common words
acq_tokens %>%
  count(word, sort = TRUE)

# tf-idf
acq_tokens %>%
  count(id, word) %>%
  bind_tf_idf(word, id, n) %>%
  arrange(desc(tf_idf))