Step 0: Setting up our context
# Loading packages
library(tidyverse)
## ── Attaching packages ──────────────────────────────────────────────── tidyverse 1.3.0 ──
## ✓ ggplot2 3.3.2 ✓ purrr 0.3.4
## ✓ tibble 3.0.3 ✓ dplyr 1.0.2
## ✓ tidyr 1.1.2 ✓ stringr 1.4.0
## ✓ readr 1.3.1 ✓ forcats 0.5.0
## ── Conflicts ─────────────────────────────────────────────────── tidyverse_conflicts() ──
## x dplyr::filter() masks stats::filter()
## x dplyr::lag() masks stats::lag()
library(tidytext)
library(tm)
## Loading required package: NLP
##
## Attaching package: 'NLP'
## The following object is masked from 'package:ggplot2':
##
## annotate
library(bindrcpp)
library(RColorBrewer)
library(dplyr)
library(tidyr)
library(ngram)
library(knitr)
library(purrr)
library(tibble)
library(wordcloud)
library(stringr)
library(readr)
library(janeaustenr)
Step 1: Exploring data
- Data loading: Use the nglish repository files
twitter_file <- "./final/en_US/en_US.twitter.txt"
news_file <- "./final/en_US/en_US.news.txt"
blogs_file <- "./final/en_US/en_US.blogs.txt"
blogs_size <- file.size(blogs_file) / (2^20)
news_size <- file.size(news_file) / (2^20)
twitter_size <- file.size(twitter_file) / (2^20)
blogs <- readLines(blogs_file, skipNul = TRUE)
news <- readLines(news_file, skipNul = TRUE)
twitter <- readLines(twitter_file, skipNul = TRUE)
blogs_lines <- length(blogs)
news_lines <- length(news)
twitter_lines <- length(twitter)
total_lines <- blogs_lines + news_lines + twitter_lines
- Distribution of characters per line, by file
blogs_nchar <- nchar(blogs)
news_nchar <- nchar(news)
twitter_nchar <- nchar(twitter)
boxplot(blogs_nchar, news_nchar, twitter_nchar, log = "y",
names = c("blogs", "news", "twitter"),
ylab = "log(Number of Characters)", xlab = "File Name")
title("Comparing Distributions of Chracters per Line")
- Total characters per file
blogs_nchar_sum <- sum(blogs_nchar)
news_nchar_sum <- sum(news_nchar)
twitter_nchar_sum <- sum(twitter_nchar)
blogs_words <- wordcount(blogs, sep = " ")
news_words <- wordcount(news, sep = " ")
twitter_words <- wordcount(twitter, sep = " ")
repo_summary <- data.frame(f_names = c("blogs", "news", "twitter"),
f_size = c(blogs_size, news_size, twitter_size),
f_lines = c(blogs_lines, news_lines, twitter_lines),
n_char = c(blogs_nchar_sum, news_nchar_sum, twitter_nchar_sum),
n_words = c(blogs_words, news_words, twitter_words))
repo_summary <- repo_summary %>% mutate(pct_n_char = round(n_char/sum(n_char), 2))
repo_summary <- repo_summary %>% mutate(pct_lines = round(f_lines/sum(f_lines), 2))
repo_summary <- repo_summary %>% mutate(pct_words = round(n_words/sum(n_words), 2))
kable(repo_summary)
| blogs |
200.4242 |
899288 |
206824505 |
37334131 |
0.36 |
0.21 |
0.37 |
| news |
196.2775 |
1010242 |
203223159 |
34372530 |
0.36 |
0.24 |
0.34 |
| twitter |
159.3641 |
2360148 |
162096241 |
30373583 |
0.28 |
0.55 |
0.30 |
saveRDS(repo_summary, "./clean_repos/repo_summary.rds")
- Read the data files into dataframes
blogs <- data_frame(text = blogs)
## Warning: `data_frame()` is deprecated as of tibble 1.1.0.
## Please use `tibble()` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_warnings()` to see where this warning was generated.
news <- data_frame(text = news)
twitter <- data_frame(text = twitter)
Step 2: Data sampling and cleaning
set.seed(1001)
sample_pct <- 0.1
blogs_sample <- blogs %>%
sample_n(., nrow(blogs)*sample_pct)
news_sample <- news %>%
sample_n(., nrow(news)*sample_pct)
twitter_sample <- twitter %>%
sample_n(., nrow(twitter)*sample_pct)
repo_sample <- bind_rows(mutate(blogs_sample, source = "blogs"),
mutate(news_sample, source = "news"),
mutate(twitter_sample, source = "twitter"))
repo_sample$source <- as.factor(repo_sample$source)
- Create filters: stopwords, profanity, non-alphanumeric’s, url’s, repeated letters(+3x)
swear_words <- read_delim("swearWords.csv", delim = "\n", col_names = FALSE)
## Parsed with column specification:
## cols(
## X1 = col_character()
## )
swear_words <- unnest_tokens(swear_words, word, X1)
replace_reg <- "[^[:alpha:][:space:]]*"
replace_url <- "http[^[:space:]]*"
replace_aaa <- "\\b(?=\\w*(\\w)\\1)\\w+\\b"
- Clean the sample. Cleaning is separated from tidying so unnest_tokens function can be used for words, and ngrams.
clean_sample <- repo_sample %>%
mutate(text = str_replace_all(text, replace_reg, "")) %>%
mutate(text = str_replace_all(text, replace_url, "")) %>%
mutate(text = str_replace_all(text, replace_aaa, "")) %>%
mutate(text = iconv(text, "ASCII//TRANSLIT"))
rm(blogs, blogs_nchar, news, news_nchar, twitter, twitter_nchar, replace_reg, replace_url, replace_aaa)
clean_sample<-na.omit(clean_sample)
#Save clean_sample data
saveRDS(clean_sample, file = "./clean_repos/clean_sample.rds" )
- Create tidy dataframe for repo sample
tidy_repo <- clean_sample %>%
unnest_tokens(word, text) %>%
anti_join(swear_words) %>%
anti_join(stop_words)
## Joining, by = "word"
## Joining, by = "word"
Step 3: Results
Most frecuent and distribution words
- Word counts: Number of unique words in repo
(repo_count <- tidy_repo %>%
summarise(keys = n_distinct(word)))
## # A tibble: 1 x 1
## keys
## <int>
## 1 155591
- Number of words to attain 50% and 90% coverage of all words in repo
cover_50 <- tidy_repo %>%
count(word) %>%
mutate(proportion = n / sum(n)) %>%
arrange(desc(proportion)) %>%
mutate(coverage = cumsum(proportion)) %>%
filter(coverage <= 0.5)
nrow(cover_50)
## [1] 1277
cover_90 <- tidy_repo %>%
count(word) %>%
mutate(proportion = n / sum(n)) %>%
arrange(desc(proportion)) %>%
mutate(coverage = cumsum(proportion)) %>%
filter(coverage <= 0.9)
nrow(cover_90)
## [1] 16962
Word distributions
cover_90 %>%
top_n(20, proportion) %>%
mutate(word = reorder(word, proportion)) %>%
ggplot(aes(word, proportion)) +
geom_col() +
xlab(NULL) +
coord_flip()
- Word distribution by source
freq <- tidy_repo %>%
count(source, word) %>%
group_by(source) %>%
mutate(proportion = n / sum(n)) %>%
spread(source, proportion) %>%
gather(source, proportion, `blogs`:`twitter`) %>%
arrange(desc(proportion), desc(n))
freq %>%
filter(proportion > 0.002) %>%
mutate(word = reorder(word, proportion)) %>%
ggplot(aes(word, proportion)) +
geom_col() +
xlab(NULL) +
coord_flip() +
facet_grid(~source, scales = "free")
cover_90 %>%
with(wordcloud(word, n, max.words = 100,
colors = brewer.pal(6, 'Dark2'), random.order = FALSE))
saveRDS(tidy_repo, "./clean_repos/tidy_repo.rds")
saveRDS(cover_90, "./clean_repos/cover_90.rds")
rm(tidy_repo, cover_50, cover_90)
n-grams
- Bigrams: Create bigrams by source using unnest_tokens
bigram_repo <- clean_sample %>%
unnest_tokens(bigram, text, token = "ngrams", n = 2)
- Number of bigrams to attain 90% coverage of all bigrams in repo
bigram_cover_90 <- bigram_repo %>%
count(bigram) %>%
mutate(proportion = n / sum(n)) %>%
arrange(desc(proportion)) %>%
mutate(coverage = cumsum(proportion)) %>%
filter(coverage <= 0.9)
nrow(bigram_cover_90)
## [1] 1392675
bigram_cover_90 %>%
top_n(20, proportion) %>%
mutate(bigram = reorder(bigram, proportion)) %>%
ggplot(aes(bigram, proportion)) +
geom_col() +
xlab(NULL) +
coord_flip()
saveRDS(bigram_cover_90, "./clean_repos/bigram_cover_90.rds")
- Trigrams: Create Trigrams by source using unnest_tokens
trigram_repo <- clean_sample %>%
unnest_tokens(trigram, text, token = "ngrams", n = 3)
- Number of trigrams to attain 90% coverage of all trigrams in repo
trigram_cover_90 <- trigram_repo %>%
count(trigram) %>%
mutate(proportion = n / sum(n)) %>%
arrange(desc(proportion)) %>%
mutate(coverage = cumsum(proportion)) %>%
filter(coverage <= 0.9)
nrow(trigram_cover_90)
## [1] 5065895
trigram_cover_90 %>%
top_n(20, proportion) %>%
mutate(trigram = reorder(trigram, proportion)) %>%
ggplot(aes(trigram, proportion)) +
geom_col() +
xlab(NULL) +
coord_flip()
saveRDS(trigram_cover_90, "./clean_repos/trigram_cover_90.rds")
- Quadgrams: Create quadgrams by source using unnest_tokens
quadgram_repo <- clean_sample %>%
unnest_tokens(quadgram, text, token = "ngrams", n = 4)
- Number of quadgrams to attain 90% coverage of all quadgrams in repo
quadgram_cover_90 <- quadgram_repo %>%
count(quadgram) %>%
mutate(proportion = n / sum(n)) %>%
arrange(desc(proportion)) %>%
mutate(coverage = cumsum(proportion)) %>%
filter(coverage <= 0.9)
nrow(quadgram_cover_90)
## [1] 7170725
quadgram_cover_90 %>%
top_n(20, proportion) %>%
mutate(quadgram = reorder(quadgram, proportion)) %>%
ggplot(aes(quadgram, proportion)) +
geom_col() +
xlab(NULL) +
coord_flip()
quadgrams_separated <- quadgram_cover_90 %>%
separate(quadgram, c("word1", "word2", "word3", "word4"), sep = " ")
quadgrams_separated
## # A tibble: 7,170,725 x 7
## word1 word2 word3 word4 n proportion coverage
## <chr> <chr> <chr> <chr> <int> <dbl> <dbl>
## 1 the end of the 740 0.0000840 0.0000840
## 2 the rest of the 659 0.0000748 0.000159
## 3 for the first time 623 0.0000708 0.000230
## 4 at the end of 616 0.0000700 0.000300
## 5 at the same time 495 0.0000562 0.000356
## 6 is going to be 458 0.0000520 0.000408
## 7 is one of the 423 0.0000480 0.000456
## 8 one of the most 401 0.0000455 0.000501
## 9 when it comes to 388 0.0000441 0.000546
## 10 going to be a 368 0.0000418 0.000587
## # … with 7,170,715 more rows
saveRDS(quadgram_cover_90, "./clean_repos/quadgram_cover_90.rds")
end <- Sys.time()
#(run_time <- end - start_time)