Introduction

The goal of this project is to demonstrate that we have successfully loaded the data, performed basic exploratory analysis, and are on track to create a prediction algorithm. Summary Statistics

# Load necessary libraries
library(tm)
## Loading required package: NLP
library(tidytext)
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(ggplot2)
## 
## Attaching package: 'ggplot2'
## The following object is masked from 'package:NLP':
## 
##     annotate
library(tidyr)
library(textmineR)
## Loading required package: Matrix
## 
## Attaching package: 'Matrix'
## The following objects are masked from 'package:tidyr':
## 
##     expand, pack, unpack
## 
## Attaching package: 'textmineR'
## The following object is masked from 'package:Matrix':
## 
##     update
## The following object is masked from 'package:stats':
## 
##     update
# Define file path
file_path <- "C:/Users/Admin/Documents/en_US/combined_file.txt"

# Function to read and tokenize text
tokenize_text <- function(file_path) {
  text <- tolower(readLines(file_path, warn = FALSE))
  text <- paste(text, collapse = " ")
  tokens <- unlist(strsplit(text, "\\W+"))
  tokens <- tokens[tokens != ""]
  return(tokens)
}

# Function to filter profanity from tokens
filter_profanity <- function(tokens, profanity_list) {
  clean_tokens <- tokens[!tokens %in% profanity_list]
  return(clean_tokens)
}

# Example profanity list (extend this list with actual profane words)
profanity_list <- c("badword1", "badword2", "badword3")

# Tokenize the text
tokens <- tokenize_text(file_path)

# Filter profanity
clean_tokens <- filter_profanity(tokens, profanity_list)

# Summary statistics
file_path <- "C:/Users/Admin/Documents/en_US/combined_file.txt"
line_count <- length(readLines(file_path, warn = FALSE))
tokens <- tokenize_text(file_path)
clean_tokens <- filter_profanity(tokens, profanity_list)
word_count <- length(clean_tokens)
unique_words <- length(unique(clean_tokens))

summary_stats <- data.frame(
  Metric = c("Line Count", "Word Count", "Unique Words"),
  Value = c(line_count, word_count, unique_words)
)

# Print summary statistics
print(summary_stats)
##         Metric    Value
## 1   Line Count   976547
## 2   Word Count 41051304
## 3 Unique Words   291098

Word Frequency Distribution

# Create a data frame of word frequencies
word_freq <- data.frame(table(clean_tokens))
colnames(word_freq) <- c("word", "freq")
word_freq <- word_freq %>% arrange(desc(freq))

# Plot word frequency distribution
library(ggplot2)
ggplot(word_freq[1:20, ], aes(x = reorder(word, -freq), y = freq)) +
  geom_bar(stat = "identity") +
  xlab("Words") + ylab("Frequency") +
  ggtitle("Top 20 Word Frequencies") +
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

N-gram Analysis

#2-grams

# Create 2-grams and calculate frequencies
bigrams <- unnest_tokens(data_frame(text = clean_tokens), bigram, text, token = "ngrams", n = 2)
## Warning: `data_frame()` was deprecated in tibble 1.1.0.
## ℹ Please use `tibble()` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.
bigram_freq <- bigrams %>% count(bigram, sort = TRUE)

# Plot top 20 2-grams
ggplot(bigram_freq[1:20, ], aes(x = reorder(bigram, -n), y = n)) +
  geom_bar(stat = "identity") +
  xlab("2-grams") + ylab("Frequency") +
  ggtitle("Top 20 2-gram Frequencies") +
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

#3-grams

# Create 3-grams and calculate frequencies
trigrams <- unnest_tokens(data_frame(text = clean_tokens), trigram, text, token = "ngrams", n = 3)
trigram_freq <- trigrams %>% count(trigram, sort = TRUE)

# Plot top 20 3-grams
ggplot(trigram_freq[1:20, ], aes(x = reorder(trigram, -n), y = n)) +
  geom_bar(stat = "identity") +
  xlab("3-grams") + ylab("Frequency") +
  ggtitle("Top 20 3-gram Frequencies") +
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

##Conclusion and Next Steps

In this report, we have provided an overview of our initial exploratory analysis. Our next steps include developing a prediction algorithm and creating a Shiny app to demonstrate its capabilities.