Peer-graded Final Assignment Capstone

2024-03-01

Synopsis

The goal of the project is to build a predictive model for suggesting next word given a text input. In this report we will summarise data exploration for next word prediction model. Dataset consists of corpus in English, German and finnish.We will focus on english text for this project. We will use data from https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip. Data consists of three corpora from blogs, new and twitter sites.

setwd("C:/Users/bfahs/OneDrive/Documents/datasciencecoursera/DF-Capstone")

# Define file paths
news_file <- "final/en_US/en_US.news.txt"
twitter_file <- "final/en_US/en_US.twitter.txt"
blogs_file <- "final/en_US/en_US.blogs.txt"

# Check if files exist
if (!file.exists(news_file) || !file.exists(twitter_file) || !file.exists(blogs_file)) {
  stop("One or more files not found.")
}

# Read the files
newsText <- readLines(news_file, encoding = "UTF-8", warn = FALSE)
twitterText <- readLines(twitter_file, encoding = "UTF-8", warn = FALSE)
blogsText <- readLines(blogs_file, encoding = "UTF-8", warn = FALSE)

Lets check the no of lines, words and size of all three texts:

sizeTwitter <- file.info("en_US.twitter.txt")$size / 1024
sizeNews <- file.info("en_US.news.txt")$size / 1024
sizeblogs <- file.info("en_US.blogs.txt")$size / 1024
wordsTwitter <- stri_count_words(twitterText)
wordsNews <- stri_count_words(newsText)
wordsbolg <- stri_count_words(blogsText)
data.frame(Source = c("Twitter", "News", "Blogs"),
           SizeMB = c(sizeTwitter, sizeNews, sizeblogs),
           NoofLines = c(length(twitterText),length(newsText), length(blogsText)),
           NoOfWords = c(sum(wordsTwitter),sum(wordsNews),sum(wordsbolg)))

##    Source SizeMB NoofLines NoOfWords
## 1 Twitter     NA   2360148  30093372
## 2    News     NA     77259   2674536
## 3   Blogs     NA    899288  37546250

Data sampling

    set.seed(1288)
if (!file.exists("mergedfile.txt")) {
    # If mergedfile.txt doesn't exist, create it or perform alternative operations
    paste("Warning: mergedfile.txt does not exist. Skipping file operations.", "\n")
} else {
    # Read from mergedfile.txt and perform file operations
    conn <- file("mergedfile.txt", "r")
    fulltext <- readLines(conn, encoding = "UTF-8", skipNul = TRUE)
    nlines <- length(fulltext)
    close(conn)
    conn <- file("sampledfile.txt", "w")
    selection <- rbinom(nlines, 1, .1)
    for (i in 1:nlines) {
      if (selection[i] == 1) {
        cat(fulltext[i], file = conn, sep = "\n")
      }
    }
    close(conn)
    paste(
      "Saved",
      sum(selection),
      "lines to file",
      "sampledfile.txt"
    )
}

## [1] "Warning: mergedfile.txt does not exist. Skipping file operations. \n"

mytf3 <- readLines("C:/Users/bfahs/OneDrive/Documents/datasciencecoursera/DF-Capstone/sampledfile.txt", encoding = "UTF-8")
myCorpus <- corpus(mytf3)

Helper methods

getProfanities <- function() {
  profanityFile <- "profanities.txt"
  if (!file.exists(profanityFile)) {
    download.file('http://www.cs.cmu.edu/~biglou/resources/bad-words.txt',
                  profanityFile)
  }
  profanities <-
    read.csv("profanities.txt",
             header = FALSE,
             stringsAsFactors = FALSE)
  profanities$V1
}
makeNgrams <- function(sentences, n = 1L) {
  words <-
    tokens(
     sentences,
      ngrams = n,
      remove_url = TRUE,
      remove_separators = TRUE,
      remove_punct = TRUE,
      remove_twitter = TRUE,
      what = "word",
      remove_hyphens = TRUE,
      remove_numbers = TRUE
    )
  words <-  tokens_remove(words, getProfanities())
}
plotMostFrequentwords <- function(nGrams, title){
    nGramList <-  unlist(nGrams, recursive = FALSE, use.names = FALSE)
  wordFreq <- table(nGramList)
  mostfreqTwoGrams <- as.data.frame(sort(wordFreq,decreasing=TRUE)[1:10])
  ggplot(mostfreqTwoGrams, aes(x= nGramList,y = Freq)) +geom_bar(stat = "identity") +
    theme(axis.text.x = element_text(angle = 90, hjust = 0))  + ggtitle(title)
}

#Data cleanup Any text prediction algorithm has to deal with offensive words. Then we will create n-grams and remove the profanities using tokens_remove feature of quanteda. We will use the list of bad words available at http://www.cs.cmu.edu/~biglou/resources/bad-words.txt

Data cleanup

# Load required libraries
library(quanteda)
# Define utility functions
getProfanities <- function() {
  profanityFile <- "profanities.txt"
  if (!file.exists(profanityFile)) {
    download.file('http://www.cs.cmu.edu/~biglou/resources/bad-words.txt',
                  profanityFile)
  }
  profanities <- read.csv(profanityFile, header = FALSE, stringsAsFactors = FALSE)
  profanities$V1
}
makeNgrams <- function(sentences, n = 1L) {
  words <- tokens(
    sentences,
    remove_url = TRUE,
    remove_separators = TRUE,
    remove_punct = TRUE,
    what = "word",
    remove_numbers = TRUE
  )
  words <- tokens_remove(words, getProfanities())
}
plotMostFrequentwords <- function(nGrams, title){
  nGramList <- unlist(nGrams, recursive = FALSE, use.names = FALSE)
  wordFreq <- table(nGramList)
  mostfreqTwoGrams <- as.data.frame(sort(wordFreq, decreasing=TRUE)[1:10])
  ggplot(mostfreqTwoGrams, aes(x = nGramList, y = Freq)) +
    geom_bar(stat = "identity") +
    theme(axis.text.x = element_text(angle = 90, hjust = 0)) +
    ggtitle(title)
}
# Extract content from each document
document_content <- sapply(myCorpus$content, "[[", "content")

# Tokenize each document's content and remove profanities
all_tokens <- lapply(document_content, function(doc) {
  tokens(doc, remove_punct = TRUE) %>%
    tokens_remove(pattern = getProfanities(), case_insensitive = TRUE)
})
all_sentences <- unlist(all_tokens) # Combine tokenized sentences from all documents
all_sentences <- tolower(all_sentences) # Convert to lowercase
all_sentences_tokens <- tokens(all_sentences) # Combine tokenized sentences into a single tokens object
# Create n-grams
twoGrams <- makeNgrams(all_sentences_tokens, n = 2)
threeGrams <- makeNgrams(all_sentences_tokens, n = 3)
FourGrams <- makeNgrams(all_sentences_tokens, n = 4)
## Exploratory analysis 
# Plot the most frequent n-grams
plotMostFrequentwords <- function(nGrams, title){
  nGramList <- unlist(nGrams, recursive = FALSE, use.names = FALSE)
  wordFreq <- table(nGramList)
  mostfreqTwoGrams <- as.data.frame(sort(wordFreq, decreasing=TRUE)[1:10])
  ggplot(mostfreqTwoGrams, aes(x = nGramList, y = Freq)) +
    geom_bar(stat = "identity") +
    theme(axis.text.x = element_text(angle = 90, hjust = 0)) +
    ggtitle(title)
}