# Load required libraries
library(readr)
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(ggplot2)
library(tokenizers)
## Warning: package 'tokenizers' was built under R version 4.4.3
library(stringr)
## Warning: package 'stringr' was built under R version 4.4.3
library(tibble)
library(knitr)

# Global chunk options
knitr::opts_chunk$set(
  echo = TRUE,
  warning = FALSE,
  message = FALSE
)

Introduction

This report demonstrates that the English SwiftKey training data have been successfully downloaded and loaded. We provide concise summary statistics, tables, and plots to highlight key characteristics of each dataset. Finally, we outline our plan for building the predictive text algorithm and Shiny app.

Data Overview

First, we inspect file sizes and line counts for the three corpora.

# Define file paths
dir <- "C:/Users/user/Desktop/Coursera-SwiftKey/final/en_US"
files <- c(
  blogs   = file.path(dir, "en_US.blogs.txt"),
  news    = file.path(dir, "en_US.news.txt"),
  twitter = file.path(dir, "en_US.twitter.txt")
)

# Compute sizes (in MB)
sizes_mb <- sapply(files, function(f) file.info(f)$size / 1024^2) %>% round(1)

# Function to count lines
count_lines <- function(f) {
  con <- file(f, "r"); on.exit(close(con))
  total <- 0L
  repeat {
    lines <- readLines(con, n = 10000, warn = FALSE)
    if (!length(lines)) break
    total <- total + length(lines)
  }
  total
}

# Compute line counts
lines_count <- sapply(files, count_lines)

overview <- tibble(
  Dataset = names(files),
  Size_MB = sizes_mb,
  Lines   = lines_count
)

# Display table
knitr::kable(overview, caption = "File sizes and line counts for each corpus")
File sizes and line counts for each corpus
Dataset Size_MB Lines
blogs 200.4 899288
news 196.3 77259
twitter 159.4 2360148

Sampled Data Statistics

We sampled 1% of blogs and Twitter, and 5% of news, for quick iteration. Below are average and median line lengths.

# Read sampled data
sample_dir <- "C:/Users/user/Desktop/Coursera-SwiftKey/final/sample"
data_files <- list.files(sample_dir, pattern = "\\.txt$", full.names = TRUE)
data_list  <- lapply(data_files, function(f) {
  raw <- readLines(f, skipNul = TRUE, warn = FALSE, encoding = "UTF-8")
  iconv(raw, "UTF-8", "UTF-8", sub = "")
})
names(data_list) <- basename(data_files)

# Compute sample statistics
stats <- tibble(
  Source        = names(data_list),
  Lines         = sapply(data_list, length),
  Avg_Length    = sapply(data_list, function(x) mean(nchar(x))),
  Median_Length = sapply(data_list, function(x) median(nchar(x)))
)

knitr::kable(stats, caption = "Sampled line counts and lengths")
Sampled line counts and lengths
Source Lines Avg_Length Median_Length
en_US.blogs.txt 9003 232.28868 158
en_US.news.txt 3821 200.06674 182
en_US.twitter.txt 23488 68.34967 64

Line Length Distribution

all_lines <- unlist(data_list)
df_lengths <- tibble(Length = nchar(all_lines))

ggplot(df_lengths, aes(x = Length)) +
  geom_histogram(bins = 50) +
  labs(
    title = "Distribution of Sampled Line Lengths",
    x     = "Characters per line",
    y     = "Frequency"
  ) +
  theme_minimal()

Lines per Data Source

 ggplot(stats, aes(x = Source, y = Lines)) +
  geom_col(fill = "steelblue") +
  labs(
    title = "Sampled Lines per Source",
    x     = "Dataset",
    y     = "Number of Lines"
  ) +
  theme_minimal()

Top 20 Most Frequent Words

# Load combined unigram counts
processed_dir <- "C:/Users/user/Desktop/Coursera-SwiftKey/final/processed"
all_unigrams  <- readRDS(file.path(processed_dir, "all_unigrams.rds"))
top_tokens    <- head(all_unigrams, 20)

knitr::kable(top_tokens, caption = "Top 20 most frequent words across all samples")
Top 20 most frequent words across all samples
token count
the 35567
to 22129
and 18867
a 18317
i 16303
of 15346
in 12127
is 9088
for 8826
you 8802
that 8451
it 8158
on 6454
my 5830
with 5673
this 4828
was 4804
be 4535
at 4371
have 4271 
ggplot(top_tokens, aes(x = reorder(token, count), y = count)) +
  geom_col(fill = "steelblue") +
  coord_flip() +
  labs(
    title = "Top 20 Most Frequent Words",
    x     = NULL,
    y     = "Frequency"
  ) +
  theme_minimal()

Next Steps

  1. Build n-gram models (uni/bi/tri-grams) with smoothing.

  2. Evaluate model performance via perplexity on held-out data.

  3. Develop a Shiny app for interactive next-word prediction.

  4. Prepare a 5-slide pitch summarizing approach, results, and roadmap.

This concise analysis confirms we’re ready to proceed to model building and app development. Feedback is welcome!