Introduction

This report presents exploratory analysis of the SwiftKey dataset. The goal is to understand the data and plan for building a text prediction algorithm and Shiny app.

Data Loading and Summary

# Load required library
library(stringi)

# Read text files
blogs <- readLines("C:/Users/shiva/OneDrive/Desktop/SwiftKeyFiles_OLD/en_US.blogs.txt", encoding = "UTF-8", skipNul = TRUE)
news <- readLines("C:/Users/shiva/OneDrive/Desktop/SwiftKeyFiles_OLD/en_US.news.txt", encoding = "UTF-8", skipNul = TRUE)
twitter <- readLines("C:/Users/shiva/OneDrive/Desktop/SwiftKeyFiles_OLD/en_US.twitter.txt", encoding = "UTF-8", skipNul = TRUE)

# Summary table
data_summary <- data.frame(
  File = c("Blogs", "News", "Twitter"),
  Lines = c(length(blogs), length(news), length(twitter)),
  Words = c(
    sum(stri_count_words(blogs)),
    sum(stri_count_words(news)),
    sum(stri_count_words(twitter))
  )
)

data_summary
##      File   Lines    Words
## 1   Blogs  899288 37546806
## 2    News 1010206 34761151
## 3 Twitter 2360148 30096690
library(tm)
library(ggplot2)

# Sample from Twitter data
sample_data <- sample(twitter, 10000)

# Clean and preprocess
corpus <- Corpus(VectorSource(sample_data))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeWords, stopwords("en"))

# Create term-document matrix
dtm <- TermDocumentMatrix(corpus)
m <- as.matrix(dtm)
word_freqs <- sort(rowSums(m), decreasing = TRUE)
top_words <- data.frame(word = names(word_freqs), freq = word_freqs)

# Plot
ggplot(top_words[1:10,], aes(x = reorder(word, -freq), y = freq)) +
  geom_bar(stat = "identity", fill = "steelblue") +
  theme_minimal() +
  xlab("Words") + ylab("Frequency") +
  ggtitle("Top 10 Frequent Words in Twitter Sample")

line_lengths <- nchar(twitter)
hist(line_lengths,
     main = "Line Lengths in Twitter Dataset",
     xlab = "Characters per line",
     col = "lightblue",
     border = "white")