packages <- c(“ggplot2”, “dplyr”, “wordcloud”, “RColorBrewer”, “tm”, “knitr”) for (pkg in packages) { if (!require(pkg, character.only = TRUE)) { install.packages(pkg, dependencies = TRUE) library(pkg, character.only = TRUE) } }
cat(“📂 Reading data…”)
blogs_path <- “final/en_US/en_US.blogs.txt” news_path <- “final/en_US/en_US.news.txt” twitter_path <- “final/en_US/en_US.twitter.txt”
if (!file.exists(blogs_path)) stop(“Blogs file not found. Please check path.”) if (!file.exists(news_path)) stop(“News file not found. Please check path.”) if (!file.exists(twitter_path)) stop(“Twitter file not found. Please check path.”)
set.seed(123) sample_size <- 5000
cat(” Reading blogs…“) blogs <- readLines(blogs_path, n = sample_size, encoding =”UTF-8”, skipNul = TRUE)
cat(” Reading news…“) con <- file(news_path, open =”rb”) news <- readLines(con, n = sample_size, encoding = “UTF-8”, skipNul = TRUE) close(con)
cat(” Reading twitter…“) twitter <- readLines(twitter_path, n = sample_size, encoding =”UTF-8”, skipNul = TRUE)
cat(“✅ Data loaded successfully!”) cat(” Blogs:“, length(blogs),”lines“) cat(” News:“, length(news),”lines“) cat(” Twitter:“, length(twitter),”lines“)
cat(“📊 Statistical analysis…”)
get_stats <- function(text, name) { text <- text[text != “” & !is.na(text)]
if (length(text) == 0) { return(data.frame( Dataset = name, Lines = 0, Words = 0, Characters = 0, Avg_Words = 0, Max_Line = 0 )) }
words <- unlist(strsplit(text, “\s+”)) words <- words[words != “”]
data.frame( Dataset = name, Lines = length(text), Words = length(words), Characters = sum(nchar(text)), Avg_Words = round(length(words) / length(text), 2), Max_Line = max(nchar(text)) ) }
stats_all <- rbind( get_stats(blogs, “Blogs”), get_stats(news, “News”), get_stats(twitter, “Twitter”) )
print(stats_all) cat(“”)
cat(“📋 Full Dataset Statistics”) cat(“—————————————-”)
full_stats <- data.frame( Dataset = c(“Blogs”, “News”, “Twitter”), File_Size_MB = c(200, 200, 150), Total_Lines = c(899288, 1010242, 2360148), Max_Line_Length = c(40833, 11385, 140) )
print(full_stats) cat(“”)
cat(“🎨 Generating plots…”)
blogs_len <- nchar(blogs) news_len <- nchar(news) twitter_len <- nchar(twitter)
lengths_df <- data.frame( Length = c(blogs_len, news_len, twitter_len), Dataset = c( rep(“Blogs”, length(blogs_len)), rep(“News”, length(news_len)), rep(“Twitter”, length(twitter_len)) ) )
lengths_df <- lengths_df[lengths_df$Length <= 1000, ]
p1 <- ggplot(lengths_df, aes(x = Length, fill = Dataset)) + geom_density(alpha = 0.6) + labs( title = “Line Length Distribution Density Plot”, subtitle = paste(“Based on”, sample_size, “line sample”), x = “Line Length (characters)”, y = “Density” ) + theme_minimal() + theme(legend.position = “top”)
print(p1)
p2 <- ggplot(lengths_df, aes(x = Dataset, y = Length, fill = Dataset)) + geom_boxplot() + coord_cartesian(ylim = c(0, 300)) + labs( title = “Line Length Distribution Boxplot”, subtitle = paste(“Based on”, sample_size, “line sample”), x = “Dataset”, y = “Line Length (characters)” ) + theme_minimal() + theme(legend.position = “none”)
print(p2)
cat(” Analyzing word frequency…“)
clean_text <- function(text) { text <- tolower(text) text <- gsub(“[^a-z\\s]”, ” “, text) text <- gsub(”\s+“,” “, text) text <- trimws(text) return(text) }
get_words <- function(text, max_lines = 2000) { if (length(text) > max_lines) { text <- text[1:max_lines] } text <- clean_text(text) words <- unlist(strsplit(text, “\s+”)) words <- words[words != “” & nchar(words) > 1] return(words) }
all_words <- c( get_words(blogs, 2000), get_words(news, 2000), get_words(twitter, 2000) )
word_freq <- table(all_words) top_words <- head(sort(word_freq, decreasing = TRUE), 20)
top_df <- data.frame( Word = names(top_words), Frequency = as.numeric(top_words) )
p3 <- ggplot(top_df, aes(x = reorder(Word, -Frequency), y = Frequency)) + geom_bar(stat = “identity”, fill = “steelblue”) + labs( title = “Top 20 Most Frequent Words”, x = “Word”, y = “Frequency” ) + theme_minimal() + theme(axis.text.x = element_text(angle = 45, hjust = 1))
print(p3)
cat(” Generating word cloud…“)
set.seed(123) wordcloud( names(word_freq), as.numeric(word_freq), max.words = 100, colors = brewer.pal(8, “Dark2”), scale = c(3, 0.5) )
cat(“🔍 Interesting Findings”) cat(“========================================”)
love_count <- sum(grepl(“love”, twitter, ignore.case = TRUE)) hate_count <- sum(grepl(“hate”, twitter, ignore.case = TRUE)) ratio <- round(love_count / hate_count, 2)
cat(“1. Sentiment Analysis in Twitter”) cat(” - Lines containing ‘love’:“, love_count,”“) cat(” - Lines containing ‘hate’:“, hate_count,”“) cat(” - love/hate ratio:“, ratio,”“) cat(” - Interpretation: ‘love’ appears”, ratio, “times more than ‘hate’”)
biostats_lines <- twitter[grepl(“biostats”, twitter, ignore.case = TRUE)]
cat(“2. Tweets containing ‘biostats’”) if (length(biostats_lines) > 0) { cat(” - Found”, length(biostats_lines), “tweet(s)”) cat(” - Content:“, biostats_lines[1],”“) } else { cat(” - No matching tweets found in sample“) } cat(”“)
exact_match <- twitter[ twitter == “A computer once beat me at chess, but it was no match for me at kickboxing”]
cat(“3. Exact Match Tweets”) cat(” - Number of matches:“, length(exact_match),”“) if (length(exact_match) > 0) { cat(” - Content:“, exact_match,”“) } cat(”“)
all_text <- c(blogs, news, twitter) max_len <- max(nchar(all_text), na.rm = TRUE) max_idx <- which.max(nchar(all_text)) max_text <- all_text[max_idx]
cat(“4. Longest Line”) cat(” - Length:“, max_len,”characters“) cat(” - Source:“, ifelse(max_idx <= length(blogs),”Blogs”, ifelse(max_idx <= length(blogs) + length(news), “News”, “Twitter”)), “”) cat(” - Preview:“, substr(max_text, 1, 100),”…“)
twitter_140 <- sum(nchar(twitter) <= 140, na.rm = TRUE) twitter_140_pct <- round(twitter_140 / length(twitter) * 100, 2)
cat(“. Twitter 140-Character Limit”) cat(” - Lines within limit:“, twitter_140,”“) cat(” - Percentage:“, twitter_140_pct,”%“)
cat(“📝 Algorithm Plan”) cat(“========================================”) cat(“: N-gram Language Model + Stupid Backoff”) cat(“Steps:”) cat(” 1. Data Preprocessing“) cat(” - Convert to lowercase“) cat(” - Remove punctuation and numbers“) cat(” - Remove extra whitespace and special characters“) cat(”. Generate N-grams“) cat(” - 1-gram (unigram)“) cat(” - 2-gram (bigram)“) cat(” - 3-gram (trigram)“) cat(” - 4-gram (fourgram)“) cat(”. Prediction Strategy“) cat(” - Input text → Take last 3 words“) cat(” - Search in 4-gram → Return if found“) cat(” - If not found → Backoff to 3-gram“) cat(” - If still not found → Backoff to 2-gram“) cat(” - Final fallback → Return most frequent 1-gram“) cat(”. Output“) cat(” - Return top 3-5 most likely candidate words“)
cat(“📱 Shiny App Plan”) cat(“========================================”) cat(“Features:”) cat(” • User input text box“) cat(” • Real-time next word prediction“) cat(” • Display top 3-5 candidate words“) cat(” • Show prediction confidence scores“) cat(” • User feedback functionality“) cat(”:“) cat(” +——————————————+“) cat(” | Smart Text Prediction Shiny App |“) cat(” +——————————————+“) cat(” | Input Text: [________________________] |“) cat(” | Predictions: [word1] [word2] [word3] |“) cat(” | History: … |“) cat(” +——————————————+“) cat(”Optimization:“) cat(” • Use data.table for fast queries“) cat(” • Preload models on startup“) cat(” • Implement caching mechanism“) cat(” • Asynchronous processing“)
cat(“💾 Saving report…”)
report <- paste0( “========================================”, “SwiftKey Text Prediction - EDA Report”, “========================================”, “Generated:”, Sys.time(), “”,
“1. Data Summary”, “—————————————-” )
for (i in 1:nrow(stats_all)) { report <- paste0(report, stats_all\(Dataset[i], " (Sample):\n", " - Lines: ", stats_all\)Lines[i], “”, ” - Total Words: “, stats_all\(Words[i], "\n", " - Total Characters: ", stats_all\)Characters[i],”“,” - Avg Words per Line: “, stats_all\(Avg_Words[i], "\n", " - Max Line Length: ", stats_all\)Max_Line[i],”” ) }
report <- paste0(report, “Full Dataset:”, ” - Blogs: “, format(full_stats[1, ”Total_Lines”], big.mark =”,“),” lines“,” - News: “, format(full_stats[2, ”Total_Lines”], big.mark =”,“),” lines“,” - Twitter: “, format(full_stats[3, ”Total_Lines”], big.mark =”,“),” lines“,
“2. Key Findings”, “—————————————-”, ” • love/hate ratio (Twitter): “, ratio,”“,” • Longest line length: “, max_len,” characters“,” • Twitter 140-char compliance: “, twitter_140_pct,”%“,” • Top 5 words: “, paste(top_df$Word[1:5], collapse =”, “),”“,
“3. Algorithm Plan”, “—————————————-”, ” Method: N-gram + Stupid Backoff“,” Start with 4-gram, backoff to 3-gram, then 2-gram“,” Final fallback: most frequent 1-gram“,
“4. Shiny App Plan”, “—————————————-”, ” User input text → Real-time prediction → Display top 3 candidates“,” Simple and intuitive interface for non-technical users” )
writeLines(report, “EDA_Report.txt”) cat(“✅ Report saved to: EDA_Report.txt”)
cat(“✅ Analysis Complete!”) cat(“========================================”) cat(“Generated Files:”) cat(” - EDA_Report.txt (Text report)“) cat(” - Plots displayed in RStudio“) cat(”📝 Next Steps:“) cat(” 1. Copy code to R Markdown file“) cat(” 2. Click Knit to generate HTML report“) cat(” 3. Publish to RPubs“) cat(” 4. Submit link to course“)