Milestone report 1

Author

Rodolfo León

Report.-

The table below reflects the number of lines and words in the three analyzed tables, as well as the mode of the most frequent word in each file. Important to note that before the count all lines were stripped of symbols, digits and punctuation, as well as of “words” consisting of a combination of these with letters. The intent is to focus the analysis only on natural language words. The three histograms plotting the number of words per line vs the number of times that number of words is repeated are strongly skewed to the right, suggesting that many lines are blank or contain characters other than actual words.

Analysis of text files
Text file	Number of lines	Number of Words	Mode of words
en_US.news.txt	77,259	69,415	17
en_US.twitter.txt	2’360,148	240,169	4
en_US.blogs.txt	899,288	238,565	2

I. File en_US.news.txt

# Open the file connection
con <- file("C:/Users/rodol/OneDrive/Coursera/Johns Hopkins Data Science Specialization/10. Capstone/Coursera-SwiftKey/final/en_US/en_US.news.txt", "r")

# Read the entire file, including any potentially incomplete final line
text_data <- readLines(con, warn = FALSE)

# Close the file connection
close(con)

# Create a corpus from the text data
corpus <- corpus(text_data)

# # Tokenize the corpus into words
tokens <- tokens(corpus)

# Line counts (number of documents)
line_counts <- ndoc(corpus)
cat("Line counts:", line_counts)

Line counts: 77259

# Tokenize the text data into words, removing punctuation, numbers, and symbols
tokens_clean <- tokens(text_data, 
                       remove_punct = TRUE,   # Remove punctuation
                       remove_numbers = TRUE, # Remove numbers
                       remove_symbols = TRUE) # Remove symbols

tokens_clean <- tokens_remove(tokens_clean, stopwords("en"))
# Remove tokens that contain a mix of letters, symbols, and digits
tokens_clean <- tokens_select(tokens_clean, pattern = "^[A-Za-z]+$", selection = "keep", valuetype = "regex")
# Count the number of words per line
words_per_line <- lengths(tokens_clean)
#cat("Words per line:", words_per_line)

# Plot the histogram of word counts per line
ggplot(data.frame(words_per_line = words_per_line), aes(x = words_per_line)) +
  geom_histogram(binwidth = 10, fill = "lightblue", color = "cadetblue") +
  labs(title = "Histogram of Word Counts per Line", x = "Number of Words", y = "Frequency") +
  theme_minimal()

# Print count by words
# Create a document-feature matrix (DFM)
dfm_clean <- dfm(tokens_clean)

# Calculate the total word counts across all documents
word_counts_clean <- colSums(dfm_clean)

# Convert the word counts to a data frame
df_word_counts_clean <- data.frame(
  word = names(word_counts_clean),
  count = as.numeric(word_counts_clean)
)
# Sort the data frame by word in alphabetical order
df_word_counts_clean_sorted_alpha <- df_word_counts_clean %>%
  arrange(word)

# Print the sorted data frame
cat("Total number of words:", nrow(df_word_counts_clean_sorted_alpha))

Total number of words: 69415

# Sort the data frame by count in descending order
df_word_counts_clean_sorted <- df_word_counts_clean %>%
  arrange(desc(count))

# Print the top 100 words
print(head(df_word_counts_clean_sorted, 10))

    word count
1   said 19170
2    one  6412
3    new  5336
4   also  4515
5    two  4439
6    can  4408
7   year  4242
8  first  4152
9   just  4144
10  last  4027

print(tail(df_word_counts_clean_sorted, 10))

           word count
69406 crucially     1
69407      kach     1
69408 margolius     1
69409   pacumio     1
69410      wcll     1
69411    krizek     1
69412    knecht     1
69413 confronts     1
69414   tenders     1
69415    hunton     1

# Create a table of word counts per line
word_count_table <- table(words_per_line)

# Find the mode (most frequent word count)
mode_word_count <- as.numeric(names(word_count_table)[which.max(word_count_table)])

#Print the word_count_table
cat("Words per line:", word_count_table)

Words per line: 165 952 2400 2188 2013 1858 1929 2150 2207 2363 2517 2602 2762 2886 2856 2897 2910 2935 2852 2753 2589 2550 2378 2160 1998 1940 1677 1644 1444 1360 1140 1063 965 875 806 659 600 532 498 409 375 321 260 272 245 234 177 163 153 114 111 126 119 63 68 76 75 62 48 54 43 40 43 36 38 31 32 27 31 20 23 22 8 19 15 20 14 6 13 17 8 14 10 8 7 6 4 6 6 4 3 8 3 3 1 7 3 3 2 1 1 4 4 3 2 3 2 2 1 2 2 4 2 2 2 1 1 1 1 1 1 1 1 3 1 1 2 1 1 1 1 1

# Print the mode
cat("\nMode of words per line:", mode_word_count)


Mode of words per line: 17

II. File en_US.twitter.txt

# Open the file connection
con <- file("C:/Users/rodol/OneDrive/Coursera/Johns Hopkins Data Science Specialization/10. Capstone/Coursera-SwiftKey/final/en_US/en_US.twitter.txt", "r")

# Read the entire file, including any potentially incomplete final line
text_data <- readLines(con, warn = FALSE)

# Close the file connection
close(con)

# Create a corpus from the text data
corpus <- corpus(text_data)

# # Tokenize the corpus into words
tokens <- tokens(corpus)

# Line counts (number of documents)
line_counts <- ndoc(corpus)
cat("Twitter line counts:", line_counts)

Twitter line counts: 2360148

# Tokenize the text data into words, removing punctuation, numbers, and symbols
tokens_clean <- tokens(text_data, 
                       remove_punct = TRUE,   # Remove punctuation
                       remove_numbers = TRUE, # Remove numbers
                       remove_symbols = TRUE) # Remove symbols

tokens_clean <- tokens_remove(tokens_clean, stopwords("en"))
# Remove tokens that contain a mix of letters, symbols, and digits
tokens_clean <- tokens_select(tokens_clean, pattern = "^[A-Za-z]+$", selection = "keep", valuetype = "regex")
# Count the number of words per line
words_per_line <- lengths(tokens_clean)
#print(words_per_line)

# Plot the histogram of word counts per line
ggplot(data.frame(words_per_line = words_per_line), aes(x = words_per_line)) +
  geom_histogram(binwidth = 1, fill = "lightblue", color = "cadetblue") +
  labs(title = "Histogram of Word Counts per Line", x = "Number of Words", y = "Frequency") +
  theme_minimal()

# Print count by words
# Create a document-feature matrix (DFM)
dfm_clean <- dfm(tokens_clean)

# Calculate the total word counts across all documents
word_counts_clean <- colSums(dfm_clean)

# Convert the word counts to a data frame
df_word_counts_clean <- data.frame(
  word = names(word_counts_clean),
  count = as.numeric(word_counts_clean)
)
# Sort the data frame by word in alphabetical order
df_word_counts_clean_sorted_alpha <- df_word_counts_clean %>%
  arrange(word)

# Print the sorted data frame
cat("Total number of words:", nrow(df_word_counts_clean_sorted_alpha))

Total number of words: 240169

# Sort the data frame by count in descending order
df_word_counts_clean_sorted <- df_word_counts_clean %>%
  arrange(desc(count))

# Print the top 100 words
print(head(df_word_counts_clean_sorted, 10))

     word  count
1    just 151020
2    like 122026
3     get 112320
4    love 106190
5    good 100763
6     day  90121
7     can  89736
8  thanks  89536
9      rt  89371
10    now  83817

print(tail(df_word_counts_clean_sorted, 10))

                   word count
240160            osuna     1
240161       eeeeepiiee     1
240162        sanginary     1
240163         hellions     1
240164           beleib     1
240165 elasticbeanbeard     1
240166              ocm     1
240167       oooohhhhhh     1
240168          nandina     1
240169       bailjumper     1

# Create a table of word counts per line
word_count_table <- table(words_per_line)

# Find the mode (most frequent word count)
mode_word_count <- as.numeric(names(word_count_table)[which.max(word_count_table)])

#Print the word_count_table
cat("Words per line:", word_count_table)

Words per line: 10270 78122 207889 238745 238828 226133 211708 193855 179122 164693 151168 134183 111681 84624 58081 35359 19118 9215 4148 1780 781 327 148 79 41 16 10 7 9 3 1 1 1 1 1

# Print the mode
cat("\nMode of words per line:", mode_word_count)


Mode of words per line: 4

III. File en_US.blogs.txt

# Open the file connection
con <- file("C:/Users/rodol/OneDrive/Coursera/Johns Hopkins Data Science Specialization/10. Capstone/Coursera-SwiftKey/final/en_US/en_US.blogs.txt", "r")

# Read the entire file, including any potentially incomplete final line
text_data <- readLines(con, warn = FALSE)

# Close the file connection
close(con)

# Create a corpus from the text data
corpus <- corpus(text_data)

# # Tokenize the corpus into words
tokens <- tokens(corpus)

# Line counts (number of documents)
line_counts <- ndoc(corpus)
cat("Line counts:", line_counts)

Line counts: 899288

# Tokenize the text data into words, removing punctuation, numbers, and symbols
tokens_clean <- tokens(text_data, 
                       remove_punct = TRUE,   # Remove punctuation
                       remove_numbers = TRUE, # Remove numbers
                       remove_symbols = TRUE) # Remove symbols

tokens_clean <- tokens_remove(tokens_clean, stopwords("en"))
# Remove tokens that contain a mix of letters, symbols, and digits
tokens_clean <- tokens_select(tokens_clean, pattern = "^[A-Za-z]+$", selection = "keep", valuetype = "regex")
# Count the number of words per line
words_per_line <- lengths(tokens_clean)
#print(words_per_line)

# Plot the histogram of word counts per line
ggplot(data.frame(words_per_line = words_per_line), aes(x = words_per_line)) +
  geom_histogram(binwidth = 50, fill = "lightblue", color = "cadetblue") +
  labs(title = "Histogram of Word Counts per Line", x = "Number of Words", y = "Frequency") +
  theme_minimal()

# Print count by words
# Create a document-feature matrix (DFM)
dfm_clean <- dfm(tokens_clean)

# Calculate the total word counts across all documents
word_counts_clean <- colSums(dfm_clean)

# Convert the word counts to a data frame
df_word_counts_clean <- data.frame(
  word = names(word_counts_clean),
  count = as.numeric(word_counts_clean)
)
# Sort the data frame by word in alphabetical order
df_word_counts_clean_sorted_alpha <- df_word_counts_clean %>%
  arrange(word)

# Print the sorted data frame
cat("Total number of words:", nrow(df_word_counts_clean_sorted_alpha))

Total number of words: 238565

# Sort the data frame by count in descending order
df_word_counts_clean_sorted <- df_word_counts_clean %>%
  arrange(desc(count))

# Print the top 100 words
print(head(df_word_counts_clean_sorted, 10))

     word  count
1     one 124743
2    just 100593
3    like  98644
4     can  98283
5    time  88562
6     get  70866
7    know  60260
8     now  60114
9  people  59410
10   also  55331

print(tail(df_word_counts_clean_sorted, 10))

              word count
238556 infrequency     1
238557      kojaks     1
238558      muskie     1
238559   sapientia     1
238560     draumur     1
238561  sogblettir     1
238562       bunds     1
238563   megablond     1
238564     thesame     1
238565     dashkov     1

# Create a table of word counts per line
word_count_table <- table(words_per_line)

# Find the mode (most frequent word count)
mode_word_count <- as.numeric(names(word_count_table)[which.max(word_count_table)])

#Print the word_count_table
# cat("Words per line:", word_count_table)
# Print the mode
cat("\nMode of words per line:", mode_word_count)


Mode of words per line: 2

Report.-

Text file

Number of lines

Number of Words

Mode of words

I. File en_US.news.txt

II. File en_US.twitter.txt

III. File en_US.blogs.txt