#Q1. Read all .txt files from the folder titled data under Assignment 1, generate a corpus. Print the corpus.
# List all .txt files in the "Assignment1_Data_IMDB_Review" folder
file_list <- list.files("/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review", pattern = "*.txt", full.names = TRUE)
# Read the documents and create a named character vector
texts <- sapply(file_list, function(fname) {
paste(readLines(fname), collapse = "\n")
}, USE.NAMES = TRUE)
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/11_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/12_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/13_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/14_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/15_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/16_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/17_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/18_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/19_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/20_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/21_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/22_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/23_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/24_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/25_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/26_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/27_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/28_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/29_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/30_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/31_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/32_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/33_7.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/34_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/35_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/36_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/37_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/38_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/39_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/40_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/41_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/42_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/43_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/44_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/45_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/46_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/47_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/48_9.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/49_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/50_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/51_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/52_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/53_7.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/54_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/55_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/56_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/57_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/58_8.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/59_10.txt'
## Warning in readLines(fname): incomplete final line found on
## '/Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/60_8.txt'
# Create a corpus
corpus <- corpus(texts)
# Print the corpus
print(corpus)
## Corpus consisting of 50 documents.
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/11_8.txt :
## "I was looking forward to The Guardian, but when I walked int..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/12_9.txt :
## "I was pleasantly surprised to find this movie showing as a s..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/13_9.txt :
## "I work at a movie theater and every Thursday night we have a..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/14_8.txt :
## "I attended an advance screening of this film not sure of wha..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/15_10.txt :
## "As others that have commented around the web... I'm a 130 pi..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/16_8.txt :
## "I went to an advance screening of this movie thinking I was ..."
##
## [ reached max_ndoc ... 44 more documents ]
#Q2. Each document has a name (e.g. 11_8.txt), extract the first two digits from the name, make that integer, and assign that integer as a document variable (docvars) named “ID”. Print the head of the summary of the updated corpus.
# Extract the first two digits from each document name and convert to integer
ids <- sapply(file_list, function(x) {
as.integer(substr(basename(x), 1, 2))
})
texts <- sapply(file_list, function(fname) {
paste(readLines(fname, warn = FALSE), collapse = "\n")
}, USE.NAMES = TRUE)
corpus <- corpus(texts)
# Assign these IDs as a document variable
docvars(corpus, "ID") <- ids
# Print the head of the summary of the updated corpus
corpus
## Corpus consisting of 50 documents and 1 docvar.
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/11_8.txt :
## "I was looking forward to The Guardian, but when I walked int..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/12_9.txt :
## "I was pleasantly surprised to find this movie showing as a s..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/13_9.txt :
## "I work at a movie theater and every Thursday night we have a..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/14_8.txt :
## "I attended an advance screening of this film not sure of wha..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/15_10.txt :
## "As others that have commented around the web... I'm a 130 pi..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/16_8.txt :
## "I went to an advance screening of this movie thinking I was ..."
##
## [ reached max_ndoc ... 44 more documents ]
print(head(summary(corpus)))
## Text Types Tokens
## 1 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/11_8.txt 327 719
## 2 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/12_9.txt 150 279
## 3 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/13_9.txt 107 175
## 4 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/14_8.txt 153 248
## 5 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/15_10.txt 173 386
## 6 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/16_8.txt 136 233
## Sentences ID
## 1 20 11
## 2 7 12
## 3 8 13
## 4 9 14
## 5 11 15
## 6 13 16
#Q3. Keep only those documents with an ID larger than 20. Print the head of the summary of the updated corpus.
# Filter out documents with ID less than or equal to 20
filtered_corpus <- corpus_subset(corpus, ID > 20)
# Print the head of the summary of the updated corpus
filtered_corpus
## Corpus consisting of 40 documents and 1 docvar.
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/21_9.txt :
## ""The Dresser" is a small but absolutely wonderful film, bril..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/22_8.txt :
## "What is worth mentioning that is omitted in the other review..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/23_10.txt :
## "Anyone who appreciates fine acting and ringing dialogue will..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/24_10.txt :
## ""Sir" has played Lear over 200 times,but tonight he can't re..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/25_10.txt :
## "This is a movie that deserves another look--if you haven't s..."
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/26_10.txt :
## "This film has its detractors, and Courtney's fey dresser may..."
##
## [ reached max_ndoc ... 34 more documents ]
print(head(summary(filtered_corpus)))
## Text Types Tokens
## 1 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/21_9.txt 132 227
## 2 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/22_8.txt 217 471
## 3 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/23_10.txt 86 182
## 4 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/24_10.txt 349 692
## 5 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/25_10.txt 143 216
## 6 /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/26_10.txt 85 124
## Sentences ID
## 1 4 21
## 2 5 22
## 3 3 23
## 4 14 24
## 5 9 25
## 6 7 26
#Q4. Tokenize the updated corpus, keep numbers but remove the punctuation. Print the head of the summary of the updated corpus.
# Tokenize the corpus, keeping numbers and removing punctuation
tokens <- tokens(filtered_corpus, remove_numbers = FALSE, remove_punct = TRUE)
# Print the head of the summary of the updated corpus
tokens
## Tokens consisting of 40 documents and 1 docvar.
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/21_9.txt :
## [1] "The" "Dresser" "is" "a" "small"
## [6] "but" "absolutely" "wonderful" "film" "brilliantly"
## [11] "acted" "by"
## [ ... and 186 more ]
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/22_8.txt :
## [1] "What" "is" "worth" "mentioning" "that"
## [6] "is" "omitted" "in" "the" "other"
## [11] "reviews" "I"
## [ ... and 412 more ]
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/23_10.txt :
## [1] "Anyone" "who" "appreciates" "fine" "acting"
## [6] "and" "ringing" "dialogue" "will" "love"
## [11] "<" "br"
## [ ... and 137 more ]
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/24_10.txt :
## [1] "Sir" "has" "played" "Lear" "over" "200"
## [7] "times" "but" "tonight" "he" "can't" "remember"
## [ ... and 588 more ]
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/25_10.txt :
## [1] "This" "is" "a" "movie" "that" "deserves"
## [7] "another" "look--if" "you" "haven't" "seen" "it"
## [ ... and 186 more ]
##
## /Users/wangyinsu/Desktop/Assignment1_Data_IMDB_Review/26_10.txt :
## [1] "This" "film" "has" "its" "detractors"
## [6] "and" "Courtney's" "fey" "dresser" "may"
## [11] "offend" "some"
## [ ... and 84 more ]
##
## [ reached max_ndoc ... 34 more documents ]
#Q5. Combine 'own', 'sir', and 'god' along with stopwords('english') to a vector called mystopwords, then create a data feature matrix (DFM) of the corpus, while removing mystopwords and punctuation. Print the top features of that dfm.
# Combine 'own', 'sir', and 'god' with standard English stopwords
mystopwords <- c("own", "sir", "god", stopwords("english"))
# Create a DFM from these tokens
dfm <- dfm(tokens)
# Remove custom stopwords from the DFM
dfm <- dfm_remove(dfm, pattern = mystopwords)
# Print the top features of the dfm
topfeatures <- topfeatures(dfm, 10) # Adjust the number as needed
print(topfeatures)
## < br > film movie great nanny
## 184 184 184 53 52 39 34
## story one characters
## 33 31 31
#Q6. After creating DFM, try to calculate count, frequency, and tf-idf for the DFM. Print the top features of each type of calculation.
# Calculate count
top_count <- topfeatures(dfm, 10)
# Calculate frequency
dfm_freq <- dfm_weight(dfm, scheme = "prop")
top_freq <- topfeatures(dfm_freq, 10)
# Calculate TF-IDF
dfm_tfidf <- dfm_tfidf(dfm)
top_tfidf <- topfeatures(dfm_tfidf, 10)
# Print the top features
print("Top Count:")
## [1] "Top Count:"
print(top_count)
## < br > film movie great nanny
## 184 184 184 53 52 39 34
## story one characters
## 33 31 31
print("Top Frequency:")
## [1] "Top Frequency:"
print(top_freq)
## < br > film movie great nanny
## 1.5361582 1.5361582 1.5361582 0.4834139 0.4280788 0.3832073 0.3039075
## characters story time
## 0.3016373 0.3010407 0.2317145
print("Top TF-IDF:")
## [1] "Top TF-IDF:"
print(top_tfidf)
## < br > movie film dresser great know
## 31.40811 31.40811 31.40811 16.81193 13.76078 13.24532 12.60895 11.53472
## one story
## 11.51994 11.44399