# Paths to the positive and negative reviews
pos_train_path <- file.path(data_path2, "pos")
neg_train_path <- file.path(data_path2, "neg")
pos_test_path <- file.path(data_path1, "pos")
neg_test_path <- file.path(data_path1, "neg")

# Read the reviews
pos_train_reviews <- read_reviews(pos_train_path, "positive")
neg_train_reviews <- read_reviews(neg_train_path, "negative")
pos_test_reviews <- read_reviews(pos_test_path, "positive")
neg_test_reviews <- read_reviews(neg_test_path, "negative")

# Combine all reviews into a single dataset
reviews <- bind_rows(pos_train_reviews, neg_train_reviews, pos_test_reviews, neg_test_reviews)

# Save to CSV
write_csv(reviews, "IMDB_Dataset.csv")
# Load the dataset
reviews <- read_csv("IMDB_Dataset.csv")
## Rows: 50000 Columns: 2
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (2): review, sentiment
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
# Display the structure of the dataset
str(reviews)
## spc_tbl_ [50,000 × 2] (S3: spec_tbl_df/tbl_df/tbl/data.frame)
##  $ review   : chr [1:50000] "Bromwell High is a cartoon comedy. It ran at the same time as some other programs about school life, such as \""| __truncated__ "If you like adult comedy cartoons, like South Park, then this is nearly a similar format about the small advent"| __truncated__ "I'm a male, not given to women's movies, but this is really a well done special story. I have no personal love "| __truncated__ "Scott Bartlett's 'OffOn' is nine minutes of pure craziness. It is a full-frontal assault of psychedelic, pulsat"| __truncated__ ...
##  $ sentiment: chr [1:50000] "positive" "positive" "positive" "positive" ...
##  - attr(*, "spec")=
##   .. cols(
##   ..   review = col_character(),
##   ..   sentiment = col_character()
##   .. )
##  - attr(*, "problems")=<externalptr>
# Plot sentiment distribution
reviews %>%
  count(sentiment) %>%
  ggplot(aes(x = sentiment, y = n, fill = sentiment)) +
  geom_col(show.legend = FALSE) +
  labs(title = "Distribution of Sentiments", x = "Sentiment", y = "Count")

# Tokenize and remove stop words
data("stop_words")
reviews_tokens <- reviews %>%
  unnest_tokens(word, review) %>%
  anti_join(stop_words)
## Joining with `by = join_by(word)`
# Calculate term frequency
term_freq <- reviews_tokens %>%
  count(word, sort = TRUE) %>%
  filter(n > 1000)

# Plot term frequency
ggplot(term_freq, aes(x = reorder(word, n), y = n)) +
  geom_col(show.legend = FALSE) +
  coord_flip() +
  labs(title = "Top Words in IMDB Reviews", x = "Words", y = "Frequency")

# Tokenize into Bigrams
bigrams <- reviews %>%
  unnest_tokens(bigram, review, token = "ngrams", n = 2)

# Separate Bigrams
bigrams_separated <- bigrams %>%
  separate(bigram, c("word1", "word2"), sep = " ")

# Load stop words
data("stop_words")

# Filter out stop words from bigrams
bigrams_filtered <- bigrams_separated %>%
  filter(!word1 %in% stop_words$word, !word2 %in% stop_words$word)

# Count bigrams
bigram_counts <- bigrams_filtered %>%
  count(word1, word2, sort = TRUE) %>%
  filter(n > 4)

# Print the first few rows of the bigram counts to verify
print(head(bigram_counts))
## # A tibble: 6 × 3
##   word1   word2        n
##   <chr>   <chr>    <int>
## 1 br      br      101039
## 2 special effects   2240
## 3 movie   br        1969
## 4 low     budget    1812
## 5 film    br        1757
## 6 sci     fi        1384
# Visualize Bigram Network
bigram_graph <- bigram_counts %>%
  graph_from_data_frame()

ggraph(bigram_graph, layout = "fr") +
  geom_edge_link(aes(edge_alpha = n), show.legend = FALSE) +
  geom_node_point(color = "lightblue", size = 5) +
  geom_node_text(aes(label = name), vjust = 1.8, hjust = 1.8) +
  theme_void() +
  labs(title = "Bigram Network of IMDB Reviews")

# Save the environment
save.image("~/Downloads/week2.RData")
library(topicmodels)

dtm <- DocumentTermMatrix(Corpus(VectorSource(reviews$review)))
lda <- LDA(dtm, k = 5, control = list(seed = 1234))
topics <- tidy(lda, matrix = "beta")

top_terms <- topics %>%
  group_by(topic) %>%
  slice_max(beta, n = 10) %>%
  ungroup() %>%
  arrange(topic, -beta)

top_terms %>%
  mutate(term = reorder_within(term, beta, topic)) %>%
  ggplot(aes(term, beta, fill = factor(topic))) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~ topic, scales = "free") +
  coord_flip() +
  scale_x_reordered() +
  labs(title = "Top Terms in Each Topic",
       x = "Terms",
       y = "Beta")

library(syuzhet)

reviews$sentiment_score <- get_sentiment(reviews$review, method = "syuzhet")

ggplot(reviews, aes(x = sentiment, y = sentiment_score, fill = sentiment)) +
  geom_boxplot() +
  labs(title = "Sentiment Score Distribution",
       x = "Sentiment",
       y = "Sentiment Score")

library(wordcloud)

pos_reviews <- reviews %>% filter(sentiment == "positive")
neg_reviews <- reviews %>% filter(sentiment == "negative")

wordcloud(pos_reviews$review, max.words = 100, colors = brewer.pal(8, "Dark2"))
## Warning in tm_map.SimpleCorpus(corpus, tm::removePunctuation): transformation
## drops documents
## Warning in tm_map.SimpleCorpus(corpus, function(x) tm::removeWords(x,
## tm::stopwords())): transformation drops documents

wordcloud(neg_reviews$review, max.words = 100, colors = brewer.pal(8, "Set1"))
## Warning in tm_map.SimpleCorpus(corpus, tm::removePunctuation): transformation
## drops documents

## Warning in tm_map.SimpleCorpus(corpus, tm::removePunctuation): transformation
## drops documents

In this project, I aim to analyze the sentiment of IMDB movie reviews. I will explore the distribution of sentiments, frequently used terms, bigrams, topic modeling, sentiment scores, and visualize the data using various charts. The data for this analysis comes from the IMDB movie review dataset, which is publicly available and widely used for sentiment analysis tasks.

The dataset consists of movie reviews collected from the IMDB website. Each review is labeled as either “positive” or “negative”. The data is organized into training and test sets, with separate directories for positive and negative reviews. The dataset is structured as follows: ·Training Data: Contains positive and negative reviews used to train the model. ·Test Data: Contains positive and negative reviews used to test the model. The data was combined into a single dataset for analysis and saved as a CSV file.

1.Data Cleaning and Preprocessing The data cleaning and preprocessing steps include reading the reviews from directories, combining them into a single dataset, handling duplicated columns, and removing missing values.

2.Sentiment Distribution The first step in my analysis is to visualize the distribution of sentiments in the dataset. This is done using a bar chart.I chose a bar chart to clearly show the count of positive and negative reviews, making it easy to compare the two sentiments.

3.Term Frequency Analysis Next, I tokenize the reviews, remove stop words, and calculate the term frequency. This helps me identify the most frequently used words in the reviews. I chose a horizontal bar chart to display term frequencies, making it easier to read the word labels.

4.Bigram Analysis I also analyze bigrams (pairs of words) to understand word pair relationships in the reviews. The bigram network visualization helps in understanding the connections between frequently used word pairs. I used a network graph for this purpose because it effectively represents relationships.

5.Topic Modeling Using Latent Dirichlet Allocation (LDA), I identify the main topics in the reviews. The topic modeling chart shows the top terms for each topic identified by the LDA model. I used a faceted bar chart to separately display the top terms for each topic.

6.Sentiment Score Distribution I calculate sentiment scores for each review and visualize the distribution by sentiment. The boxplot shows the distribution of sentiment scores for positive and negative reviews, providing insights into the sentiment intensity.

7.Word Clouds Finally, I create word clouds for positive and negative reviews to visualize the most prominent words. Word clouds are effective for visualizing the most frequent words in a large text corpus, and different color palettes help distinguish between positive and negative reviews.

This analysis provides a comprehensive overview of IMDB movie reviews, highlighting sentiment distribution, frequently used terms, bigrams, topics, sentiment scores, and word clouds. The visualizations and models developed in this report offer valuable insights into the nature of movie reviews on IMDB. The methods used here can be applied to similar text analysis tasks in other domains.