sentiment_analysis
GuillermoSchneider
2024-03-08
PROJECT 3: Guillermo Schneider, Richie Rivera, Lucas Weyrich, Jonathan Cruz
Summary:
Search through reddit comments Search for comments that include Skills see what positive or negative words are around in those comments average sentiment score for each Skill sentiment analysis over time
#install.packages("tidytext")
#install.packages("textdata")
#install.packages("tidyverse")
#install.packages("wordcloud")
#install.packages("datawizard")
library(tidytext)## Warning: package 'tidytext' was built under R version 4.3.3library(textdata)## Warning: package 'textdata' was built under R version 4.3.3library(tidyverse)## Warning: package 'tidyverse' was built under R version 4.3.3## Warning: package 'ggplot2' was built under R version 4.3.3## Warning: package 'tibble' was built under R version 4.3.3## Warning: package 'tidyr' was built under R version 4.3.3## Warning: package 'readr' was built under R version 4.3.3## Warning: package 'purrr' was built under R version 4.3.3## Warning: package 'forcats' was built under R version 4.3.3## Warning: package 'lubridate' was built under R version 4.3.3## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr 1.1.4 ✔ readr 2.1.5
## ✔ forcats 1.0.0 ✔ stringr 1.5.1
## ✔ ggplot2 3.5.0 ✔ tibble 3.2.1
## ✔ lubridate 1.9.3 ✔ tidyr 1.3.1
## ✔ purrr 1.0.2
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errorslibrary(wordcloud)## Warning: package 'wordcloud' was built under R version 4.3.3## Loading required package: RColorBrewerlibrary(caret)## Warning: package 'caret' was built under R version 4.3.3## Loading required package: lattice
##
## Attaching package: 'caret'
##
## The following object is masked from 'package:purrr':
##
## liftlibrary(datawizard)## Warning: package 'datawizard' was built under R version 4.3.3https://www.tidytextmining.com/sentiment
GRABBING SENTIMENT LABELED DICTIONARIES
AFINN from Finn Årup Nielsen
afinn <- get_sentiments("afinn")bing from Bing Liu and collaborators
bing <- get_sentiments("bing")nrc from Saif Mohammad and Peter Turney
nrc <- get_sentiments("nrc")afinn_nrc_sentiments <- afinn %>%
inner_join(nrc) ## Joining with `by = join_by(word)`SCRAPING THANKS TO LUCAS
sample of 8000 posts from lucas
test_scrape <- read.csv("https://raw.githubusercontent.com/GuillermoCharlesSchneider/DATA-607/main/Project-3/test_scrape.csv")full 125k posts from lucas
full_scrape <- read.csv("https://raw.githubusercontent.com/riverar9/cuny-msds-project-3/main/part_2/reddit%20scrape.csv")UPDATED SKILLS
Skills provided by jon:
Skills <- read.csv("https://raw.githubusercontent.com/GuillermoCharlesSchneider/DATA-607/main/Project-3/skills.csv")separate skills into single words, while still keeping the general skill bucket
Skills$word <- tolower(Skills$Skills)
Skills_single_word <- Skills %>%
unnest_tokens(word, word)
Skills_single_word <- Skills_single_word[-c(4,9,10,11,20,21,22,23,25,28,31,40,41,42,43,46,48,50,55,57,62,65,67,71,72,75,80,83,84,92,99,136,140,141,146,147,148,150,151,152,153,154), ]
row.names(Skills_single_word) <- 1:nrow(Skills_single_word)TWO WORD SKILLS MERGED INTO ONE and trimmed (ex:data science -> datascience), multiple words per bucket. this is a stupid method, but i was too tired to figure out a new function, and i needed to look through them all anyway to see which needed to be combined and which didnt:
for(x in c(1,5,10,13,21,24,23,26,29,35,37,39,42,46,48,50,56,60,59,64,63,66,73,72,75,77,80,79,82,84,86,91,93,98,97,102,101,100,106,108,110)){
Skills_single_word$word[x] <- paste(Skills_single_word$word[x],Skills_single_word$word[x+1],sep = "")
}
Skills_single_word <- Skills_single_word[-c(2,6,11,14,17,18,22,25,24,27,30,36,38,40,43,47,49,51,57,61,60,65,64,67,74,73,76,78,81,80,83,85,87,90,92,94,99,98,103,102,101,107,109,111), ]
row.names(Skills_single_word) <- 1:nrow(Skills_single_word)write.csv(Skills_single_word, "Updated_Skills.csv", row.names=FALSE)MERGED 2 WORD SKILLS (ex: data science -> datascience) IN SCRAPED REDDIT COMMENTS
TEST SCRAPE 8k
test_scrape$comment <- tolower(test_scrape$comment)test 8k data, once again im sure we could write a function, but ive already commited to this stupid copy paste, maybe one of you can make a: gsub inputs(skills…), desired outputs (skills merged…)
test_scrape$comment <- gsub("data science", "datascience,",test_scrape$comment)
test_scrape$comment <- gsub("data analysis", "dataanalysis,",test_scrape$comment)
test_scrape$comment <- gsub("cloud databases", "clouddatabases",test_scrape$comment)
test_scrape$comment <- gsub("machine learning", "machinelearning",test_scrape$comment)
test_scrape$comment <- gsub("apache spark", "apachespark",test_scrape$comment)
test_scrape$comment <- gsub("pivot table", "pivottable",test_scrape$comment)
test_scrape$comment <- gsub("case studies", "casestudies",test_scrape$comment)
test_scrape$comment <- gsub("convolutional neuralnet works", "convolutionalneuralnetworks",test_scrape$comment)
test_scrape$comment <- gsub("data ethics", "dataethics",test_scrape$comment)
test_scrape$comment <- gsub("data collection", "datacollection,",test_scrape$comment)
test_scrape$comment <- gsub("data exploration", "dataexploration,",test_scrape$comment)
test_scrape$comment <- gsub("artificial intelligence", "artificialintelligence,",test_scrape$comment)
test_scrape$comment <- gsub("machine learning", "machinelearning,",test_scrape$comment)
#ive started to regret it about here
test_scrape$comment <- gsub("ask questions", "askquestions,",test_scrape$comment)
test_scrape$comment <- gsub("data driven", "datadriven",test_scrape$comment)
test_scrape$comment <- gsub("big data", "bigdata",test_scrape$comment)
test_scrape$comment <- gsub("descriptive statistics", "descriptivestatistics",test_scrape$comment)
test_scrape$comment <- gsub("recurrent neural network", "recurrentneuralnetwork",test_scrape$comment)
test_scrape$comment <- gsub("deep neural networks", "deepneuralnetworks",test_scrape$comment)
test_scrape$comment <- gsub("hyper parameter tuning", "hyperparametertuning",test_scrape$comment)
test_scrape$comment <- gsub("exploratory data analysis", "exploratorydataanalysis",test_scrape$comment)
test_scrape$comment <- gsub("neural network architecture", "neuralnetworkarchitecture",test_scrape$comment)
test_scrape$comment <- gsub("regression analysis", "regressionanalysis",test_scrape$comment)
test_scrape$comment <- gsub("data aggregation", "dataaggregation",test_scrape$comment)
test_scrape$comment <- gsub("deep learning", "deeplearning",test_scrape$comment)
test_scrape$comment <- gsub("data cleansing", "datacleansing",test_scrape$comment)
test_scrape$comment <- gsub("statistical hypothesis testing", "statisticalhypothesistesting",test_scrape$comment)
test_scrape$comment <- gsub("relational database management systems", "relationaldatabasemanagementsystems",test_scrape$comment)
test_scrape$comment <- gsub("information technology", "informationtechnology",test_scrape$comment)
test_scrape$comment <- gsub("data architecture", "dataarchitecture",test_scrape$comment)
test_scrape$comment <- gsub("process data", "processdata",test_scrape$comment)separate 8000 comments into single words
tidy_test_scrape <- test_scrape %>%
unnest_tokens(word, comment)TEST_SCRAPE: filter to just comments containing skills
TEST <- tidy_test_scrape %>%
inner_join(Skills_single_word) ## Joining with `by = join_by(word)`## Warning in inner_join(., Skills_single_word): Detected an unexpected many-to-many relationship between `x` and `y`.
## ℹ Row 1460 of `x` matches multiple rows in `y`.
## ℹ Row 49 of `y` matches multiple rows in `x`.
## ℹ If a many-to-many relationship is expected, set `relationship =
## "many-to-many"` to silence this warning.comment_ids <- distinct(TEST, X, Skills)
TEST2 <- filter(tidy_test_scrape, tidy_test_scrape$X %in% comment_ids$X)
TEST2 <- TEST2 %>%
left_join(afinn) ## Joining with `by = join_by(word)`TEST2 <- TEST2 %>%
left_join(bing) ## Joining with `by = join_by(word)`TEST2 <- TEST2 %>%
left_join(Skills_single_word)## Joining with `by = join_by(word)`## Warning in left_join(., Skills_single_word): Detected an unexpected many-to-many relationship between `x` and `y`.
## ℹ Row 631 of `x` matches multiple rows in `y`.
## ℹ Row 49 of `y` matches multiple rows in `x`.
## ℹ If a many-to-many relationship is expected, set `relationship =
## "many-to-many"` to silence this warning.#MEAN excluding all non sentiment words. sum of sentiment values / number of only sentiment words
TEST3 <- TEST2 %>%
group_by(X) %>%
summarize(sentiment_score = mean(value,na.rm=T))
#setting each word without a sentiment, to a 0 sentiment value
TEST2$value[is.na(TEST2$value)] <- 0
#MEAN of sentiments / total words (including nonsentiments) in comment
TEST4 <- TEST2 %>%
group_by(X) %>%
summarize(sentiment_score = mean(value))
TEST5 <- TEST2 %>%
group_by(X) %>%
summarize(sentiment_score = sum(value))write.csv(tidy_test_scrape, "tidy_test_scrape_merged.csv", row.names=FALSE)FULL SCRAPE
full_scrape TO LOWER CASE
full_scrape$comment <- tolower(full_scrape$comment)CLEANING TWO WORDS TO ONE. once again no function… zzz
full_scrape$comment <- gsub("data science", "datascience,",full_scrape$comment)
full_scrape$comment <- gsub("data analysis", "dataanalysis,",full_scrape$comment)
full_scrape$comment <- gsub("cloud databases", "clouddatabases",full_scrape$comment)
full_scrape$comment <- gsub("machine learning", "machinelearning",full_scrape$comment)
full_scrape$comment <- gsub("apache spark", "apachespark",full_scrape$comment)
full_scrape$comment <- gsub("pivot table", "pivottable",full_scrape$comment)
full_scrape$comment <- gsub("case studies", "casestudies",full_scrape$comment)
full_scrape$comment <- gsub("convolutional neuralnet works", "convolutionalneuralnetworks",full_scrape$comment)
full_scrape$comment <- gsub("data ethics", "dataethics",full_scrape$comment)
full_scrape$comment <- gsub("data collection", "datacollection,",full_scrape$comment)
full_scrape$comment <- gsub("data exploration", "dataexploration,",full_scrape$comment)
full_scrape$comment <- gsub("artificial intelligence", "artificialintelligence,",full_scrape$comment)
full_scrape$comment <- gsub("machine learning", "machinelearning,",full_scrape$comment)
#ive started to regret it about here
full_scrape$comment <- gsub("ask questions", "askquestions,",full_scrape$comment)
full_scrape$comment <- gsub("data driven", "datadriven",full_scrape$comment)
full_scrape$comment <- gsub("big data", "bigdata",full_scrape$comment)
full_scrape$comment <- gsub("descriptive statistics", "descriptivestatistics",full_scrape$comment)
full_scrape$comment <- gsub("recurrent neural network", "recurrentneuralnetwork",full_scrape$comment)
full_scrape$comment <- gsub("deep neural networks", "deepneuralnetworks",full_scrape$comment)
full_scrape$comment <- gsub("hyper parameter tuning", "hyperparametertuning",full_scrape$comment)
full_scrape$comment <- gsub("exploratory data analysis", "exploratorydataanalysis",full_scrape$comment)
full_scrape$comment <- gsub("neural network architecture", "neuralnetworkarchitecture",full_scrape$comment)
full_scrape$comment <- gsub("regression analysis", "regressionanalysis",full_scrape$comment)
full_scrape$comment <- gsub("data aggregation", "dataaggregation",full_scrape$comment)
full_scrape$comment <- gsub("deep learning", "deeplearning",full_scrape$comment)
full_scrape$comment <- gsub("data cleansing", "datacleansing",full_scrape$comment)
full_scrape$comment <- gsub("statistical hypothesis testing", "statisticalhypothesistesting",full_scrape$comment)
full_scrape$comment <- gsub("relational database management systems", "relationaldatabasemanagementsystems",full_scrape$comment)
full_scrape$comment <- gsub("information technology", "informationtechnology",full_scrape$comment)
full_scrape$comment <- gsub("data architecture", "dataarchitecture",full_scrape$comment)
full_scrape$comment <- gsub("process data", "processdata",full_scrape$comment)separate 125k comments into single words
tidy_full_scrape <- full_scrape %>%
unnest_tokens(word, comment)FULL SCRAPE: FIND COMMENTS CONTAINING SKILLS
left join sentiments into the full scrape
sentiment_tidy_merged_full_scrape <- tidy_full_scrape %>%
left_join(afinn) ## Joining with `by = join_by(word)`sentiment_tidy_merged_full_scrape <- sentiment_tidy_merged_full_scrape %>%
left_join(bing) ## Joining with `by = join_by(word)`## Warning in left_join(., bing): Detected an unexpected many-to-many relationship between `x` and `y`.
## ℹ Row 683897 of `x` matches multiple rows in `y`.
## ℹ Row 2301 of `y` matches multiple rows in `x`.
## ℹ If a many-to-many relationship is expected, set `relationship =
## "many-to-many"` to silence this warning.one hot encoding is a good solution for the NRC multiple sentiments per word THANK YOU LUCAS
#nrc$word = as.factor(nrc$word)
#onehotnrc <- predict(dummyVars(" ~ .", data = nrc), newdata = nrc)
#onehotnrc = as.data.frame(onehotnrc)
#group by sentiment
#lucas will use for nrc analysis if needed
#library(caret)
#one_hot_encoded <- dummyVars("~.", data = data)
#encoded_data <- data.frame(predict(one_hot_encoded, newdata = data))CLEANED TIDIED DATASET WITH MERGED WORDS
write.csv(sentiment_tidy_merged_full_scrape, "tidy_full_scrape_merged_sentiments.csv", row.names=FALSE)count of single word Skills words in the tidied scape:
#8000 comments
tidy_test_scrape %>%
inner_join(Skills_single_word) %>%
count(word, sort = TRUE) %>%
head(10)## Joining with `by = join_by(word)`## Warning in inner_join(., Skills_single_word): Detected an unexpected many-to-many relationship between `x` and `y`.
## ℹ Row 1460 of `x` matches multiple rows in `y`.
## ℹ Row 49 of `y` matches multiple rows in `x`.
## ℹ If a many-to-many relationship is expected, set `relationship =
## "many-to-many"` to silence this warning.## word n
## 1 datascience 797
## 2 python 441
## 3 ai 427
## 4 r 377
## 5 machinelearning 276
## 6 statistics 208
## 7 sql 185
## 8 excel 125
## 9 jupyter 119
## 10 probability 71#125k comments
tidy_full_scrape %>%
inner_join(Skills_single_word) %>%
count(word, sort = TRUE) %>%
head(10)## Joining with `by = join_by(word)`## Warning in inner_join(., Skills_single_word): Detected an unexpected many-to-many relationship between `x` and `y`.
## ℹ Row 14950 of `x` matches multiple rows in `y`.
## ℹ Row 23 of `y` matches multiple rows in `x`.
## ℹ If a many-to-many relationship is expected, set `relationship =
## "many-to-many"` to silence this warning.## word n
## 1 python 9213
## 2 sql 9070
## 3 datascience 6760
## 4 r 5121
## 5 ai 3055
## 6 machinelearning 2648
## 7 pandas 2563
## 8 excel 2430
## 9 database 2297
## 10 pipelines 1995joyful words from NRC:
nrc_joy <- get_sentiments("nrc") %>%
filter(sentiment == "joy")count of joyful words in the tidied single word scrape, matching 1 to 1 words w inner join on “word” column:
tidy_test_scrape %>%
inner_join(nrc_joy) %>%
count(word, sort = TRUE) %>%
head(10)## Joining with `by = join_by(word)`## word n
## 1 good 870
## 2 pretty 278
## 3 pay 250
## 4 kind 208
## 5 money 175
## 6 salary 174
## 7 love 170
## 8 true 144
## 9 excel 125
## 10 create 121count of all single words
t <- tidy_full_scrape %>%
count(word, sort = TRUE)Time Series graphs (thanks LUCAS)
library(ggplot2)
library(zoo)## Warning: package 'zoo' was built under R version 4.3.3##
## Attaching package: 'zoo'## The following objects are masked from 'package:base':
##
## as.Date, as.Date.numericsentiment_comments_containing_skills_full_scrape <- comments_containing_skills_full_scrape %>%
left_join(afinn) ## Joining with `by = join_by(word, value)`sentiment_comments_containing_skills_full_scrape <- sentiment_comments_containing_skills_full_scrape %>%
left_join(bing) ## Joining with `by = join_by(word)`## Warning in left_join(., bing): Detected an unexpected many-to-many relationship between `x` and `y`.
## ℹ Row 283948 of `x` matches multiple rows in `y`.
## ℹ Row 6718 of `y` matches multiple rows in `x`.
## ℹ If a many-to-many relationship is expected, set `relationship =
## "many-to-many"` to silence this warning.time_series_full_scrape = sentiment_tidy_merged_full_scrape %>%
group_by(date) %>%
summarize(positive = sum(sentiment == "positive", na.rm = TRUE),
negative = sum(sentiment == "negative", na.rm = TRUE)) %>%
mutate(sentiment_value = positive - negative) %>%
mutate(roll_avg_7 = rollmean(sentiment_value, k = 7, fill = NA)) %>%
mutate(roll_avg_30 = rollmean(sentiment_value, k = 30, fill = NA)) %>%
mutate(roll_avg_30_pos = rollmean(positive, k = 30, fill = NA)) %>%
mutate(roll_avg_30_neg = rollmean(negative, k = 30, fill = NA))
time_series_skills_scrape = sentiment_comments_containing_skills_full_scrape %>%
group_by(date) %>%
summarize(positive = sum(sentiment == "positive", na.rm = TRUE),
negative = sum(sentiment == "negative", na.rm = TRUE)) %>%
mutate(sentiment_value = positive - negative) %>%
mutate(roll_avg_7 = rollmean(sentiment_value, k = 7, fill = NA)) %>%
mutate(roll_avg_30 = rollmean(sentiment_value, k = 30, fill = NA)) %>%
mutate(roll_avg_30_pos = rollmean(positive, k = 30, fill = NA)) %>%
mutate(roll_avg_30_neg = rollmean(negative, k = 30, fill = NA))
time_series_full_scrape$date = as.Date(time_series_full_scrape$date)
time_series_skills_scrape$date = as.Date(time_series_skills_scrape$date)
ggplot(aes(x = date, y = sentiment_value), data = time_series_full_scrape) +
geom_line() +
theme_minimal() +
labs(x = 'Date', y = 'Sentiment Count (Positive - Negative)', title = 'Sentiment Counts for r/DataScience and r/DataEngineer')
ggplot(aes(x = date, y = roll_avg_30), data = time_series_full_scrape) +
geom_line() +
theme_minimal() +
labs(x = 'Date', y = 'Sentiment Count (Positive - Negative)', title = '30-day Rolling Mean of Sentiment for r/DataScience and r/DataEngineer')## Warning: Removed 29 rows containing missing values or values outside the scale range
## (`geom_line()`).
ggplot(time_series_full_scrape, aes(x = date)) +
geom_line(aes(y = roll_avg_30_pos, color = "Positive")) +
geom_line(aes(y = roll_avg_30_neg, color = "Negative")) +
labs(x = "Date", y = " Sentiment Count", title = "30-day Rolling Mean of Sentiment for r/DataScience and r/DataEngineer", color = "Sentiment") +
scale_color_manual(values = c("Positive" = "#4CCD99", "Negative" = "#EE4266")) +
theme_minimal()## Warning: Removed 29 rows containing missing values or values outside the scale range
## (`geom_line()`).## Warning: Removed 29 rows containing missing values or values outside the scale range
## (`geom_line()`).
ggplot(aes(x = date, y = sentiment_value), data = time_series_skills_scrape) +
geom_line() +
theme_minimal() +
labs(x = 'Date', y = 'Sentiment Count (Positive - Negative)', title = 'Sentiment Counts for Skills in r/DataScience and r/DataEngineer')
ggplot(aes(x = date, y = roll_avg_30), data = time_series_skills_scrape) +
geom_line() +
theme_minimal() +
labs(x = 'Date', y = 'Sentiment Count (Positive - Negative)', title = '30-day Rolling Mean of Sentiment for Skills in r/DataScience and r/DataEngineer')## Warning: Removed 29 rows containing missing values or values outside the scale range
## (`geom_line()`).
ggplot(time_series_skills_scrape, aes(x = date)) +
geom_line(aes(y = roll_avg_30_pos, color = "Positive")) +
geom_line(aes(y = roll_avg_30_neg, color = "Negative")) +
labs(x = "Date", y = " Sentiment Count", title = "30-day Rolling Mean of Sentiment for Skills in r/DataScience and r/DataEngineer", color = "Sentiment") +
scale_color_manual(values = c("Positive" = "#4CCD99", "Negative" = "#EE4266")) +
theme_minimal()## Warning: Removed 29 rows containing missing values or values outside the scale range
## (`geom_line()`).
## Removed 29 rows containing missing values or values outside the scale range
## (`geom_line()`).
potential word cloud code?
library(wordcloud) w <- sort(rowSums(t), decreasing = TRUE) set.seed(222) wordcloud(word = word(w), freq = w, max.words = 150, random.order = F, min.freq = 5, colors = brewer.pal(8, ‘Dark2’), scale = c(5, 0.3), rot.per = 0.7)