This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Cmd+Shift+Enter.

This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Cmd+Shift+Enter.

Install necessary packages. Comment after installation

# install.packages('tm')
# install.packages('RColorBrewer')
# install.packages('wordcloud')
# installed.packages('dplyr')
# install.packages("readr")
# install.packages("plyr")
# install.packages("stringr")
# install.packages("stringi")
# install.packages('plotly')

Include the packages.

library('tm')
library('RColorBrewer')
library('wordcloud')
library('readr')
library('dplyr')
library("plyr")
library("stringr")
library("stringi")
library('plotly')

Process data

entrepreneurshipData <- readRDS("entrepreneurship.RDS")
BreneData <- readRDS("BreneBrown.RDS")
Etweets <- entrepreneurshipData$text
Btweets <- BreneData$text

# Read dictionaries
money.words = scan('moneyWords.txt', what='character', comment.char=';')
fear.words = scan('fearWords.txt', what='character', comment.char=';')
pos.words = scan('positive-words.txt', what='character', comment.char=';')
neg.words = scan('negative-words.txt', what='character', comment.char=';')

# Clean data
clean.text = function(x)
{
  # remove rt
  x = gsub("rt", "", x)
  # remove Rt
  x = gsub("Rt", "", x)
  # remove at
  x = gsub("@\\w+", "", x)
  # remove punctuation
  x = gsub("[[:punct:]]", "", x)
  # remove numbers
  x = gsub("[[:digit:]]", "", x)
  # remove links http
  x = gsub("http\\w+", "", x)
  # remove https
  x = gsub("https", "", x)
  # remove tabs
  x = gsub("[ |\t]{2,}", "", x)
  # remove blank spaces at the beginning
  x = gsub("^ ", "", x)
  # remove blank spaces at the end
  x = gsub(" $", "", x)
  # tolower
#  x = tolower(x)
  return(x)
}

# clean tweets
Etweets = clean.text(Etweets)
Btweets = clean.text(Btweets)

Topic Analysis

score.topic = function(sentences, dict, .progress='none')
{
  
  # we got a vector of sentences. plyr will handle a list
  # or a vector as an "l" for us
  # we want a simple array of scores back, so we use
  # "l" + "a" + "ply" = "laply":
  scores = laply(sentences, function(sentence, dict) {
    
    # clean up sentences with R's regex-driven global substitute, gsub():
    sentence = gsub('[[:punct:]]', '', sentence)
    sentence = gsub('[[:cntrl:]]', '', sentence)
    sentence = gsub('\\d+', '', sentence)
    # and convert to lower case:
    # sentence = tolower(sentence)
    
    # split into words. str_split is in the stringr package
    word.list = str_split(sentence, '\\s+')
    # sometimes a list() is one level of hierarchy too much
    words = unlist(word.list)
    
    # compare our words to the dictionaries of positive & negative terms
    topic.matches = match(words, dict)
    
    # match() returns the position of the matched term or NA
    # we just want a TRUE/FALSE:
    topic.matches = !is.na(topic.matches)
    
    # and conveniently enough, TRUE/FALSE will be treated as 1/0 by sum():
    score = sum(topic.matches)
    
    return(score)
  }, dict, .progress=.progress )
  
  topicscores.df = data.frame(score=scores, text=sentences)
  return(topicscores.df)
}

topic.scores= score.topic(Btweets, money.words, .progress='none')
# topic.scores= score.topic(Etweets, fear.words, .progress='none')

topic.mentioned = subset(topic.scores, score !=0)

N= nrow(topic.scores)
Nmentioned = nrow(topic.mentioned)

dftemp=data.frame(topic=c("Mentioned", "Not Mentioned"), 
                  number=c(Nmentioned,N-Nmentioned))

p <- plot_ly(data=dftemp, labels = ~topic, values = ~number, type = 'pie') %>%
  layout(title = 'Pie Chart of Tweets Talking about Money',
         xaxis = list(showgrid = FALSE, zeroline = FALSE, showticklabels = FALSE),
         yaxis = list(showgrid = FALSE, zeroline = FALSE, showticklabels = FALSE))
p

How about tweets showing fear

topic.scores= score.topic(Btweets, fear.words, .progress='none')
# topic.scores= score.topic(Etweets, fear.words, .progress='none')

topic.mentioned = subset(topic.scores, score !=0)

N= nrow(topic.scores)
Nmentioned = nrow(topic.mentioned)

dftemp=data.frame(topic=c("Mentioned", "Not Mentioned"), 
                  number=c(Nmentioned,N-Nmentioned))

p <- plot_ly(data=dftemp, labels = ~topic, values = ~number, type = 'pie') %>%
  layout(title = 'Pie Chart of Tweets Showing Fear',
         xaxis = list(showgrid = FALSE, zeroline = FALSE, showticklabels = FALSE),
         yaxis = list(showgrid = FALSE, zeroline = FALSE, showticklabels = FALSE))
p
# List the tweets showing fear
topic.mentioned$text
##  [1] RTDaysOfCode D Roundstas today with Rubythe self a cash register Im still shaky on amp class variables                       
##  [2] RTDaysOfCode D Roundstas today with Rubythe self a cash register Im still shaky on amp class variables                       
##  [3] RTDaysOfCode D Roundstas today with Rubythe self a cash register Im still shaky on amp class variables                       
##  [4] RTDaysOfCode D Roundstas today with Rubythe self a cash register Im still shaky on amp class variables                       
##  [5] RTDaysOfCode D Roundstas today with Rubythe self a cash register Im still shaky on amp class variables                       
##  [6] DaysOfCode D Roundstas today with Rubythe self a cash register Im still shaky on amp class variab                            
##  [7] daringgreatly never be with those cold and timid souls who neither know victory or defeat livingwithpurpose                  
##  [8] I had a dream thatandwere related and now Im a little suspicious Ill say it                                                  
##  [9] Its not failure were afraid ofits shame amp blame\nBeing vulnerablecourage BreneBrown says it best TED                       
## [10] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [11] RTAs daring leaders we have to stay curious about our own blind spots amp how to pull those issues into view amp we need to  
## [12] As daring leaders we have to stay curious about our own blind spots amp how to pull those issues into view amp we need       
## [13] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [14] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [15] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [16] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [17] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [18] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [19] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [20] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [21] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [22] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [23] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [24] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [25] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [26] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [27] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [28] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [29] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [30] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [31] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [32] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [33] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [34] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [35] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [36] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [37] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [38] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [39] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [40] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [41] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [42] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [43] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [44] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [45] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [46] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [47] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [48] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [49] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [50] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [51] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [52] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [53] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [54] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [55] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [56] Yes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave an              
## [57] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [58] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [59] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [60] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [61] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [62] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [63] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [64] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [65] RTYes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave and wor       
## [66] Yes I am imperfect and vulnerable and sometimes afraid but that doesnt change the truth that I am also brave an              
## [67] I suppose feeling anxious about how the other person takes it means you care so see it as a good thing                       
## [68] Im always clear but its not always what people want to hear so then I feel anxious etc                                       
## [69] While Im nervous to post about this on a public platform Im trying to live by my belief that vulnerability is no             
## [70] I think gradually is the key word here So gradually that ppl werent horrified earl                                           
## [71] Reading it tonight Im afraid to move forward But cant bare to stand still                                                    
## [72] My bookclub choseBravingTheWildernessI am not a self help book reader I was shocked by how much thi                          
## [73] We must be guardians of a space that allows students to breathe and be curious and explore the world and be who t            
## [74] RTWhen were anxious disconnected vulnerable alone amp feeling helpless booze amp food amp work amp bingewatching endles      
## [75] RTWhen were anxious disconnected vulnerable alone amp feeling helpless booze amp food amp work amp bingewatching endles      
## [76] When were anxious disconnected vulnerable alone amp feeling helpless booze amp food amp work amp bingewatching end           
## [77] RTMen walk this tightrope where any sign of weakness illicits shame amp so theyre afraid to make themselves vulnerable for fe
## [78] RTMen walk this tightrope where any sign of weakness illicits shame amp so theyre afraid to make themselves vulnerable for fe
## 2327 Levels:  ️ ️️️️️️ ️️️ ️️️️ \n \n\n ... ציטוט שבועי של דר ברנה בראון כאשר אנו אדיבים לעצמנו אנו יוצרים מאגר של חמלה שאנו יכולים להרחיב לאחרים\n\n דף הא

Sentiment Analysis

score.sentiment = function(sentences, pos.words, neg.words, .progress='none')
{
  
  # we got a vector of sentences. plyr will handle a list
  # or a vector as an "l" for us
  # we want a simple array of scores back, so we use
  # "l" + "a" + "ply" = "laply":
  scores = laply(sentences, function(sentence, pos.words, neg.words) {
    
    # clean up sentences with R's regex-driven global substitute, gsub():
    sentence = gsub('[[:punct:]]', '', sentence)
    sentence = gsub('[[:cntrl:]]', '', sentence)
    sentence = gsub('\\d+', '', sentence)
    # and convert to lower case:
    sentence = tolower(sentence)
    
    # split into words. str_split is in the stringr package
    word.list = str_split(sentence, '\\s+')
    # sometimes a list() is one level of hierarchy too much
    words = unlist(word.list)
    
    # compare our words to the dictionaries of positive & negative terms
    pos.matches = match(words, pos.words)
    neg.matches = match(words, neg.words)
    
    # match() returns the position of the matched term or NA
    # we just want a TRUE/FALSE:
    pos.matches = !is.na(pos.matches)
    neg.matches = !is.na(neg.matches)
    
    # and conveniently enough, TRUE/FALSE will be treated as 1/0 by sum():
    score = sum(pos.matches) - sum(neg.matches)
    
    return(score)
  }, pos.words, neg.words, .progress=.progress )
  
  scores.df = data.frame(score=scores, text=sentences)
  return(scores.df)
}

sentiment.scores= score.sentiment(Btweets, pos.words, neg.words, .progress='none')

score <- sentiment.scores$score
p <- plot_ly(x = ~score, type = "histogram")
p

Word cloud of negative tweets

require(tm)
require(wordcloud)
require(RColorBrewer)

negativeTweets = subset(sentiment.scores, score < 0)$text

corpus = Corpus(VectorSource(negativeTweets))
# corpus = Corpus(VectorSource(cmail))
# create term-document matrix
tdm = TermDocumentMatrix(
  corpus,
  control = list(
    wordLengths=c(3,20),
    removePunctuation = TRUE,
    stopwords = c("the", "a", stopwords("english")),
    removeNumbers = TRUE, tolower = FALSE) )

# convert as matrix
tdm = as.matrix(tdm)

# get word counts in decreasing order
word_freqs = sort(rowSums(tdm), decreasing=TRUE) 
# word_freqs = word_freqs[-(1:12)]
# create a data frame with words and their frequencies
dm = data.frame(word=names(word_freqs), freq=word_freqs)

#Plot corpus in a clored graph; need RColorBrewer package

wordcloud(head(dm$word, 100), head(dm$freq, 100), random.order=FALSE, colors=brewer.pal(8, "Dark2"))

#check top 50 most mentioned words
head(word_freqs, 20)
##  imperfect       also vulnerable      truth        amp      brave 
##         79         72         71         68         67         66 
##     afraid     doesnt     change  sometimes      RTYes        wor 
##         59         58         58         56         53         53 
##      youre      shame   emotions        hea      arena        Not 
##         50         48         44         43         41         41 
## interested       love 
##         40         39

Add a new chunk by clicking the Insert Chunk button on the toolbar or by pressing Cmd+Option+I.

When you save the notebook, an HTML file containing the code and output will be saved alongside it (click the Preview button or press Cmd+Shift+K to preview the HTML file).

The preview shows you a rendered HTML copy of the contents of the editor. Consequently, unlike Knit, Preview does not run any R code chunks. Instead, the output of the chunk when it was last run in the editor is displayed.

Add a new chunk by clicking the Insert Chunk button on the toolbar or by pressing Cmd+Option+I.

When you save the notebook, an HTML file containing the code and output will be saved alongside it (click the Preview button or press Cmd+Shift+K to preview the HTML file).

The preview shows you a rendered HTML copy of the contents of the editor. Consequently, unlike Knit, Preview does not run any R code chunks. Instead, the output of the chunk when it was last run in the editor is displayed.