DATA 607 Project3

Project Team : DeepfriedData

Bonnie Cooper

David Moste

Abdellah Ait

Gehad Gad

#Import libraries
library(tidyverse)

## Warning: package 'tidyverse' was built under R version 3.6.3

## -- Attaching packages ------------------------------------------------------ tidyverse 1.3.0 --

## v ggplot2 3.3.0     v purrr   0.3.3
## v tibble  2.1.3     v dplyr   0.8.3
## v tidyr   1.0.0     v stringr 1.4.0
## v readr   1.3.1     v forcats 0.5.0

## Warning: package 'tidyr' was built under R version 3.6.2

## Warning: package 'readr' was built under R version 3.6.3

## Warning: package 'dplyr' was built under R version 3.6.2

## Warning: package 'forcats' was built under R version 3.6.3

## -- Conflicts --------------------------------------------------------- tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x dplyr::lag()    masks stats::lag()

library(fmsb)
library(wordcloud)

## Warning: package 'wordcloud' was built under R version 3.6.3

## Loading required package: RColorBrewer

library(tm)

## Warning: package 'tm' was built under R version 3.6.3

## Loading required package: NLP

## 
## Attaching package: 'NLP'

## The following object is masked from 'package:ggplot2':
## 
##     annotate

library(ggpubr)

## Warning: package 'ggpubr' was built under R version 3.6.2

## Loading required package: magrittr

## 
## Attaching package: 'magrittr'

## The following object is masked from 'package:purrr':
## 
##     set_names

## The following object is masked from 'package:tidyr':
## 
##     extract

#Delete/filter the data 
Data2indeed <- Data1indeed[ ,c(5,16) ]

#Display the  data.

head(Data2indeed)

##     sentence value
## 1       data   813
## 2 experience   481
## 3       work   335
## 4     browse   299
## 5    science   222
## 6     center   207

summary(Data2indeed)

##      sentence       value       
##  ability :  1   Min.   :  4.00  
##  able    :  1   1st Qu.:  6.00  
##  academic:  1   Median :  9.00  
##  access  :  1   Mean   : 20.85  
##  accuracy:  1   3rd Qu.: 18.00  
##  accurate:  1   Max.   :813.00  
##  (Other) :994

#Import the reddit data 

Data1reddit <- read.csv ("https://github.com/SmilodonCub/DATA607/raw/master/reddit1000_NLP.csv"
)

#Display the data
head(Data1reddit)

##   X doc_id paragraph_id sentence_id sentence token_id   token   lemma upos
## 1 1   doc1            1           1     data        1    data    data NOUN
## 2 2   doc2            1           1     work        1    work    work NOUN
## 3 3   doc3            1           1      job        1     job     job NOUN
## 4 4   doc4            1           1   people        1  people  people NOUN
## 5 5   doc5            1           1  science        1 science science NOUN
## 6 6   doc6            1           1     know        1    know    know VERB
##   xpos        feats head_token_id dep_rel deps            misc value
## 1   NN  Number=Sing             0    root   NA SpacesAfter=\\n  2697
## 2   NN  Number=Sing             0    root   NA SpacesAfter=\\n  1533
## 3   NN  Number=Sing             0    root   NA SpacesAfter=\\n  1294
## 4  NNS  Number=Plur             0    root   NA SpacesAfter=\\n  1236
## 5   NN  Number=Sing             0    root   NA SpacesAfter=\\n  1232
## 6   VB VerbForm=Inf             0    root   NA SpacesAfter=\\n  1212

##     words Count
## 1    data  2697
## 2    work  1533
## 3     job  1294
## 4  people  1236
## 5 science  1232
## 6    know  1212

summary(Data2reddit)

##         words         Count       
##  ability   :  1   Min.   :  27.0  
##  absolutely:  1   1st Qu.:  37.0  
##  abstract  :  1   Median :  59.0  
##  academia  :  1   Mean   : 113.2  
##  academic  :  1   3rd Qu.: 117.2  
##  access    :  1   Max.   :2697.0  
##  (Other)   :994

#combine the two data in order to get a correlation.

Data3 <- cbind(Data2indeed, Data2reddit)
#Display the data
head(Data3)

##     sentence value   words Count
## 1       data   813    data  2697
## 2 experience   481    work  1533
## 3       work   335     job  1294
## 4     browse   299  people  1236
## 5    science   222 science  1232
## 6     center   207    know  1212

cor(Data3$value,Data3$Count)

## [1] 0.9762292

Correlation shows how strongly the variables are related. The correlation ranges from -1.0 to +1.0. The closer the correlation (r) to +1 or -1, the more closely the two variables are related.

ggscatter(Data3, x= "value", y= "Count", add = "reg.line", cor.coef = TRUE, conf.int = TRUE)

## `geom_smooth()` using formula 'y ~ x'