#GGplot Now lets take a look at some ggplot barplots

we’ll start with making a dataframe based on tooth data

df <- data.frame(dose = c("D0.5", "D1", "D2"), 
                 len = c(4.2, 10, 29.5))

df
##   dose  len
## 1 D0.5  4.2
## 2   D1 10.0
## 3   D2 29.5

And now lets make a second data frame

df2 <- data.frame(supp=rep(c("VC", "OJ"), each = 3), 
                  dose = rep(c("D0.5", "D1", "D2"), 2), 
                  len = c(6.8, 15, 33, 4.2, 10, 29.5))

df2
##   supp dose  len
## 1   VC D0.5  6.8
## 2   VC   D1 15.0
## 3   VC   D2 33.0
## 4   OJ D0.5  4.2
## 5   OJ   D1 10.0
## 6   OJ   D2 29.5

lets load up ggplot2

library(ggplot2)

lets set our perameters for ggplot

theme_set(
  theme_classic() +
    theme(legend.position = "top")
)

lets start with some basic barplots using the tooth data

f <- ggplot(df, aes(x = dose, y = len))

f + geom_col()

now lets change the fill, and add labels to the top

f + geom_col(fill = "darkblue") + 
  geom_text(aes(label = len), vjust = -0.3)

now lets add the labels beside the bars

f + geom_col(fill = "darkblue") + 
  geom_text(aes(label = len), vjust = 1.6, color = "white")

now lets change the bar plot colors by group

f + geom_col(aes(color = dose), fill = "white") + 
  scale_color_manual(values = c("blue", "gold", "red"))

this is kind of hard to see, so lets change the fill

f + geom_col(aes(fill = dose)) + 
  scale_fill_manual(values = c("blue", "gold", "red"))

how do we do this with multiple groups?

ggplot(df2, aes(x = dose, y = len)) + 
  geom_col(aes(color = supp, fill = supp), position = position_stack()) + 
  scale_color_manual(values = c("blue", "gold")) + 
  scale_fill_manual(values = c("blue", "gold"))

p <- ggplot(df2, aes(x = dose, y = len)) + 
  geom_col(aes(color = supp, fill = supp), position = position_dodge(0.8), width = 0.7) + 
  scale_color_manual(values = c("blue", "gold")) + 
  scale_fill_manual(values = c("blue", "gold"))
p

now lets add those labels to the dodged barplot

p + geom_text(
  aes(label = len, group = supp), 
  position = position_dodge(0.8), 
  vjust = -0.3, size = 3.5
)

now what if we want to add labels to our stacked barplots? for this we need dplyr

library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
df2 <- df2 %>%
  group_by(dose) %>%
  arrange(dose, desc(supp)) %>%
  mutate (lab_ypos = cumsum(len) - 0.5 * len)
df2
## # A tibble: 6 × 4
## # Groups:   dose [3]
##   supp  dose    len lab_ypos
##   <chr> <chr> <dbl>    <dbl>
## 1 VC    D0.5    6.8      3.4
## 2 OJ    D0.5    4.2      8.9
## 3 VC    D1     15        7.5
## 4 OJ    D1     10       20  
## 5 VC    D2     33       16.5
## 6 OJ    D2     29.5     47.8

now lets recreate our stacked graphs

ggplot(df2, aes(x = dose, y = len)) + 
  geom_col(aes(fill = supp), width = 0.7) + 
  geom_text(aes(y = lab_ypos, label = len, group = supp), color = "white") + 
  scale_color_manual(values = c("blue", "gold")) + 
  scale_fill_manual(values = c("blue", "gold"))

Boxplots

Lets look at some boxplots

data("ToothGrowth")

Lets change the dose to a factor and look at the top of the dataframe

ToothGrowth$dose <- as.factor(ToothGrowth$dose)

head(ToothGrowth , 4)
##    len supp dose
## 1  4.2   VC  0.5
## 2 11.5   VC  0.5
## 3  7.3   VC  0.5
## 4  5.8   VC  0.5

Lets load ggplot

library(ggplot2)

Lets set the theme of our plot to classic

theme_set(
  theme_bw() +
    theme(legend.position = "top")
)

Lets start with a very basic box plot with dose vs length

tg <- ggplot(ToothGrowth, aes(x = dose, y = len))
tg + geom_boxplot()

Now lets look at a boxplot with points for the mean

tg + geom_boxplot(notch = TRUE, fill = "lightgrey") + 
  stat_summary(fun.y = mean, geom = "point", shape = 18, size = 2.5, color = "indianred")
## Warning: The `fun.y` argument of `stat_summary()` is deprecated as of ggplot2 3.3.0.
## ℹ Please use the `fun` argument instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.

we can also change the scale number of variables included, and their order

tg + geom_boxplot() + 
  scale_x_discrete(limits = c("0.5", "2"))
## Warning: Removed 20 rows containing missing values (`stat_boxplot()`).

lets put our x axis in descending order

tg + geom_boxplot() + 
  scale_x_discrete(limits = c(".2", "1", "0.5"))
## Warning: Removed 20 rows containing missing values (`stat_boxplot()`).

we can also change boxplot colors by groups

tg + geom_boxplot(aes(color = dose)) + 
  scale_color_manual(values = c("indianred", "blue1", "green2"))

what if we want to display our data subset by oj vs vitamin c?

tg2 <- tg + geom_boxplot(aes(fill = supp), position = position_dodge(0.9)) + 
  scale_fill_manual(values = c("#999999", "#E69f00"))

tg2

we can also arrange this as two plots with facet_wrap

tg2 + facet_wrap(~supp)

Histograms

set.seed(1234)

wdata = data.frame(
  sex = factor(rep(c("F", "M"), each = 200)),
  weight = c(rnorm(200, 56), rnorm(200, 58))
)

head(wdata, 4)
##   sex   weight
## 1   F 54.79293
## 2   F 56.27743
## 3   F 57.08444
## 4   F 53.65430

Now lets load dplyr

library(dplyr)

mu <- wdata %>%
  group_by(sex) %>%
  summarise(grp.mean = mean(weight))

Now lets load the plotting package

library(ggplot2)

theme_set(
  theme_classic() + 
    theme(legend.position = "bottom")
)

Now lets create a ggplot object

a <- ggplot(wdata, aes(x = weight))

a + geom_histogram(bins = 30, color = "black", fill = "grey") + 
  geom_vline(aes(xintercept = mean(weight)),
             linetype = "dashed", size = 0.6)
## Warning: Using `size` aesthetic for lines was deprecated in ggplot2 3.4.0.
## ℹ Please use `linewidth` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.

Now lets change the color by group

a + geom_histogram(aes(color = sex), fill = "white", position = "identity") + 
  scale_color_manual(values = c("#00AFBB", "#E7B800"))
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

a + geom_histogram(aes(color = sex, fill = sex), position = "identity") + 
  scale_color_manual(values = c("#00AFBB", "#E7B800")) + 
  scale_fill_manual(values = c("indianred", "lightblue1"))
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

what if we want to combine density plots and histograms?

a + geom_histogram(aes(y = stat(density)),
                   color = "black", fill = "white") + 
  geom_density(alpha = 0.2, fill = "#FF6666")
## Warning: `stat(density)` was deprecated in ggplot2 3.4.0.
## ℹ Please use `after_stat(density)` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

a + geom_histogram(aes(y = stat(density), color = sex), fill = "white", position = "identity") + 
  geom_density(aes(color = sex), size = 1) + 
  scale_color_manual(values = c("indianred", "lightblue1"))
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

Dotplots

First lets load the required packages

library(ggplot2)

lets set our theme

theme_set(
  theme_dark() +
    theme(legend.position = "top")
)

First lets initiate a ggplot called TG

data("ToothGrowth")
ToothGrowth$dose <- as.factor(ToothGrowth$dose)

tg <- ggplot(ToothGrowth, aes(x=dose, y=len))

lets create a dotplot with a summary statistic

tg + geom_dotplot(binaxis = "y", stackdir = "center", fill = "lightgray") + 
  stat_summary(fun = mean, fun.args = list(mult=1))
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.
## Warning: Removed 3 rows containing missing values (`geom_segment()`).

lets add a boxplot and a dotplot together

tg + geom_boxplot(width = 0.5) + 
  geom_dotplot(binaxis = "y", stackdir = "center", fill = "white")
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.

tg + geom_violin(trim = FALSE) + 
  geom_dotplot(binaxis = "y", stackdir = "center", fill = "#999999") + 
  stat_summary(fun = mean, fun.args = list(mult = 1))
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.
## Warning: Removed 3 rows containing missing values (`geom_segment()`).

lets create a dotplot with multiple groups

tg + geom_boxplot(width = 0.5) + 
  geom_dotplot(aes(fill = supp), binaxis = "y", stackdir = "center") + 
  scale_fill_manual(values = c("indianred", "lightblue"))
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.

tg + geom_boxplot(aes(color = supp), width = 0.5, position = position_dodge(0.8)) + 
  geom_dotplot(aes(fill = supp, color = supp), binaxis = "y", stackdir = "center", 
               dotsize = 0.8, position = position_dodge(0.8)) + 
  scale_fill_manual(values = c("#00AFBB", "#E7B800")) + 
  scale_color_manual(values = c("#00AFBB", "#E7B800"))
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.

Lineplots

now lets change it up and look at some lineplots

we’ll start by making a custom dataframe like the tooh dataset. this way we can see the lines and stuff we are modifying

df <- data.frame(dose = c("D0.5", "D1", "D2"), 
                 len = c(4.2, 10, 29.5))

now lets create a second dataframe for plotting groups

df2 <- data.frame(supp = rep(c("VC", "OJ"), each = 3), 
                 dose = rep(c("D0.5", "D1", "D2"), 2), 
                 len = c(6.8, 15, 33, 4.2, 10, 29.5))

df2
##   supp dose  len
## 1   VC D0.5  6.8
## 2   VC   D1 15.0
## 3   VC   D2 33.0
## 4   OJ D0.5  4.2
## 5   OJ   D1 10.0
## 6   OJ   D2 29.5

now lets load ggplot2 and set a theme

library(ggplot2)

theme_set(
  theme_gray()+
    theme(legend.position = "right")
)

now lets do some line plots. first we will build a function to display all the different line types

generateRLineTypes <- function(){
  oldPar <- par()
  par(font = 2, mar = c(0,0,0,0))
  plot(1, pch="", ylim =c(0,6), xlim=c(0,0.7), axes = FALSE, xlab = "", ylab = "")
  for(i in 0:6) lines(c(0.3, 0.7), c(i,i), lty=i, lwd = 3)
  text(rep(0.1, 6), 0:6, labels = c("0. 'Blank'", "1.'solid'", "2.'dashed'", "3.'dotted'", "4.'dotdash'", "5.'longdash'", "6.'twodash'"))
  par(mar=oldPar$mar, font=oldPar$font)
  
}

generateRLineTypes()

now lets build a basic line plot

p <- ggplot(data = df, aes(x =dose, y = len, group = 1))

p + geom_line() + geom_point()

now lets modify the line type and color

p + geom_line(linetype = "dashed", color = "steelblue") + 
  geom_point(color = "steelblue")

now lets try a step graph, which indicates a threshold type progression

p + geom_step() + geom_point()

now lets move on to making multiple groups. first we’ll create our ggplot object

p <- ggplot(df2, aes(x=dose, y=len, group=supp))

now lets change line types and point shapes by group

p + geom_line(aes(linetype=supp, color=supp)) + 
  geom_point(aes(shape = supp, color = supp)) + 
  scale_color_manual(values = c("red", "blue"))

now lets look at linetypes with a numeric x axis

df3 <- data.frame(supp = rep(c("VC", "OJ"), each = 3), 
                  dose = rep(c("0.5", "1", "2"), 2),
                  len=c(6.8, 15, 33, 4.2, 10, 29.5))

df3
##   supp dose  len
## 1   VC  0.5  6.8
## 2   VC    1 15.0
## 3   VC    2 33.0
## 4   OJ  0.5  4.2
## 5   OJ    1 10.0
## 6   OJ    2 29.5

now lets plot where both axises are treated as continuous labels

df3$dose <- as.numeric(as.vector(df3$dose))
ggplot(data = df3, aes(x=dose, y=len, group=supp, color=supp)) + 
  geom_line() + geom_point()

now lets look at a line graph with having the x axis as dates. we’ll use the built in economics time series for this example

head(economics)
## # A tibble: 6 × 6
##   date         pce    pop psavert uempmed unemploy
##   <date>     <dbl>  <dbl>   <dbl>   <dbl>    <dbl>
## 1 1967-07-01  507. 198712    12.6     4.5     2944
## 2 1967-08-01  510. 198911    12.6     4.7     2945
## 3 1967-09-01  516. 199113    11.9     4.6     2958
## 4 1967-10-01  512. 199311    12.9     4.9     3143
## 5 1967-11-01  517. 199498    12.8     4.7     3066
## 6 1967-12-01  525. 199657    11.8     4.8     3018
ggplot(data = economics, aes(x=date, y=pop)) +
  geom_line()

now lets subset the data

ss <- subset(economics, date > as.Date("2006-1-1"))
ggplot(data=ss, aes(x=date, y=pop)) + geom_line()

we can also change the line size, for instance by another vairable like unemployment

ggplot(data=economics, aes(x=date, y=pop)) +
  geom_line(aes(size=unemploy/pop))

we can also plot multiple time-series data

ggplot(economics, aes(x=date)) + 
  geom_line(aes(y=psavert), color = "darkred") + 
  geom_line(aes(y=uempmed), color = "steelblue", linetype = "twodash")

lastly lets make this into an area plot

ggplot(economics, aes(x=date)) + 
  geom_area(aes(y=psavert), fill = "#999999", 
            color = "#999999", alpha = 0.5) +
  geom_area(aes(y=uempmed), fill = "#E69F00",
            color = "#E69F00", alpha = 0.5)

Ridge Plots

first lets load the required packages

library(ggplot2)
library(ggridges)

now lets load our saved data

?airquality
air <- ggplot(airquality) + aes(Temp, Month, group = Month) + geom_density_ridges()

air
## Picking joint bandwidth of 2.65

now lets add some pazzaz to our graph

library(viridis)
## Loading required package: viridisLite
ggplot(airquality) + aes(Temp, Month, group = Month, fill = ..x..) +
  geom_density_ridges_gradient() + 
  scale_fill_viridis(option = "c", name = "Temp")
## Warning in viridisLite::viridis(256, alpha, begin, end, direction, option):
## Option 'c' does not exist. Defaulting to 'viridis'.
## Warning: The dot-dot notation (`..x..`) was deprecated in ggplot2 3.4.0.
## ℹ Please use `after_stat(x)` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.
## Picking joint bandwidth of 2.65

last thing we will do is create a facet plot for all our data

library(tidyr)

airquality %>%
gather(key = "Measurement", value = "value", Ozone, Solar.R, Wind, Temp) %>%
  ggplot() + aes(value, Month, group = Month) + 
  geom_density_ridges() +
  facet_wrap(~ Measurement, scales = "free")
## Picking joint bandwidth of 11
## Picking joint bandwidth of 40.1
## Picking joint bandwidth of 2.65
## Picking joint bandwidth of 1.44
## Warning: Removed 44 rows containing non-finite values
## (`stat_density_ridges()`).

Density Plots

a density plot is a nice alternative to a histogram

set.seed(1234)

wdata = data.frame(
  sex = factor(rep(c("F", "M"), each = 200)),
  weight = c(rnorm(200, 55), rnorm(200, 58))
)
library(dplyr)
mu <- wdata %>%
  group_by(sex) %>%
summarize(grp.mean=mean(weight))

now lets load the graphing packages

library(ggplot2)
theme_set(
  theme_classic() + 
    theme(legend.position = "right")
)

now lets do the basic plot funcitn, first we will create a ggplot object

d <- ggplot(wdata, aes(x=weight))

now lets do a basic denstity plot

d + geom_density() + 
  geom_vline(aes(xintercept=mean(weight)), linetype = "dashed")

now lets change the y axis to count instead of density

d + geom_density(aes(y=stat(count)), fill = "lightgrey") + 
  geom_vline(aes(xintercept=mean(weight)), linetype = "dashed")

d + geom_density(aes(color = sex)) + 
  scale_color_manual(values = c("darkgrey", "gold"))

lastly, lets fill the density plots

d + geom_density(aes(fill=sex), alpha=0.4)+
  geom_vline(aes(xintercept = grp.mean, color = sex), data = mu, linetype = "dashed") + 
  scale_color_manual(values=c("grey", "gold")) + 
  scale_fill_manual(values = c("grey", "gold"))

Plotly Line Graphs

first lets load our required package

library(plotly)
## 
## Attaching package: 'plotly'
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## The following object is masked from 'package:stats':
## 
##     filter
## The following object is masked from 'package:graphics':
## 
##     layout

lets start with a scatter plot of the orange data set

Orange <- as.data.frame(Orange)

plot_ly(data = Orange, x=~age, y=~circumference)
## No trace type specified:
##   Based on info supplied, a 'scatter' trace seems appropriate.
##   Read more about this trace type -> https://plotly.com/r/reference/#scatter
## No scatter mode specifed:
##   Setting the mode to markers
##   Read more about this attribute -> https://plotly.com/r/reference/#scatter-mode

now lets add some more info

plot_ly(data=Orange, x=~age, y=~circumference,
        color=~Tree, size=~age,
        text=~paste("Tree ID", Tree, "<br>Age", "Circ:", circumference)
)
## No trace type specified:
##   Based on info supplied, a 'scatter' trace seems appropriate.
##   Read more about this trace type -> https://plotly.com/r/reference/#scatter
## No scatter mode specifed:
##   Setting the mode to markers
##   Read more about this attribute -> https://plotly.com/r/reference/#scatter-mode
## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

now lets create a random distribution and add it to our dataframe

trace_1 <- rnorm(35, mean=120, sd=10)
new_data <- data.frame(Orange, trace_1)

we’ll use the random numbers as lines on the graph

plot_ly(data=new_data, x=~age, y=~circumference, color=~Tree, size=~age, text=~paste("Tree ID", Tree, "<br>Age", "Circ:", circumference)) %>%
  add_trace(y=~trace_1, mode='lines') %>%
  add_trace(y=~circumference, mode='markers')
## No trace type specified:
##   Based on info supplied, a 'scatter' trace seems appropriate.
##   Read more about this trace type -> https://plotly.com/r/reference/#scatter
## No trace type specified:
##   Based on info supplied, a 'scatter' trace seems appropriate.
##   Read more about this trace type -> https://plotly.com/r/reference/#scatter
## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

now lets create a graph with the option of showing as a scatter or line, and add labels

plot_ly(data=Orange, x=~age, y=~circumference, color=~Tree, size=~circumference, text=~paste("Tree ID", Tree, "<br>Age", "Circ:", circumference)) %>%
  add_trace(y=~circumference, mode='markers') %>%
  layout(
    title="Plot or Orange data with switchable trace",
    updatemenus=list(
      type='downdrop',
      y=0.8,
      buttons=list(
        list(method='restyle',
             args=list('mode', 'markers'),
             label="Marker"
             ),
        list(method="restyle",
             args=list('mode', 'lines'),
             labels="Lines"
             )
      )
    )
  )
## No trace type specified:
##   Based on info supplied, a 'scatter' trace seems appropriate.
##   Read more about this trace type -> https://plotly.com/r/reference/#scatter
## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

## Warning: `line.width` does not currently support multiple values.

Plotly 3D

first lets load our package

library(plotly)

now lets create random 3d matrix

d <- data.frame(
  x <- seq(1, 10, by = 0.5),
  y <- seq(1, 10, by = 0.5)
)

z <- matrix(rnorm(length(d$x) * length(d$y)), nrow = length(d$x), ncol = length(d$y))

now lets plot our 3D data

plot_ly(d, x=~x, y=~y, z=~z) %>%
  add_surface()

now lets add sone more aspects to it, such as topography

plot_ly(d, x=~x, y=~y, z=~z) %>%
  add_surface(
    contours=list(
      show=TRUE,
      usecolormap=TRUE,
      highlightcolor="FF0000",
      project=list(z=TRUE)
    )
  )

now lets look at a 3d scatterplot

plot_ly(longley, x=~GNP, y=~Population, z=~Employed, marker=list(color=~GNP)) %>%
  add_markers()

Error Bars

first lets load the required package

library(ggplot2)
library(dplyr)
library(plotrix)

theme_set(
  theme_classic() +
    theme(legend.position ='top')
)

lets again use the tooth data for this exercise

df <- ToothGrowth
df$dose <- as.factor(df$dose)

now lets use dplyr for manipulation purposes

df.summary <- df %>%
  group_by(dose) %>%
  summarise(
    sd = sd(len, na.rm = TRUE),
    stderr=std.error(len, na.rm=TRUE),
    len=mean(len),
  )

df.summary
## # A tibble: 3 × 4
##   dose     sd stderr   len
##   <fct> <dbl>  <dbl> <dbl>
## 1 0.5    4.50  1.01   10.6
## 2 1      4.42  0.987  19.7
## 3 2      3.77  0.844  26.1

lets now look at some key functions

  • geom_crossbar() for hollow bars with middle indicated by a horizontal line
  • geom_errorbar() for error bars
  • geom_errorbarh() for horizontal error bars
  • geom_linerange() for drawing an interval represented by a vertical line
  • geom_pointrange() for creating an interval represented by the vertical line but with a point in the middle

lets start by creating a ggplot object

tg <- ggplot(
  df.summary,
  aes(x=dose, y=len, ymin=len, len=-sd, ymax=len+sd)
)

now lets look at the most basic error bars

tg + geom_pointrange()

tg + geom_errorbar(width=0.2) +
  geom_point(size=1.5)

now lets create horizontal bars by manipulating our graph

ggplot(df.summary, aes(x=len, y=dose, xmin=len-sd, xmax=len+sd)) +
  geom_errorbarh(height=0.2)

this just gives you an idea of error bars on the horizontal axis

now lets look at adding jitter points to our data

ggplot(df, aes(dose, len)) + 
  geom_jitter(position = position_jitter(0.2), color = "darkgrey") + 
                geom_pointrange(aes(ymin = len-sd, ymax=len+sd), data = df.summary)

now lets try error bars on a violin plot

ggplot(df, aes(dose, len)) + 
  geom_violin(color = "darkgrey", trim=FALSE) +
  geom_pointrange(aes(ymin=len-sd, ymax=len+sd), data=df.summary)

now how about a line graph

ggplot(df.summary, aes(dose, len)) + 
  geom_line(aes(group =1)) +
  geom_errorbar(aes(ymin=len-stderr, ymax=len+stderr), width=0.2) +
  geom_point(size=2)

now lets make a bar graph with halve error bars

ggplot(df.summary, aes(dose, len)) + 
  geom_col(fill = "lightgrey", color = "black") +
  geom_errorbar(aes(ymin=len, ymax=len+stderr), width=0.2)

you can see that by not specifying wmin = len-stderr we have an essence cut our error bar in half. lets add jitter points to line points, we must use the original data frame for the jitter point and the summary df for the geom layers

ggplot(df, aes(dose, len)) + 
  geom_jitter(position=position_jitter(0.2), color="darkgrey") + 
  geom_line(aes(group=1), data =df.summary) +
  geom_errorbar(aes(ymin=len-stderr, ymax=len+stderr), data=df.summary, width=0.2) +
  geom_point(data=df.summary, size =0.2)

what about adding jitter points to a bar plot

ggplot(df, aes(dose, len)) +
  geom_col(data=df.summary, fill=NA, color="black") +
  geom_jitter(position=position_jitter(0.3), color="darkgrey") +
  geom_errorbar(aes(ymin=len-stderr, ymax=len+stderr), 
                data=df.summary, width=0.2)

what if we wanted to have our error bars per group (oj vs vc)

df.summary2 <- df %>%
  group_by(dose, supp) %>%
  summarize(
    sd=sd(len),
    stderr=std.error(len),
    len=mean(len)
    
  )
## `summarise()` has grouped output by 'dose'. You can override using the
## `.groups` argument.
df.summary2
## # A tibble: 6 × 5
## # Groups:   dose [3]
##   dose  supp     sd stderr   len
##   <fct> <fct> <dbl>  <dbl> <dbl>
## 1 0.5   OJ     4.46  1.41  13.2 
## 2 0.5   VC     2.75  0.869  7.98
## 3 1     OJ     3.91  1.24  22.7 
## 4 1     VC     2.52  0.795 16.8 
## 5 2     OJ     2.66  0.840 26.1 
## 6 2     VC     4.80  1.52  26.1

now you can see we have mean and error for each dose and supp

ggplot(df.summary2, aes(dose, len)) +
  geom_pointrange(
    aes(ymin=len-stderr, ymax=len+stderr, color =supp),
    position=position_dodge(0.3)
  )+
  scale_color_manual(values=c("indianred", "lightblue"))

how about line plots with multiple error bars

ggplot(df.summary2, aes(dose, len)) +
  geom_line(aes(linetype=supp, group=supp)) +
  geom_point()+
  geom_errorbar(aes(ymin=len-stderr, ymax=len+stderr, group=supp), width=0.2)

and the same with bar plot

ggplot(df.summary2, aes(dose, len)) +
  geom_col(aes(fill=supp), position=position_dodge(0.8), width=0.7) +
  geom_errorbar(
    aes(ymin=len-sd, ymax=len+sd, group=supp),
    width=0.2, position=position_dodge(0.8))+
      scale_fill_manual(values=c("indianred", "lightblue"))

now lets add some jitter points

ggplot(df, aes(dose, len, color=supp)) +
  geom_jitter(position=position_dodge(0.2)) +
  geom_line(aes(group=supp), data=df.summary2)+
  geom_point()+
  geom_errorbar(aes(ymin=len-stderr, ymax=len+stderr, group =supp), data=df.summary2, width=0.2)

ggplot(df, aes(dose, len, color=supp)) +
  geom_col(data=df.summary2, position=position_dodge(0.8), width=0.7, fill="white")+
  geom_jitter(
    position=position_jitterdodge(jitter.width=0.2, dodge.width=0.8))+
  geom_errorbar(
    aes(ymin=len-stderr, ymax=len+stderr), data=df.summary2,
    width=0.2, position=position_dodge(0.8))+
  scale_color_manual(values=c("indianred", "lightblue"))+
      theme(legend.position ="top")

ECDF Plots

now lets do an empuiracle cumulative distribution function. this reports anh given number percentile of individual that are aove or bloew that threshoold

set.seed(1234)

wdata = data.frame(
  sex = factor(rep(c("F", "M"), each = 200)),
  weight = c(rnorm(200, 55), rnorm(200, 58)
))

nwo lets look at our data frame

head(wdata, 5)
##   sex   weight
## 1   F 53.79293
## 2   F 55.27743
## 3   F 56.08444
## 4   F 52.65430
## 5   F 55.42912

now lets load our plotting package

library(ggplot2)

theme_set(
  theme_classic() +
    theme(legend.position = "bottom")
)

now lets create our ECDF plot

ggplot(wdata, aes(x=weight)) +
  stat_ecdf(aes(color=sex, linetype=sex),
            geom="step", size=1.5) +
  scale_color_manual(values = c("#00AFBB", "#E7B900")) +
  labs(y="weight")

QQ Plots

now lets take a look at qq plots. these are used to determine the given data follows a normal distribution

#install.packages("ggpubr")

set.seed(1234)

now lets randomly generate some data

wdata=data.frame(
  sex=factor(rep(c("F", "M"), each=200)),
  weight=c(rnorm(200, 55), rnorm(200, 58))
)

now lets set our theme for our graphing with ggplot

library(ggplot2)

theme_set(
  theme_classic() +
    theme(legend.position="top")
)

create a qq plot of the weight

ggplot(wdata, aes(sample=weight)) +
  stat_qq(aes(color=sex)) +
  scale_color_manual(values=c("#0037C2FF", "#FC4E07")) +
  labs(y="weight")

#install.packeges(ggpubr)
library(ggpubr)

ggqqplot(wdata, x="weight", 
         color="sex",
         palettes=c("#0037C2FF", "#FC4E07"),
         ggtheme=theme_pubclean())

now what would a non-normal distribution look like?

#install.packages(mnonr)
library(mnonr)

data2 <- mnonr::mnonr(n=1000, p=2, ms=3, mk=61, Sigma=matrix(c(1, 0.5, 0.5, 1), 2, 2), initial=NULL)

data2 <- as.data.frame(data2)

now lets plot the non-normal data

ggplot(data2, aes(sample=V1)) +
  stat_qq()

ggqqplot(data2, x="V1", 
         palette="#0073C2FF", 
         ggtheme=theme_pubclean())

Facet Plots

lets look at how to put multiple plots together into a single figure

library(ggpubr)
library(ggplot2)

theme_set(
  theme_bw() +
    theme(legend.position="top")
)

first lets. create a nice box plot

df <- ToothGrowth
df$dose <- as.factor(df$dose)

and create a plot obbject

p <- ggplot(df, aes(x=dose, y=len)) +
  geom_boxplot(aes(fill=supp), position=position_dodge(0.9)) +
  scale_fill_manual(values=c("#00AFBB", "#E7B800"))

p

now lets look at the gvplot facit fucntion

p + facet_grid(rows=vars(supp))

now lets do a facet with multiple variables

p+ facet_grid(rows=vars(dose), cols=vars(supp))

p

now lets look at the facet _ wrap function. this allows facet to be placed side by side

p + facet_wrap(vars(dose), ncol=2)

now how do we combine multiple plots using ggarrange lets start by making some of these plots. first we will define a color palette and data

my3cols <- c("#E7B800", "#2E9FDF", "#FC4E07")
ToothGrowth$dose <- as.factor(ToothGrowth$dose)

now lets make some basic plots

p <- ggplot(ToothGrowth, aes(x=dose, y=len))
bxp <- p + geom_boxplot(aes(color=dose)) +
  scale_color_manual(values=my3cols)

ok now lets do a dotplot

dp <- p + geom_dotplot(aes(color=dose, fill=dose), binaxis='y', stackdir='center') +
  scale_color_manual(values = my3cols) +
  scale_fill_manual(values = my3cols)

now lets create a lineplot

lp <- ggplot(economics, aes(x=date, y=psavert))+
  geom_line(color="indianred")
figure <- ggarrange(bxp, dp, lp, labels = c("A", "B", "C"), ncol=2, nrow=2)
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.
figure

this looks great, but we can make it look better

figure2 <- ggarrange(
  lp, 
  ggarrange(bxp, dp, ncol=2, labels=c("B", "C")),
  nrow=2,
  labels="A")
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.
figure2

ok this looks really good, but you’ll notice that there are two legends that are the same

ggarrange(
  bxp, dp, labels=c("A", "B"),
  common.legend=TRUE, legend="bottom")
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.

lastly we should export the plot

ggexport(figure2, filename="facetfigure.pdf")
## file saved to facetfigure.pdf

we can also export multiple plots to a pdf

ggexport(bxp, dp, lp, filename="multi.pdf")
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.
## file saved to multi.pdf

lastly we can export to pdf with multiple pages and multiple columns

ggexport(bxp, dp, lp, bxp, filename="test2.pdf", nrow=2, ncol=1)
## Bin width defaults to 1/30 of the range of the data. Pick better value with
## `binwidth`.
## file saved to test2.pdf

Heat Maps

lets get started with heatmaps

#install.packeges(heatmap3)
library(heatmap3)

now lets get our data

data <- ldeaths

data2 <- do.call(cbind, split(data, cycle(data)))
dimnames(data2) <- dimnames(.preformat.ts(data))

now lets generate a heatmap

heatmap(data2)

heatmap(data2, Rowv= NA, Colv=NA)

now lets playh with the colors

rc <- rainbow(nrow(data2), start = 0, end=0.3)
cc <- rainbow(ncol(data2), start=0, end=0.3)

now lets apply our colors

heatmap(data2, ColSideColors=cc)

library(RColorBrewer)
heatmap(data2, ColSideColors=cc, 
        col = colorRampPalette(brewer.pal(8, "PiYG"))(25))

Missing Values

if you encounter an unusual value in your dtaset, and you simply want to move on to the rest of your analysis, you have two options

drop the entire row with the strange values:

library(dplyr)
library(ggplot2)

diamonds <- diamonds

diamonds2 <- diamonds %>%
  filter(between(y, 3, 20))

in this instance, y is the width of the diamond, so anything under 3 or above 20 is excluded

this isn’t recommended if there is one bad measurement, that doesn’t mean they are all bad

instead, we can replace the unusual values with missing values

diamonds3 <- diamonds %>%
  mutate(y=ifelse(y<3 | y >20, NA, y))

like R, ggplot2 subscribes to the idea that missing values shouldn’t pass silently into the night

ggplot(data = diamonds3, mapping=aes(x=x, y=y)) +
  geom_point()
## Warning: Removed 9 rows containing missing values (`geom_point()`).

if you want to suppress that warning, you can use na.rm =TRUE

ggplot(data=diamonds3, mapping=aes(x=x, y=y)) +
  geom_point(na.rm=TRUE
  )

other times you want to understand what makes observations with missing values different to the observations with recorded values. For example, in the NYCflights13 dataset, missing values in the departure time variable indicate that the flight was cancelled. so you might want to compare the schedule daparture time for the cancelled and non-cancelled times

library(nycflights13)

nycflights13::flights %>%
  mutate(
    cancelled = is.na(dep_time),
    sched_hour=sched_dep_time %/% 100,
    sched_min=sched_dep_time %% 100,
    sched_dep_time=sched_hour + sched_min/60 ) %>%
      ggplot(mapping=aes(sched_dep_time)) +
      geom_freqpoly(mapping=aes(color=cancelled), bindwith=1/4)
## Warning in geom_freqpoly(mapping = aes(color = cancelled), bindwith = 1/4):
## Ignoring unknown parameters: `bindwith`
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

Outliers

What if we want to know what our outliers are? First, we need to load the required packages

library(outliers)
library(ggplot2)
library(readxl)

And reload the dataset because we removed the outliers

Air_data <- read_xlsx("AirQualityUCI.xlsx")

lets create a dfucntion to grubb test to identiyfy all outliers the grubbs test identifies in a univariate dataset that is presumed to come from a normal distrubution

grubbs.flag <- function(x) {
  #lets create a variable called outliers and save nothing to it, and add to the variable
  # as we identify them
  outliers <- NULL
  # we'll create a variable called test to identify which univariates we are testing
  test <- x
  # now using the outliers package, use grubbs.tset to find outliers in our variable
  grubbs.result <- grubbs.test(test)
  pv <- grubbs.result$p.value
  # now lets search through our p values for ones that are outside of 0.5
  while(pv < 0.5) {
    # anythibng with a p value less than 0.5, we add to our empty outlier vector
    outliers <- c(outliers, as.numeric(strsplit(grubbs.result$alternative, "")[[1]][3]))
    # now we want to move those outliers from our test variable 
    test <- x[!x %in% outliers]
    # and run the grubbs test again without the outliers
    grubbs.result <- grubbs.test(test)
    pv <- grubbs.result$p.value
  }
  return(data.frame(x=x, outliers = (x %in% outliers)))
  
}

##```{r}

##identified_outliers <- suppressWarnings(grubbs.flag(Air_data$AH))

##```

now we can create a histogram showoing where the outliers were

##```{r}

##ggplot(grubbs.flag(Air_data\(AH), aes(x=Air_data\)AH, color=outlier, fill=outlier)) + geom_histogram(bindwith=diff(range(Air_data$AH))/30)+ theme_bw()

##```

Covariance

library(ggplot2)

ggplot(data=diamonds, mapping=aes(x=price)) +
  geom_freqpoly(mapping=aes(color=cut), bindwith=500)
## Warning in geom_freqpoly(mapping = aes(color = cut), bindwith = 500): Ignoring
## unknown parameters: `bindwith`
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

its hard to see the difference in distribution because the counts differ so much

ggplot(diamonds) +
  geom_bar(mapping=aes(x=cut))

to make the comparison easier, we need to swap the display on the y-axis. instead of displaying count, we will display density, which is the count standardized so that the area under the curve is one

ggplot(data=diamonds, mapping=aes(x=price, y= ..density..)) +
  geom_freqpoly(mapping=aes(color=cut), bindwith=500)
## Warning in geom_freqpoly(mapping = aes(color = cut), bindwith = 500): Ignoring
## unknown parameters: `bindwith`
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

it appears that fair diamonds (the lowest cut quality) have the highest average price. But, maybe thats because frequency polygons are a little hard to interpret.

another alternative is the box plot this is a type of visual shorthand for a distribution of values

ggplot(data=diamonds, mapping=aes(x=cut, y=price)) +
  geom_boxplot()

we see much less information about the distribution, but this is more compact so we can easily compare them. It supports the counter intuitive finding that better quality diamonds are cheaper on average

lets look at some car data

ggplot(data=mpg, mapping=aes(x=class, y=hwy)) +
  geom_boxplot()

ggplot(data=mpg) +
  geom_boxplot(mapping=aes(x=reorder(class, hwy, FUN=median), y=hwy))

if you have long variable names, you can switch the axis and flip it 90 degrees

ggplot(data=mpg) +
  geom_boxplot(mapping=aes(x=reorder(class, hwy, FUN=median), y=hwy)) +
  coord_flip()

to visualize the correlation between two continuous variables, we can use a scatter plot

ggplot(data=diamonds)+
  geom_point(mapping=aes(x=carat, y=price))

scatter plots become less useful as the size of your data set grows because we get overplot. we can fix this by using the alpha aes

ggplot(data=diamonds) +
  geom_point(mapping=aes(x=carat, y=price), alpha=1/100)

EDA

first lets laod a required library

library(RCurl)
## 
## Attaching package: 'RCurl'
## The following object is masked from 'package:tidyr':
## 
##     complete
library(dplyr)
library(tidyr)

now lets get our data

site <- "https://raw.githubusercontent.com/nytimes/covid-19-data/master/colleges/colleges.csv"

College_Data <- read.csv(site)

first lets use the str function that shows the structure fo the object

str(College_Data)
## 'data.frame':    1948 obs. of  9 variables:
##  $ date      : chr  "2021-05-26" "2021-05-26" "2021-05-26" "2021-05-26" ...
##  $ state     : chr  "Alabama" "Alabama" "Alabama" "Alabama" ...
##  $ county    : chr  "Madison" "Montgomery" "Limestone" "Lee" ...
##  $ city      : chr  "Huntsville" "Montgomery" "Athens" "Auburn" ...
##  $ ipeds_id  : chr  "100654" "100724" "100812" "100858" ...
##  $ college   : chr  "Alabama A&M University" "Alabama State University" "Athens State University" "Auburn University" ...
##  $ cases     : int  41 2 45 2742 220 4 263 137 49 76 ...
##  $ cases_2021: int  NA NA 10 567 80 NA 49 53 10 35 ...
##  $ notes     : chr  "" "" "" "" ...

what if we want to arrange our data set alphanetically by college

alphabetical <- College_Data %>%
  arrange(College_Data)

the glimpse package is another way to preview data

glimpse(College_Data)
## Rows: 1,948
## Columns: 9
## $ date       <chr> "2021-05-26", "2021-05-26", "2021-05-26", "2021-05-26", "20…
## $ state      <chr> "Alabama", "Alabama", "Alabama", "Alabama", "Alabama", "Ala…
## $ county     <chr> "Madison", "Montgomery", "Limestone", "Lee", "Montgomery", …
## $ city       <chr> "Huntsville", "Montgomery", "Athens", "Auburn", "Montgomery…
## $ ipeds_id   <chr> "100654", "100724", "100812", "100858", "100830", "102429",…
## $ college    <chr> "Alabama A&M University", "Alabama State University", "Athe…
## $ cases      <int> 41, 2, 45, 2742, 220, 4, 263, 137, 49, 76, 67, 0, 229, 19, …
## $ cases_2021 <int> NA, NA, 10, 567, 80, NA, 49, 53, 10, 35, 5, NA, 10, NA, 19,…
## $ notes      <chr> "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",…

we can also subset with select

College_Cases <- select(College_Data, college, cases)

we can also filter or subset with the filter function

Louisiana_Cases <- filter(College_Data, state=="Louisiana")

lets filter our small amount of states

South_Cases <- filter(College_Data, state=="Louisiana" | state== "Texas" | state == "Arkansas" | state=="Mississippi")

lets look at some time series data

library(lubridate)
## 
## Attaching package: 'lubridate'
## The following objects are masked from 'package:base':
## 
##     date, intersect, setdiff, union
library(dplyr)
library(ggplot2)
library(gridExtra)
## 
## Attaching package: 'gridExtra'
## The following object is masked from 'package:dplyr':
## 
##     combine
library(scales)
## 
## Attaching package: 'scales'
## The following object is masked from 'package:plotrix':
## 
##     rescale
## The following object is masked from 'package:viridis':
## 
##     viridis_pal

now lets load data

state_site <- "https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-states.csv"

State_Data <- read.csv(state_site)

lets create group_by object using the state column

state_cases <- group_by(State_Data, state)

class(state_cases)
## [1] "grouped_df" "tbl_df"     "tbl"        "data.frame"

how many measurements were made by state? this gives us an idea on when states began reporting

Days_since_first_reported <- tally(state_cases)

lets visualize some data

first lets start off with s ome definitions

Data - obvious - the stuff we want to visualize

Layer - made of geometric elements and requisite statistical information, including geometric objects which represents the plot

scales - used to map values in the data space that is used for creation of values (color, size, shape, etc.)

coordinate system - describes how the data coordinates are mapped together in relation to the plan on the graphic

faceting - how to break up data in to subsets to display multiple types of groups of data

theme - controls the finer points of the display, such as font size and background color

options(repr.plot.width = 6, repr.plot.height=6)

class(College_Data)
## [1] "data.frame"
head(College_Data)
##         date   state     county       city ipeds_id
## 1 2021-05-26 Alabama    Madison Huntsville   100654
## 2 2021-05-26 Alabama Montgomery Montgomery   100724
## 3 2021-05-26 Alabama  Limestone     Athens   100812
## 4 2021-05-26 Alabama        Lee     Auburn   100858
## 5 2021-05-26 Alabama Montgomery Montgomery   100830
## 6 2021-05-26 Alabama     Walker     Jasper   102429
##                           college cases cases_2021 notes
## 1          Alabama A&M University    41         NA      
## 2        Alabama State University     2         NA      
## 3         Athens State University    45         10      
## 4               Auburn University  2742        567      
## 5 Auburn University at Montgomery   220         80      
## 6  Bevill State Community College     4         NA
summary(College_Data)
##      date              state              county              city          
##  Length:1948        Length:1948        Length:1948        Length:1948       
##  Class :character   Class :character   Class :character   Class :character  
##  Mode  :character   Mode  :character   Mode  :character   Mode  :character  
##                                                                             
##                                                                             
##                                                                             
##                                                                             
##    ipeds_id           college              cases          cases_2021    
##  Length:1948        Length:1948        Min.   :   0.0   Min.   :   0.0  
##  Class :character   Class :character   1st Qu.:  32.0   1st Qu.:  23.0  
##  Mode  :character   Mode  :character   Median : 114.5   Median :  65.0  
##                                        Mean   : 363.5   Mean   : 168.1  
##                                        3rd Qu.: 303.0   3rd Qu.: 159.0  
##                                        Max.   :9914.0   Max.   :3158.0  
##                                                         NA's   :337     
##     notes          
##  Length:1948       
##  Class :character  
##  Mode  :character  
##                    
##                    
##                    
## 

now lets take a look at a different data set

iris <- as.data.frame(iris)

class(iris)
## [1] "data.frame"
head(iris)
##   Sepal.Length Sepal.Width Petal.Length Petal.Width Species
## 1          5.1         3.5          1.4         0.2  setosa
## 2          4.9         3.0          1.4         0.2  setosa
## 3          4.7         3.2          1.3         0.2  setosa
## 4          4.6         3.1          1.5         0.2  setosa
## 5          5.0         3.6          1.4         0.2  setosa
## 6          5.4         3.9          1.7         0.4  setosa
summary(iris)
##   Sepal.Length    Sepal.Width     Petal.Length    Petal.Width   
##  Min.   :4.300   Min.   :2.000   Min.   :1.000   Min.   :0.100  
##  1st Qu.:5.100   1st Qu.:2.800   1st Qu.:1.600   1st Qu.:0.300  
##  Median :5.800   Median :3.000   Median :4.350   Median :1.300  
##  Mean   :5.843   Mean   :3.057   Mean   :3.758   Mean   :1.199  
##  3rd Qu.:6.400   3rd Qu.:3.300   3rd Qu.:5.100   3rd Qu.:1.800  
##  Max.   :7.900   Max.   :4.400   Max.   :6.900   Max.   :2.500  
##        Species  
##  setosa    :50  
##  versicolor:50  
##  virginica :50  
##                 
##                 
## 

lets start by creating a scatter plot of the college data

ggplot(data = College_Data, aes(x=cases, y=cases_2021)) +
  geom_point() +
  theme_minimal()
## Warning: Removed 337 rows containing missing values (`geom_point()`).

now lets do the iris data

ggplot(data=iris, aes(Sepal.Width, y=Sepal.Length)) +
  geom_point() +
  theme_minimal()

lets color coordinate our college data

ggplot(data=College_Data, aes(x=cases, y=cases_2021, color=state)) +
  geom_point() +
  theme_minimal()
## Warning: Removed 337 rows containing missing values (`geom_point()`).

lets color coordinate the iris data

ggplot(data=iris, aes(x=Sepal.Width, y=Sepal.Length, color=Species)) +
  geom_point() +
  theme_minimal()

lets run a simple histogram of our louisiana cases

hist(Louisiana_Cases$cases, freq=NULL, density=NULL, breaks=10, xlab="Total Cases", ylab="Frequency", main="Total College Covid-19 Infections (Louisiana")

lets run a simple histogram for the iris data

hist(iris$Sepal.Width, freq = NULL, density=NULL, breaks=10, xlab="Sepal Width", ylab= "Frequnecy", main="Iris Sepal Width")

lets create a ggplot histogram

histogram_college <- ggplot(data=Louisiana_Cases, aes(x=cases))

histogram_college + geom_histogram(bindwith=100, color = "black", aes(fill=county)) +
  xlab("cases") + ylab("Frequency") + ggtitle("Histogram of Covid 19 cases in Louisiana")
## Warning in geom_histogram(bindwith = 100, color = "black", aes(fill = county)):
## Ignoring unknown parameters: `bindwith`
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

lets create a ggplot for the iris data

histogram_iris <- ggplot(data=iris, aes(x=Sepal.Width))

histogram_iris + geom_histogram(bindwith=0.2, color = "black", aes(fill=Species)) +
  xlab("Sepal Width") + ylab("Frequency") + ggtitle("Histogram of Iris Sepal width by Species")
## Warning in geom_histogram(bindwith = 0.2, color = "black", aes(fill =
## Species)): Ignoring unknown parameters: `bindwith`
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

maybe a density plot makes more sense for college data

ggplot(South_Cases)+
  geom_density(aes(x=cases, fill=state), alpha=0.25)

lets do it with the iris data

ggplot(iris)+
  geom_density(aes(x=Sepal.Width, fill=Species, alpha=0.25))

lets look at the plots for iris

ggplot(data=iris, aes(x=Species, y=Sepal.Length, color=Species)) +
  geom_violin()+
  theme_classic()+
  theme(legend.position="none")

now lets try the south data

ggplot(data=South_Cases, aes(x=state, y=cases, color=state)) +
  geom_violin()+
  theme_gray()+
  theme(legend.position = "none")

now lets take a look at risidual plots, this graph displauys the residuals on the vertical axis, and the independent variable on the horizontal axis. In the event tht the points in a residual plot are dispersed in a random manner arround the horizontal axis, it is appropriate to use a linear regression. If they are not randomly dispersed, a non liner model is more appropriate

ggplot(lm(Sepal.Length~Sepal.Width, data=iris)) +
  geom_point(aes(x=.fitted, y=.resid))

now lets look at the southern cases

ggplot(lm(cases~cases_2021, data=South_Cases)) +
  geom_point(aes(x=.fitted, y=.resid))

a linear model is not a good call for the state cases

now lets do some correlations

obesity <- read.csv("Obesity_insurance.csv")
library(tidyr)
library(dplyr)
library(plyr)
## ------------------------------------------------------------------------------
## You have loaded plyr after dplyr - this is likely to cause problems.
## If you need functions from both plyr and dplyr, please load plyr first, then dplyr:
## library(plyr); library(dplyr)
## ------------------------------------------------------------------------------
## 
## Attaching package: 'plyr'
## The following object is masked from 'package:ggpubr':
## 
##     mutate
## The following objects are masked from 'package:plotly':
## 
##     arrange, mutate, rename, summarise
## The following objects are masked from 'package:dplyr':
## 
##     arrange, count, desc, failwith, id, mutate, rename, summarise,
##     summarize

lets look at the structure of the dataset

str(obesity)
## 'data.frame':    1338 obs. of  7 variables:
##  $ age     : int  19 18 28 33 32 31 46 37 37 60 ...
##  $ sex     : chr  "female" "male" "male" "male" ...
##  $ bmi     : num  27.9 33.8 33 22.7 28.9 ...
##  $ children: int  0 1 3 0 0 0 1 3 2 0 ...
##  $ smoker  : chr  "yes" "no" "no" "no" ...
##  $ region  : chr  "southwest" "southeast" "southeast" "northwest" ...
##  $ charges : num  16885 1726 4449 21984 3867 ...

lets look at the column classes

class(obesity)
## [1] "data.frame"

and get a summary of the distribution of the variables

summary(obesity)
##       age            sex                 bmi           children    
##  Min.   :18.00   Length:1338        Min.   :15.96   Min.   :0.000  
##  1st Qu.:27.00   Class :character   1st Qu.:26.30   1st Qu.:0.000  
##  Median :39.00   Mode  :character   Median :30.40   Median :1.000  
##  Mean   :39.21                      Mean   :30.66   Mean   :1.095  
##  3rd Qu.:51.00                      3rd Qu.:34.69   3rd Qu.:2.000  
##  Max.   :64.00                      Max.   :53.13   Max.   :5.000  
##     smoker             region             charges     
##  Length:1338        Length:1338        Min.   : 1122  
##  Class :character   Class :character   1st Qu.: 4740  
##  Mode  :character   Mode  :character   Median : 9382  
##                                        Mean   :13270  
##                                        3rd Qu.:16640  
##                                        Max.   :63770

now lets look at the distribution for the insurance charges

hist(obesity$charges)

we can also get an idea of the distribution using a boxplot

boxplot(obesity$charges)

boxplot(obesity$bmi)

now lets look at the correlations. the cor() comand is used to determine the correlations between two vectors, all of the columns of the data frame, or two data frames. The cov() command, or the otherhand examines the covariance. The cor.test() command carries out a test as to the significance of hte correlation

cor(obesity$charges, obesity$bmi)
## [1] 0.198341

this test uses a spearman Rho correlation, or oyu can use Kendall’s tau by specifying it

cor(obesity$charges, obesity$bmi, method='kendall')
## [1] 0.08252397

this correlation measures strength of a correlation between -1 and 1.

now lets looks at the TrietJen=Moore test. this is used for univariate datasets. The algorith depicts the detection of the outliers in a univariate dataset.

TietjenMoore <- function(dataSeries, k)
{
  n=length(dataSeries)
  # comput the absolute residuals
  r = abs(dataSeries-mean(dataSeries))
  #sort data according to the size of the residual
  df = data.frame(dataSeries, r)
  dfs = df[order(df$r),]
  # create a subset of the data without the largest value
  klarge = c((n-k+1):n)
  subdataSeries = dfs$dataSeries[-klarge]
  #compute the sums of the squares
  ksub=(subdataSeries-mean(subdataSeries)) **2
  all=(df$dataSeries-mean(df$dataSeries)) **2
  #compute the test statistic
  sum(ksub)/sum(all)
  
}

this function helps to compute the absolute residuals and sorts data according to the size of the residuals. later, we will focus on the computation of the sum of squares

FindOutliersTietjenMooreTest <- function(dataSeries, k, alpha=0.5){
  ek <- TietjenMoore(dataSeries, k)
  #compute critical values based on simulation
  test = c(1:10000)
  for (i in 1:10000){
    dataSeriesdataSeries=rnorm(length(dataSeries))
    test[i]=TietjenMoore(dataSeriesdataSeries, k)}
  Talpha=quantile(test, alpha)
  list(T=ek, Talpha=Talpha)
  }

this function helps us compute the critical values based on the simulation data. now lets demonstrate these functions with sample data and the obesity dataset for evaluating this algorithm

the critical region for TietjenMoore is determined by simulation the simulatin is performed by generating a standard normal random sample of size n, and computing the TietjenMoore statistic obtained from the data is compared to this reference distribution. The values of the test statistic is between zero and one. If there are no outliers in the data, the test statistic is close to 1. If there are outliers, the statistic is close to 0. Thus, the test is always a lower, one tailed test regardless of whihc test statistic issued, lk or ek

first we will look at charges

boxplot(obesity$charges)

FindOutliersTietjenMooreTest(obesity$charges, 7)
## $T
## [1] 0.9284433
## 
## $Talpha
##       50% 
## 0.9505333

lets check the bmi

boxplot(obesity$bmi)

FindOutliersTietjenMooreTest(obesity$bmi, 100)
## $T
## [1] 0.6426243
## 
## $Talpha
##       50% 
## 0.6350666

Probability plots

library(ggplot2)
library(tigerstats)
## Loading required package: abd
## Loading required package: nlme
## 
## Attaching package: 'nlme'
## The following object is masked from 'package:dplyr':
## 
##     collapse
## Loading required package: lattice
## Loading required package: grid
## Loading required package: mosaic
## Registered S3 method overwritten by 'mosaic':
##   method                           from   
##   fortify.SpatialPolygonsDataFrame ggplot2
## 
## The 'mosaic' package masks several functions from core packages in order to add 
## additional features.  The original behavior of these functions should not be affected by this.
## 
## Attaching package: 'mosaic'
## The following object is masked from 'package:Matrix':
## 
##     mean
## The following object is masked from 'package:plyr':
## 
##     count
## The following object is masked from 'package:scales':
## 
##     rescale
## The following object is masked from 'package:plotrix':
## 
##     rescale
## The following object is masked from 'package:plotly':
## 
##     do
## The following objects are masked from 'package:dplyr':
## 
##     count, do, tally
## The following object is masked from 'package:ggplot2':
## 
##     stat
## The following objects are masked from 'package:stats':
## 
##     binom.test, cor, cor.test, cov, fivenum, IQR, median, prop.test,
##     quantile, sd, t.test, var
## The following objects are masked from 'package:base':
## 
##     max, mean, min, prod, range, sample, sum
## Welcome to tigerstats!
## To learn more about this package, consult its website:
##  http://homerhanumat.github.io/tigerstats

we will use the probability plot function and their output dnorm: density function of the normal distribution using the density, it is possible to determine the probability of events. Or for example, you may wonder what is the likilihood that a person has an IQ of exactly 140? In this case, you would need to retrieve the density of the IQ ditribition at values 140. The IQ distirbition can be modeled with a mean of 100 and a standard deviation of 15. The corresponding density is:

bmi.mean <- mean(obesity$bmi)
bmi.sd <- sd(obesity$bmi)

lets create a plot of our normal distritbution

bmi.dist <- dnorm(obesity$bmi, mean=bmi.mean, sd=bmi.sd)
bmi.df <- data.frame("bmi" = obesity$bmi, "Density" = bmi.dist)

ggplot(bmi.df, aes(x=bmi, y=Density)) +
  geom_point()

This gives us the probability of every single point occuring now lets use the pnorm function for more info

bmi.dist <- pnorm(obesity$bmi, mean=bmi.mean, sd=bmi.sd)
bmi.df <- data.frame("bmi" = obesity$bmi, "Density" = bmi.dist)

ggplot(bmi.df, aes(x=bmi, y=Density))+
  geom_point()

what if we want to find the proababilty of the bmi being greater than 40 in our distribution?

pp_greater <- function(x) {
  paste(round(100 * pnorm(x,, mean=30.66339, sd=6.09818, lower.tail=FALSE), 2), "%")
}

pp_greater(40)
## [1] "6.29 %"

what about the probability that a bmi is less than 40 in our population

pp_less <- function(x) {
  paste(round(100 * (1-pnorm(x, mean=30.66339, sd=6.09818, lower.tail=FALSE)), 2), "%")
}

pp_less(40)
## [1] "93.71 %"

What if we want to find the area in between?

pnormGC(c(20, 40), region= "between", mean=30.66339, sd=6.09818, graph=TRUE)

## [1] 0.8969428

What if we want to know the quantiles? lets use the qnrom fuinciton. we nueed to assume a normal distribition for this

What bmi represents the lowest 1% of the population

qnorm(0.01, mean=30.66339, sd=6.09818, lower.tail=TRUE)
## [1] 16.4769

what if we want a random sample of values within your distribution?

subset <- rnorm(50, mean=30.66339, sd=6.09818)

hist(subset)

subset2 <- rnorm(5000, mean = 30.66339, sd=6.09818)

hist(subset2)

shapiro-wilk test

so now we can know how to generate a normal distribution, how do we tell if our samples came from a normal distribution?

shapiro.test(obesity$charges[1:5])
## 
##  Shapiro-Wilk normality test
## 
## data:  obesity$charges[1:5]
## W = 0.84164, p-value = 0.1695

you can see here from a small sample size, we would reject the null hypothesis that the samples came from a normal distribution. we can increase the power of the etst by increasing hte sample size

shapiro.test(obesity$charges[1:1000])
## 
##  Shapiro-Wilk normality test
## 
## data:  obesity$charges[1:1000]
## W = 0.8119, p-value < 2.2e-16

now lets check out age

shapiro.test(obesity$age[1:1000])
## 
##  Shapiro-Wilk normality test
## 
## data:  obesity$age[1:1000]
## W = 0.94406, p-value < 2.2e-16

lastly bmi

shapiro.test(obesity$bmi[1:1000])
## 
##  Shapiro-Wilk normality test
## 
## data:  obesity$bmi[1:1000]
## W = 0.99471, p-value = 0.001426

Time series data

first lets load our packages

library(readr)
## 
## Attaching package: 'readr'
## The following object is masked from 'package:scales':
## 
##     col_factor
library(readxl)

Air_data <- read_xlsx("AirQualityUCI.xlsx")

Data - data of measurement time - time of measurement CO(GT) - average hourly CO2 PT08,s1(CO) - tin oxide hourly average sensor response NMHC - average hourly non-metallic hyudrocarbon concentration C6HC - average benzene concentration PT08.s3(NMHC) - titania average hourly sensor response NOx - average hourly NOX concentration NO2 - average hourly NO2 concentration T - temper RH - relative humidity AH - absolute humidity

str(Air_data)
## tibble [9,357 × 15] (S3: tbl_df/tbl/data.frame)
##  $ Date         : POSIXct[1:9357], format: "2004-03-10" "2004-03-10" ...
##  $ Time         : POSIXct[1:9357], format: "1899-12-31 18:00:00" "1899-12-31 19:00:00" ...
##  $ CO(GT)       : num [1:9357] 2.6 2 2.2 2.2 1.6 1.2 1.2 1 0.9 0.6 ...
##  $ PT08.S1(CO)  : num [1:9357] 1360 1292 1402 1376 1272 ...
##  $ NMHC(GT)     : num [1:9357] 150 112 88 80 51 38 31 31 24 19 ...
##  $ C6H6(GT)     : num [1:9357] 11.88 9.4 9 9.23 6.52 ...
##  $ PT08.S2(NMHC): num [1:9357] 1046 955 939 948 836 ...
##  $ NOx(GT)      : num [1:9357] 166 103 131 172 131 89 62 62 45 -200 ...
##  $ PT08.S3(NOx) : num [1:9357] 1056 1174 1140 1092 1205 ...
##  $ NO2(GT)      : num [1:9357] 113 92 114 122 116 96 77 76 60 -200 ...
##  $ PT08.S4(NO2) : num [1:9357] 1692 1559 1554 1584 1490 ...
##  $ PT08.S5(O3)  : num [1:9357] 1268 972 1074 1203 1110 ...
##  $ T            : num [1:9357] 13.6 13.3 11.9 11 11.2 ...
##  $ RH           : num [1:9357] 48.9 47.7 54 60 59.6 ...
##  $ AH           : num [1:9357] 0.758 0.725 0.75 0.787 0.789 ...
library(tidyr)
library(dplyr)
library(lubridate)
library(hms)
## 
## Attaching package: 'hms'
## The following object is masked from 'package:lubridate':
## 
##     hms
library(ggplot2)

lets get rid of the date in the time column

Air_data$Time <- as_hms(Air_data$Time)

glimpse(Air_data)
## Rows: 9,357
## Columns: 15
## $ Date            <dttm> 2004-03-10, 2004-03-10, 2004-03-10, 2004-03-10, 2004-…
## $ Time            <time> 18:00:00, 19:00:00, 20:00:00, 21:00:00, 22:00:00, 23:…
## $ `CO(GT)`        <dbl> 2.6, 2.0, 2.2, 2.2, 1.6, 1.2, 1.2, 1.0, 0.9, 0.6, -200…
## $ `PT08.S1(CO)`   <dbl> 1360.00, 1292.25, 1402.00, 1375.50, 1272.25, 1197.00, …
## $ `NMHC(GT)`      <dbl> 150, 112, 88, 80, 51, 38, 31, 31, 24, 19, 14, 8, 16, 2…
## $ `C6H6(GT)`      <dbl> 11.881723, 9.397165, 8.997817, 9.228796, 6.518224, 4.7…
## $ `PT08.S2(NMHC)` <dbl> 1045.50, 954.75, 939.25, 948.25, 835.50, 750.25, 689.5…
## $ `NOx(GT)`       <dbl> 166, 103, 131, 172, 131, 89, 62, 62, 45, -200, 21, 16,…
## $ `PT08.S3(NOx)`  <dbl> 1056.25, 1173.75, 1140.00, 1092.00, 1205.00, 1336.50, …
## $ `NO2(GT)`       <dbl> 113, 92, 114, 122, 116, 96, 77, 76, 60, -200, 34, 28, …
## $ `PT08.S4(NO2)`  <dbl> 1692.00, 1558.75, 1554.50, 1583.75, 1490.00, 1393.00, …
## $ `PT08.S5(O3)`   <dbl> 1267.50, 972.25, 1074.00, 1203.25, 1110.00, 949.25, 73…
## $ T               <dbl> 13.600, 13.300, 11.900, 11.000, 11.150, 11.175, 11.325…
## $ RH              <dbl> 48.875, 47.700, 53.975, 60.000, 59.575, 59.175, 56.775…
## $ AH              <dbl> 0.7577538, 0.7254874, 0.7502391, 0.7867125, 0.7887942,…
plot(Air_data$AH, Air_data$RH, main="Humidity Analysis", xlab="Absolute Humidity", ylab="Relative Humidity")

notice we have an outlier in our data

t.test(Air_data$RH, Air_data$AH)
## 
##  Welch Two Sample t-test
## 
## data:  Air_data$RH and Air_data$AH
## t = 69.62, df = 17471, p-value < 2.2e-16
## alternative hypothesis: true difference in means is not equal to 0
## 95 percent confidence interval:
##  45.01707 47.62536
## sample estimates:
## mean of x mean of y 
## 39.483611 -6.837604

Text Mining

First we will look at the unnest_token functin

Lets start by looking at an Emily Dickenson package

text <- c("Because I could not stop from Death = ", 
          "He kindly stopped for me - ",
          "The Carriage held but just Ourselves - ",
          "and Immortality")

text
## [1] "Because I could not stop from Death = " 
## [2] "He kindly stopped for me - "            
## [3] "The Carriage held but just Ourselves - "
## [4] "and Immortality"

This is a typical character vector that we might want to analyze. In order to turn it into a tidytext dataset we need to put it into a dataframe

library(dplyr)

text_df <- tibble(line = 1:4, text=text)

text_df
## # A tibble: 4 × 2
##    line text                                     
##   <int> <chr>                                    
## 1     1 "Because I could not stop from Death = " 
## 2     2 "He kindly stopped for me - "            
## 3     3 "The Carriage held but just Ourselves - "
## 4     4 "and Immortality"

Reminder: a tibble is a modern clas sof data frame within R. its available in the dplyr and tibble packes, that has a convenient print method, will not convert strongs to factors, and does not use row names. Tibbles are greta for use with tidy tools

next we will use the unest_tokens function

First we have to output column name that will be created as the text is unnested into it

library(tidytext)

text_df %>%
  unnest_tokens(word, text)
## # A tibble: 20 × 2
##     line word       
##    <int> <chr>      
##  1     1 because    
##  2     1 i          
##  3     1 could      
##  4     1 not        
##  5     1 stop       
##  6     1 from       
##  7     1 death      
##  8     2 he         
##  9     2 kindly     
## 10     2 stopped    
## 11     2 for        
## 12     2 me         
## 13     3 the        
## 14     3 carriage   
## 15     3 held       
## 16     3 but        
## 17     3 just       
## 18     3 ourselves  
## 19     4 and        
## 20     4 immortality

lets use the janeustenr package to analyze some Jane Austen texts. there are 6 books in this package

library(janeaustenr)
library(dplyr)
library(stringr)

original_books <- austen_books() %>%
  group_by(book) %>%
  dplyr::mutate(linenumber = row_number(),
         chapter = cumsum(str_detect(text, regex("^chapter [\\divxlc]",
                                                 ignore_case = TRUE)))) %>%
  ungroup()

original_books
## # A tibble: 73,422 × 4
##    text                    book                linenumber chapter
##    <chr>                   <fct>                    <int>   <int>
##  1 "SENSE AND SENSIBILITY" Sense & Sensibility          1       0
##  2 ""                      Sense & Sensibility          2       0
##  3 "by Jane Austen"        Sense & Sensibility          3       0
##  4 ""                      Sense & Sensibility          4       0
##  5 "(1811)"                Sense & Sensibility          5       0
##  6 ""                      Sense & Sensibility          6       0
##  7 ""                      Sense & Sensibility          7       0
##  8 ""                      Sense & Sensibility          8       0
##  9 ""                      Sense & Sensibility          9       0
## 10 "CHAPTER 1"             Sense & Sensibility         10       1
## # ℹ 73,412 more rows

To work with this as a tidy dataset, we need to restructure it in the one-token-per-row-format, which as we saw earlier is done with the unnest_tokens() function

library(tidytext)
tidy_books <- original_books %>%
  unnest_tokens(word, text)

tidy_books
## # A tibble: 725,055 × 4
##    book                linenumber chapter word       
##    <fct>                    <int>   <int> <chr>      
##  1 Sense & Sensibility          1       0 sense      
##  2 Sense & Sensibility          1       0 and        
##  3 Sense & Sensibility          1       0 sensibility
##  4 Sense & Sensibility          3       0 by         
##  5 Sense & Sensibility          3       0 jane       
##  6 Sense & Sensibility          3       0 austen     
##  7 Sense & Sensibility          5       0 1811       
##  8 Sense & Sensibility         10       1 chapter    
##  9 Sense & Sensibility         10       1 1          
## 10 Sense & Sensibility         13       1 the        
## # ℹ 725,045 more rows

this function uses the tokenizers package to seperaate each line of text in the original dataframe into tokens the default tokenizing is for words, but other options including characters, n-grams, sentences, lines, or paraghrapsh can be used now that the data is in one word per row format, we can manipulate it with tools like dplyr often in text, we will want to remove stop words. stop words that are not useful for an analysis.

we can remove stop words with an anti_join()

data(stop_words)

tidy_books <- tidy_books %>%
  anti_join(stop_words)
## Joining with `by = join_by(word)`

the stop words data set in the tidy text package contains stop words from three lexicons. we can use them all together and have three, or filter to only use one set of stop words if thats more appropriate for your analysis

tidy_books %>%
  count(word, sort = TRUE)
## # A tibble: 13,914 × 2
##    word       n
##    <chr>  <int>
##  1 miss    1855
##  2 time    1337
##  3 fanny    862
##  4 dear     822
##  5 lady     817
##  6 sir      806
##  7 day      797
##  8 emma     787
##  9 sister   727
## 10 house    699
## # ℹ 13,904 more rows

because weve been using tidy tools, our word counts are stored in a tidy data frame. this allows us to pipe this directly into ggplot2. for example, we can create a visualization of the most common words

library(ggplot2)

tidy_books %>%
  count(word, sort = TRUE) %>%
  filter(n >600) %>%
  mutate(word, reorder(word, n)) %>%
  ggplot(aes(n, word)) +
  geom_col() +
  labs(y = NULL, x = "word count")

the gutenberg package

this package provides access to the public domain works from the gutenberg project. this package includes tools for both downloading books and a complete dataset of project gutenberg metadata that can be used to find works of interest. we will mostly use the function gutenberg_download()

word frequencies

lets looks at some biology texts, starting with darwin

the voyage of the Beagle - 944 on the origin of species by the means of natural selection -1228 the expression of emotions in man and animals - 1227 the descent of man, and selection in relation to sex - 2300

we can access these works using hte gutenberg_download() adn the project gutenberg IDnumbers

library(gutenbergr)

darwin <- gutenberg_download(c(944, 1227, 1228, 2300), mirror = "http://www.mirrorservice.org/sites/ftp.ibiblio.org/pub/docs/books/gutenberg/")
## Warning: ! Could not download a book at
##   http://www.mirrorservice.org/sites/ftp.ibiblio.org/pub/docs/books/gutenberg//1/2/2/1228/1228.zip.
## ℹ The book may have been archived.
## ℹ Alternatively, You may need to select a different mirror.
## → See https://www.gutenberg.org/MIRRORS.ALL for options.

Lets break these into tokens

tidy_darwin <- darwin %>%
  unnest_tokens(word, text) %>%
  anti_join (stop_words)
## Joining with `by = join_by(word)`

lets check out the most common darwin words

tidy_darwin %>%
  count(word, sort = TRUE)
## # A tibble: 22,315 × 2
##    word        n
##    <chr>   <int>
##  1 male     1620
##  2 species  1452
##  3 males    1286
##  4 female   1173
##  5 birds    1124
##  6 sexes    1062
##  7 animals  1013
##  8 females  1003
##  9 sexual    749
## 10 vol       712
## # ℹ 22,305 more rows

now lets get some work from Thomas Hunt Morgan who is credited with discovering chromosomes

Regeneration - 57198 The genetic and operative evidence relating to secondary sexual characteristics - 57460 Evolution and adaptation - 63540

morgan <- gutenberg_download(c(57198, 57460, 63540), mirror = "http://www.mirrorservice.org/sites/ftp.ibiblio.org/pub/docs/books/gutenberg/")

lets tokenize this

tidy_morgan <- morgan %>%
  unnest_tokens(word, text) %>%
  anti_join(stop_words)
## Joining with `by = join_by(word)`

what are the most common words?

tidy_morgan %>%
  count(word, sort = TRUE)
## # A tibble: 13,855 × 2
##    word             n
##    <chr>        <int>
##  1 species        869
##  2 regeneration   814
##  3 piece          702
##  4 cut            669
##  5 male           668
##  6 forms          631
##  7 selection      604
##  8 cells          576
##  9 found          552
## 10 development    546
## # ℹ 13,845 more rows

lastly lets look at thomas henry huxley

evidence as to mans place in nature - 2931 on the receptor of the origin of species - 2089 evolution and ethics and other essays - 2940 science and culture, and other essays - 52344

huxley <- gutenberg_download(c(2931, 2940, 52344), mirror = "http://www.mirrorservice.org/sites/ftp.ibiblio.org/pub/docs/books/gutenberg/")
tidy_huxley <- huxley %>%
  unnest_tokens(word, text)%>%
  anti_join(stop_words)
## Joining with `by = join_by(word)`
tidy_huxley %>%
  count(word, sort = TRUE)
## # A tibble: 15,772 × 2
##    word          n
##    <chr>     <int>
##  1 nature      316
##  2 species     295
##  3 life        275
##  4 time        265
##  5 existence   251
##  6 knowledge   228
##  7 animals     220
##  8 animal      213
##  9 natural     199
## 10 world       190
## # ℹ 15,762 more rows

now lets calculate the frequency for each word for the workds of darwin, morgan, and huxley by binding the frames together

library(tidyr)

frequency <- bind_rows(mutate(tidy_morgan, author = "Thomas Hunt Morgan"),
                       mutate(tidy_darwin, author = "Charles Darwin"),
                       mutate(tidy_huxley, author = "Thomas Henry Huxley")) %>%
  mutate(word = str_extract(word, "[a-z]+")) %>%
  count(author, word) %>%
  group_by(author) %>%
  mutate(proportion = n/ sum(n)) %>%
  select(-n) %>%
  pivot_wider(names_from = author, values_from = proportion) %>%
  pivot_longer('Thomas Hunt Morgan': 'Charles Darwin', names_to = "author", values_to = "proportion")

frequency
## # A tibble: 92,787 × 3
##    word    author               proportion
##    <chr>   <chr>                     <dbl>
##  1 a       Thomas Hunt Morgan   0.000589  
##  2 a       Thomas Henry Huxley  0.0000151 
##  3 a       Charles Darwin       0.0000452 
##  4 ab      Thomas Hunt Morgan   0.0000473 
##  5 ab      Thomas Henry Huxley  0.0000172 
##  6 ab      Charles Darwin       0.00000430
##  7 abaiss  Thomas Hunt Morgan  NA         
##  8 abaiss  Thomas Henry Huxley NA         
##  9 abaiss  Charles Darwin       0.00000430
## 10 abandon Thomas Hunt Morgan   0.00000215
## # ℹ 92,777 more rows

now we need to change the table so that each author has its own row

frequency2 <- pivot_wider(frequency, names_from = author, values_from = proportion)

frequency2
## # A tibble: 30,929 × 4
##    word        `Thomas Hunt Morgan` `Thomas Henry Huxley` `Charles Darwin`
##    <chr>                      <dbl>                 <dbl>            <dbl>
##  1 a                     0.000589              0.0000151        0.0000452 
##  2 ab                    0.0000473             0.0000172        0.00000430
##  3 abaiss               NA                    NA                0.00000430
##  4 abandon               0.00000215           NA                0.00000215
##  5 abandoned             0.00000430            0.00000430       0.00000215
##  6 abashed              NA                    NA                0.00000215
##  7 abatement            NA                     0.00000430       0.00000215
##  8 abbot                NA                     0.00000430       0.00000215
##  9 abbott               NA                    NA                0.00000430
## 10 abbreviated          NA                    NA                0.00000860
## # ℹ 30,919 more rows

now lets plot

library(scales)

ggplot(frequency2, aes(x= `Charles Darwin`, y = `Thomas Hunt Morgan`), color = abs(- 'Charles Darwin' - 'Thomas Hunt Morgan')) +
  geom_abline(color = "gray40", lty =2) +
  geom_jitter(alpha = 0.1, size = 2.5, width=0.3, height = 0.3) +
  geom_text(aes(label = word), check_overlap = TRUE, vjust = 1.5) +
  scale_x_log10(labels = percent_format())+
  scale_y_log10(labels = percent_format())+
  scale_color_gradient(limits=c(0, 0.001),
                       low = "darkslategray4", high = "gray75")+
  theme(legend.position="none")+
  labs(y="Thomas Hunt Morgan", x = "Charles Darwin")
## Warning: Removed 23812 rows containing missing values (`geom_point()`).
## Warning: Removed 23813 rows containing missing values (`geom_text()`).

library(scales)

ggplot(frequency2, aes(x= `Charles Darwin`, y = `Thomas Henry Huxley`), color = abs(- 'Charles Darwin' - 'Thomas Henry Huxley')) +
  geom_abline(color = "gray40", lty =2) +
  geom_jitter(alpha = 0.1, size = 2.5, width=0.3, height = 0.3) +
  geom_text(aes(label = word), check_overlap = TRUE, vjust = 1.5) +
  scale_x_log10(labels = percent_format())+
  scale_y_log10(labels = percent_format())+
  scale_color_gradient(limits=c(0, 0.001),
                       low = "darkslategray4", high = "gray75")+
  theme(legend.position="none")+
  labs(y="Thomas Henry Huxley", x = "Charles Darwin")
## Warning: Removed 22785 rows containing missing values (`geom_point()`).
## Warning: Removed 22786 rows containing missing values (`geom_text()`).

library(scales)

ggplot(frequency2, aes(x= `Thomas Hunt Morgan`, y = `Thomas Henry Huxley`), color = abs(- 'Thomas Hunt Morgan' - 'Thomas Henry Huxley')) +
  geom_abline(color = "gray40", lty =2) +
  geom_jitter(alpha = 0.1, size = 2.5, width=0.3, height = 0.3) +
  geom_text(aes(label = word), check_overlap = TRUE, vjust = 1.5) +
  scale_x_log10(labels = percent_format())+
  scale_y_log10(labels = percent_format())+
  scale_color_gradient(limits=c(0, 0.001),
                       low = "darkslategray4", high = "gray75")+
  theme(legend.position="none")+
  labs(y="Thomas Henry Huxley", x = "Thomas Hunt Morgan")
## Warning: Removed 25084 rows containing missing values (`geom_point()`).
## Warning: Removed 25085 rows containing missing values (`geom_text()`).

Sentiments

The sentiment data set

there are a variety of methods that exist for evaluating the opinion or emotion of hte text

AFFIN Bing nrc

bing categorizes words in a binary fashion into positive or negative nrd categorizes into positive, negative, anger, anticipation, disgust, fear, joy, sadness, surprise, amd trust AFFIN assigns a score between -5 and 5 with negative indicating negative sentiment and psotive

the function get_sentiments() allows us to get the specific sentiment lexicon with the masuere for each one

library(tidytext)
library(textdata)

afinn <- get_sentiments("afinn")

afinn
## # A tibble: 2,477 × 2
##    word       value
##    <chr>      <dbl>
##  1 abandon       -2
##  2 abandoned     -2
##  3 abandons      -2
##  4 abducted      -2
##  5 abduction     -2
##  6 abductions    -2
##  7 abhor         -3
##  8 abhorred      -3
##  9 abhorrent     -3
## 10 abhors        -3
## # ℹ 2,467 more rows
bing <- get_sentiments("bing")

bing
## # A tibble: 6,786 × 2
##    word        sentiment
##    <chr>       <chr>    
##  1 2-faces     negative 
##  2 abnormal    negative 
##  3 abolish     negative 
##  4 abominable  negative 
##  5 abominably  negative 
##  6 abominate   negative 
##  7 abomination negative 
##  8 abort       negative 
##  9 aborted     negative 
## 10 aborts      negative 
## # ℹ 6,776 more rows
nrc <- get_sentiments("nrc")

nrc
## # A tibble: 13,872 × 2
##    word        sentiment
##    <chr>       <chr>    
##  1 abacus      trust    
##  2 abandon     fear     
##  3 abandon     negative 
##  4 abandon     sadness  
##  5 abandoned   anger    
##  6 abandoned   fear     
##  7 abandoned   negative 
##  8 abandoned   sadness  
##  9 abandonment anger    
## 10 abandonment fear     
## # ℹ 13,862 more rows

these libraries were created either using crowdourcing or cloud computing/ai like amazon mechanical turk or by labor of one off the authors, and then validated with crowdsourcing

library(gutenbergr)
library(dplyr)
library(stringr)

darwin <- gutenberg_download(c(944, 1227, 1228, 2300), mirror = "https://gutenberg.nabasny.com/")
## Warning: ! Could not download a book at
##   https://gutenberg.nabasny.com//1/2/2/1228/1228.zip.
## ℹ The book may have been archived.
## ℹ Alternatively, You may need to select a different mirror.
## → See https://www.gutenberg.org/MIRRORS.ALL for options.
tidy_books <- darwin %>%
  group_by(gutenberg_id) %>%
  dplyr::mutate(linenumber = row_number(), chapter = cumsum(str_detect(text, regex("^chapter [\\divxlc]", ignore_case = TRUE)))) %>%
  ungroup() %>%
  unnest_tokens(word, text)

tidy_books
## # A tibble: 629,573 × 4
##    gutenberg_id linenumber chapter word   
##           <int>      <int>   <int> <chr>  
##  1          944          1       0 the    
##  2          944          1       0 voyage 
##  3          944          1       0 of     
##  4          944          1       0 the    
##  5          944          1       0 beagle 
##  6          944          1       0 by     
##  7          944          2       0 charles
##  8          944          2       0 darwin 
##  9          944          8       0 about  
## 10          944          8       0 the    
## # ℹ 629,563 more rows

lets add books name instead of GID

colnames(tidy_books)[1] <- "book"

tidy_books$book[tidy_books$book == 944] <- "The Voyage of the Beagle"
tidy_books$book[tidy_books$book == 1227] <- "The Expression of the Emotions in Man and Animals"
tidy_books$book[tidy_books$book == 1228] <- "On the Origin of Species By Means of Natural Selection"
tidy_books$book[tidy_books$book == 2300] <- "The Descent of Man, and Selection in Relation to Sex"

tidy_books
## # A tibble: 629,573 × 4
##    book                     linenumber chapter word   
##    <chr>                         <int>   <int> <chr>  
##  1 The Voyage of the Beagle          1       0 the    
##  2 The Voyage of the Beagle          1       0 voyage 
##  3 The Voyage of the Beagle          1       0 of     
##  4 The Voyage of the Beagle          1       0 the    
##  5 The Voyage of the Beagle          1       0 beagle 
##  6 The Voyage of the Beagle          1       0 by     
##  7 The Voyage of the Beagle          2       0 charles
##  8 The Voyage of the Beagle          2       0 darwin 
##  9 The Voyage of the Beagle          8       0 about  
## 10 The Voyage of the Beagle          8       0 the    
## # ℹ 629,563 more rows

now that we have a tidy format one word per row, we are ready for sentiment analysis, first lets use nrc

nrc_joy <- get_sentiments("nrc") %>%
  filter(sentiment == "joy")

tidy_books %>%
  filter(book == "The Voyage of the Beagle") %>%
  inner_join(nrc_joy) %>%
  count(word, sort = TRUE)
## Joining with `by = join_by(word)`
## # A tibble: 277 × 2
##    word           n
##    <chr>      <int>
##  1 found        301
##  2 good         161
##  3 remarkable   114
##  4 green         95
##  5 kind          92
##  6 tree          86
##  7 present       85
##  8 food          78
##  9 beautiful     61
## 10 elevation     60
## # ℹ 267 more rows

we can also examine how senitment changes throughout a work

library(tidyr)

Charles_Darwin_sentiment <- tidy_books %>%
  inner_join(get_sentiments("bing")) %>%
  count(book, index = linenumber %% 80, sentiment) %>%
  pivot_wider(names_from = sentiment, values_from = n, values_fill = 0) %>%
  dplyr::mutate(sentiment = positive - negative)
## Joining with `by = join_by(word)`

now lets plot it

library(ggplot2)

ggplot(Charles_Darwin_sentiment, aes(index, sentiment, fill=book)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~book, ncol = 2, scales = "free_x")

lets compare the three sentiment dictions

there are several options for sentiment lexicons, you might want some more info on which is appropriate for your purpose. here we will use all three of our dictionaries and examine how the sentiment changes across the art of TVOTB

library(tidyr)

voyage <- tidy_books %>%
  filter(book == "The Voyage of the Beagle")

voyage
## # A tibble: 208,118 × 4
##    book                     linenumber chapter word   
##    <chr>                         <int>   <int> <chr>  
##  1 The Voyage of the Beagle          1       0 the    
##  2 The Voyage of the Beagle          1       0 voyage 
##  3 The Voyage of the Beagle          1       0 of     
##  4 The Voyage of the Beagle          1       0 the    
##  5 The Voyage of the Beagle          1       0 beagle 
##  6 The Voyage of the Beagle          1       0 by     
##  7 The Voyage of the Beagle          2       0 charles
##  8 The Voyage of the Beagle          2       0 darwin 
##  9 The Voyage of the Beagle          8       0 about  
## 10 The Voyage of the Beagle          8       0 the    
## # ℹ 208,108 more rows

lets again use interger division to define larger sections of the tex tthat span multipole lines, and we can use the same pattern with ‘count’ pivot() and mutate() to find the net sentiment in each of these sectiosn of text

affin <- voyage %>%
  inner_join(get_sentiments("afinn")) %>%
  group_by(index = linenumber %% 80) %>%
  summarise(sentiment = sum(value)) %>%
  dplyr::mutate(method = "AFINN")
## Joining with `by = join_by(word)`
bing_and_nrc <- bind_rows(
  voyage %>%
    inner_join(get_sentiments("bing")) %>%
    dplyr::mutate(method = "Bing et al."),
  voyage %>%
    inner_join(get_sentiments("nrc")%>%
                 filter(sentiment %in% c("positive", "negative"))
               ) %>%
    dplyr::mutate(method = "NRC")) %>%
  count(method, index = linenumber %% 80, sentiment) %>%
  pivot_wider(names_from = sentiment,
              values_from = n,
              values_fill = 0) %>%
  dplyr::mutate(sentiment = positive - negative)
## Joining with `by = join_by(word)`
## Joining with `by = join_by(word)`
## Warning in inner_join(., get_sentiments("nrc") %>% filter(sentiment %in% : Detected an unexpected many-to-many relationship between `x` and `y`.
## ℹ Row 1154 of `x` matches multiple rows in `y`.
## ℹ Row 4245 of `y` matches multiple rows in `x`.
## ℹ If a many-to-many relationship is expected, set `relationship =
##   "many-to-many"` to silence this warning.

we can now estimate the net sentiment (postive - negative) ni each chunk of the novel text for each lexicon. lets bind them all together and visualize with ggplot

bind_rows(afinn, bing_and_nrc) %>%
  ggplot(aes(index, sentiment, fill = method)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~method, ncol = 1, scales = "free_y")
## Warning: Removed 2477 rows containing missing values (`position_stack()`).

lets look at the counts in eaceh directory

get_sentiments("nrc") %>%
  filter(sentiment %in% c("positive", "negative")) %>%
  count(sentiment)
## # A tibble: 2 × 2
##   sentiment     n
##   <chr>     <int>
## 1 negative   3316
## 2 positive   2308
get_sentiments("bing") %>%
  count(sentiment)
## # A tibble: 2 × 2
##   sentiment     n
##   <chr>     <int>
## 1 negative   4781
## 2 positive   2005
bing_word_counts <- tidy_books %>%
  inner_join(get_sentiments("bing")) %>%
  count(word, sentiment, sort = TRUE) %>%
  ungroup()
## Joining with `by = join_by(word)`
bing_word_counts
## # A tibble: 2,421 × 3
##    word       sentiment     n
##    <chr>      <chr>     <int>
##  1 great      positive    964
##  2 like       positive    705
##  3 well       positive    701
##  4 good       positive    384
##  5 doubt      negative    306
##  6 respect    positive    266
##  7 wild       negative    266
##  8 bright     positive    252
##  9 remarkable positive    247
## 10 savages    negative    217
## # ℹ 2,411 more rows

this can be shown visually

bing_word_counts %>%
  group_by(sentiment) %>%
  slice_max(n, n = 10) %>%
  ungroup() %>%
  dplyr::mutate(word = reorder(word, n)) %>%
  ggplot(aes(n, word, fill = sentiment)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~sentiment, scale = "free_y")+
  labs(x = "Contribution to Sentiment", y=NULL)

lets spot an anomaly in the dataset

custom_stop_words <- bind_rows(tibble(word = c("wild", "dark", "great", "like"), lexicon = c("custom")), stop_words)

custom_stop_words
## # A tibble: 1,153 × 2
##    word      lexicon
##    <chr>     <chr>  
##  1 wild      custom 
##  2 dark      custom 
##  3 great     custom 
##  4 like      custom 
##  5 a         SMART  
##  6 a's       SMART  
##  7 able      SMART  
##  8 about     SMART  
##  9 above     SMART  
## 10 according SMART  
## # ℹ 1,143 more rows

word clouds:

we can see that tidy text mining and sentiment analysis works well with ggplot2, but having our data in tidy format leads to other nice graphing techniques

lets use the wordcloud package

library(wordcloud)

tidy_books %>%
  anti_join(stop_words) %>%
  count(word) %>%
  with(wordcloud(word, n, max.words = 100))
## Joining with `by = join_by(word)`
## Warning in wordcloud(word, n, max.words = 100): female could not be fit on
## page. It will not be plotted.
## Warning in wordcloud(word, n, max.words = 100): birds could not be fit on page.
## It will not be plotted.

lets also look at comparison.cloud which way require turning the dataframe into a matrix we can change to matrix using the acast() function

library(reshape2)
## 
## Attaching package: 'reshape2'
## The following object is masked from 'package:tigerstats':
## 
##     tips
## The following object is masked from 'package:tidyr':
## 
##     smiths
tidy_books %>%
  inner_join(get_sentiments("bing")) %>%
  count(word, sentiment, sort = TRUE) %>%
  acast(word ~ sentiment, value.var = "n", fill = 0) %>%
  comparison.cloud(colors = c("gray20", "gray80"), max.words = 100)
## Joining with `by = join_by(word)`

looking at units beyond words

lots of useful work can be done by tokenzing at the word level, but sometimes its nice to look at different units of text. for example, we can look beyond unigrams

#```{r}

bingnegative <- get_sentiments(“bing”) %>% filter(sentiment == “negative”)

wordcounts <- tidy_books %>% group_by(book, chapter) %>% summarize(words = n())

tidy_books %>% semi_join(bingnegative) %>% group_by(book, chapter) %>% summarize(negativewords = n()) %>% left_join(wordcounts, by = c(“book”, “chapter”)) %>% dplyr::mutate(ratio = negativewords/words) %>% filter(chapter !=0) %>% slice_max(ratio, n = 1) %>% ungroup()

#```

N Grams

so far weve only looked at single words, bu many intersting analyses are based on the relationship beteween wrods

lets look at some methods of tidytext for calculating and visualizing word relationship

library(dplyr)
library(tidytext)

darwin_books <- gutenberg_download(c(944, 1227, 1228, 2300), mirror = "https://gutenberg.nabasny.com/")
## Warning: ! Could not download a book at
##   https://gutenberg.nabasny.com//1/2/2/1228/1228.zip.
## ℹ The book may have been archived.
## ℹ Alternatively, You may need to select a different mirror.
## → See https://www.gutenberg.org/MIRRORS.ALL for options.
colnames(darwin_books)[1] <- "book"

darwin_books$book[darwin_books$book == 944] <- "The Voyage of the Beagle"
darwin_books$book[darwin_books$book == 1227] <- "The Expression of the Emotions in Man and Animals"
darwin_books$book[darwin_books$book == 1228] <- "On the Origin of Species By Means of Natural Selection"
darwin_books$book[darwin_books$book == 2300] <- "The Descent of Man, and Selection in Relation to Sex"

darwin_bigrams <- darwin_books %>%
  unnest_tokens(bigram, text, token = "ngrams", n=2)

darwin_bigrams
## # A tibble: 580,636 × 2
##    book                     bigram        
##    <chr>                    <chr>         
##  1 The Voyage of the Beagle the voyage    
##  2 The Voyage of the Beagle voyage of     
##  3 The Voyage of the Beagle of the        
##  4 The Voyage of the Beagle the beagle    
##  5 The Voyage of the Beagle beagle by     
##  6 The Voyage of the Beagle charles darwin
##  7 The Voyage of the Beagle <NA>          
##  8 The Voyage of the Beagle <NA>          
##  9 The Voyage of the Beagle <NA>          
## 10 The Voyage of the Beagle <NA>          
## # ℹ 580,626 more rows

this data is still in tidytext format, and its structured as one-token per row. each token is a bigram.

counting and filtering ngram

darwin_bigrams %>%
  count(bigram, sort = TRUE)
## # A tibble: 206,416 × 2
##    bigram       n
##    <chr>    <int>
##  1 of the    9359
##  2 <NA>      7283
##  3 in the    4214
##  4 on the    3465
##  5 to the    2297
##  6 it is     1527
##  7 that the  1523
##  8 and the   1356
##  9 the same  1348
## 10 of a      1345
## # ℹ 206,406 more rows

most of the common bigrams are stop_words, this can be a good time to use tidyr separate command which splits a column into multiple based on a delimiter. this will let use make a column for word one and word two

library(tidyr)

bigrams_separated <- darwin_bigrams %>%
  separate(bigram, c("word1", "word2"), sep = " ")

bigrams_filtered <- bigrams_separated %>%
  filter(!word1 %in% stop_words$word) %>%
  filter(!word2 %in% stop_words$word)

bigrams_filtered
## # A tibble: 79,272 × 3
##    book                     word1   word2  
##    <chr>                    <chr>   <chr>  
##  1 The Voyage of the Beagle charles darwin 
##  2 The Voyage of the Beagle <NA>    <NA>   
##  3 The Voyage of the Beagle <NA>    <NA>   
##  4 The Voyage of the Beagle <NA>    <NA>   
##  5 The Voyage of the Beagle <NA>    <NA>   
##  6 The Voyage of the Beagle <NA>    <NA>   
##  7 The Voyage of the Beagle online  edition
##  8 The Voyage of the Beagle <NA>    <NA>   
##  9 The Voyage of the Beagle degree  symbol 
## 10 The Voyage of the Beagle degs    italics
## # ℹ 79,262 more rows

new bigram counts

bigram_counts <- bigrams_filtered %>%
  unite(bigram, word1, word2, sep = " ")

bigram_counts
## # A tibble: 79,272 × 2
##    book                     bigram        
##    <chr>                    <chr>         
##  1 The Voyage of the Beagle charles darwin
##  2 The Voyage of the Beagle NA NA         
##  3 The Voyage of the Beagle NA NA         
##  4 The Voyage of the Beagle NA NA         
##  5 The Voyage of the Beagle NA NA         
##  6 The Voyage of the Beagle NA NA         
##  7 The Voyage of the Beagle online edition
##  8 The Voyage of the Beagle NA NA         
##  9 The Voyage of the Beagle degree symbol 
## 10 The Voyage of the Beagle degs italics  
## # ℹ 79,262 more rows

we may also be interested in trigrams, which are three word combos

trigrams <- darwin_books %>%
  unnest_tokens(trigram, text, token = "ngrams", n=3) %>%
  separate(trigram, c("word1", "word2", "word3"), sep = " ") %>%
  filter(!word1 %in% stop_words$word,
         !word2 %in% stop_words$word,
         !word3 %in% stop_words$word) %>%
  count(word1, word2, word3, sort = TRUE)

trigrams
## # A tibble: 17,771 × 4
##    word1         word2  word3           n
##    <chr>         <chr>  <chr>       <int>
##  1 <NA>          <NA>   <NA>         7965
##  2 tierra        del    fuego          84
##  3 secondary     sexual characters     79
##  4 captain       fitz   roy            45
##  5 de            la     physionomie    30
##  6 domestication vol    ii             26
##  7 vol           ii     pp             22
##  8 vertebrates   vol    iii            21
##  9 proc          zoolog soc            18
## 10 proc          zool   soc            17
## # ℹ 17,761 more rows

lets analyze some bigrams

bigrams_filtered %>%
  filter(word2 == "selection") %>%
  count(book, word1, sort = TRUE)
## # A tibble: 24 × 3
##    book                                                 word1           n
##    <chr>                                                <chr>       <int>
##  1 The Descent of Man, and Selection in Relation to Sex sexual        254
##  2 The Descent of Man, and Selection in Relation to Sex natural       156
##  3 The Descent of Man, and Selection in Relation to Sex unconscious     6
##  4 The Descent of Man, and Selection in Relation to Sex continued       5
##  5 The Expression of the Emotions in Man and Animals    natural         4
##  6 The Descent of Man, and Selection in Relation to Sex artificial      2
##  7 The Descent of Man, and Selection in Relation to Sex ordinary        2
##  8 The Descent of Man, and Selection in Relation to Sex careful         1
##  9 The Descent of Man, and Selection in Relation to Sex consequent      1
## 10 The Descent of Man, and Selection in Relation to Sex la              1
## # ℹ 14 more rows

lets again look at tf-idf across bigrams across darwins works

bigram_tf_idf <- bigram_counts %>%
  count(book, bigram) %>%
  bind_tf_idf(bigram, book, n) %>%
  arrange(desc(tf_idf))

bigram_tf_idf
## # A tibble: 51,213 × 6
##    book                                       bigram     n      tf   idf  tf_idf
##    <chr>                                      <chr>  <int>   <dbl> <dbl>   <dbl>
##  1 The Expression of the Emotions in Man and… nerve…    47 0.00350 1.10  0.00384
##  2 The Descent of Man, and Selection in Rela… sexua…   138 0.00315 1.10  0.00346
##  3 The Expression of the Emotions in Man and… la ph…    35 0.00260 1.10  0.00286
##  4 The Voyage of the Beagle                   bueno…    54 0.00245 1.10  0.00269
##  5 The Voyage of the Beagle                   capta…    53 0.00240 1.10  0.00264
##  6 The Descent of Man, and Selection in Rela… secon…   101 0.00231 1.10  0.00254
##  7 The Voyage of the Beagle                   fitz …    50 0.00227 1.10  0.00249
##  8 The Expression of the Emotions in Man and… muscl…    30 0.00223 1.10  0.00245
##  9 The Expression of the Emotions in Man and… orbic…    29 0.00216 1.10  0.00237
## 10 The Descent of Man, and Selection in Rela… sexua…   254 0.00580 0.405 0.00235
## # ℹ 51,203 more rows
bigram_tf_idf %>%
  arrange(desc(tf_idf)) %>%
  group_by(book) %>%
  slice_max(tf_idf, n = 10) %>%
  ungroup() %>%
  mutate(bigram = reorder(bigram, tf_idf)) %>%
  ggplot(aes(tf_idf, bigram, fill = book)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~book, ncol = 2, scales = "free") + 
  labs(x = "tf-idf of bigrams", y = NULL)

using bigrams to provide context in sentiment analysis

bigrams_separated %>%
  filter(word1 == "not") %>%
  count(word1, word2, sort = TRUE)
## # A tibble: 740 × 3
##    word1 word2      n
##    <chr> <chr>  <int>
##  1 not   be        90
##  2 not   only      89
##  3 not   have      80
##  4 not   a         78
##  5 not   to        74
##  6 not   the       63
##  7 not   at        56
##  8 not   been      54
##  9 not   appear    51
## 10 not   know      47
## # ℹ 730 more rows

by doing sentiment analysis on bigrams, we can examine how often senitment associated words are preceded by a modifier like not or other negating words

AFINN <- get_sentiments("afinn")

AFINN
## # A tibble: 2,477 × 2
##    word       value
##    <chr>      <dbl>
##  1 abandon       -2
##  2 abandoned     -2
##  3 abandons      -2
##  4 abducted      -2
##  5 abduction     -2
##  6 abductions    -2
##  7 abhor         -3
##  8 abhorred      -3
##  9 abhorrent     -3
## 10 abhors        -3
## # ℹ 2,467 more rows

we can examine the most frequent words that are preceded by “not” and associate with sentiment

not_words <- bigrams_separated %>%
  filter(word1 == "not") %>%
  inner_join(AFINN, by = c(word2 = "word")) %>%
  count(word2, value, sort = TRUE)

not_words
## # A tibble: 97 × 3
##    word2     value     n
##    <chr>     <dbl> <int>
##  1 like          2    10
##  2 doubt        -1     9
##  3 wish          1     7
##  4 admit        -1     5
##  5 easy          1     5
##  6 pretend      -1     5
##  7 reach         1     5
##  8 difficult    -1     4
##  9 extend        1     4
## 10 help          2     4
## # ℹ 87 more rows

lets visualize

library(ggplot2)

not_words %>%
  mutate(contribution = n * value) %>%
  arrange(desc(abs(contribution))) %>%
  head(20) %>%
  mutate(word2 = reorder(word2, contribution)) %>%
  ggplot(aes(n * value, word2, fill = n * value > 0)) +
  geom_col(show.legend = FALSE) +
  labs(x = "Sentiment value * number or occurences", y = "words preceded by \"not\"")

negation_words <- c("not", "never", "non", "without")

negated_words <- bigrams_separated %>%
  filter(word1 %in% negation_words) %>%
  inner_join(AFINN, by = c(word2 = "word")) %>%
  count(word1, word2, value, sort = TRUE)

negated_words
## # A tibble: 137 × 4
##    word1   word2     value     n
##    <chr>   <chr>     <dbl> <int>
##  1 not     like          2    10
##  2 not     doubt        -1     9
##  3 not     wish          1     7
##  4 without doubt        -1     7
##  5 not     admit        -1     5
##  6 not     easy          1     5
##  7 not     pretend      -1     5
##  8 not     reach         1     5
##  9 not     difficult    -1     4
## 10 not     extend        1     4
## # ℹ 127 more rows

lets visualize the negation words

negated_words %>%
  mutate(contribution = n * value,
         word2=reorder(paste(word2, word1, sep = "_"), contribution)) %>%
  group_by(word1) %>%
  slice_max(abs(contribution), n=12, with_ties = FALSE) %>%
  ggplot(aes(word2, contribution, fill = n * value >0 )) +
  geom_col(show.legend = FALSE) + 
  facet_wrap(~ word1, scales = "free") +
  scale_x_discrete(labels = function(x) gsub("_.+$", "", x)) +
  xlab("words preceded by negation term") +
  ylab("Sentiment value * # of occurences") +
  coord_flip()

visualize a network of bigrams with ggraph

library(igraph)
## 
## Attaching package: 'igraph'
## The following object is masked from 'package:mosaic':
## 
##     compare
## The following objects are masked from 'package:lubridate':
## 
##     %--%, union
## The following object is masked from 'package:plotly':
## 
##     groups
## The following object is masked from 'package:tidyr':
## 
##     crossing
## The following objects are masked from 'package:dplyr':
## 
##     as_data_frame, groups, union
## The following objects are masked from 'package:stats':
## 
##     decompose, spectrum
## The following object is masked from 'package:base':
## 
##     union
bigram_counts <- bigrams_filtered %>%
  count(word1, word2, sort = TRUE)

bigram_graph <- bigram_counts %>%
  filter(n > 20) %>%
  graph_from_data_frame()
## Warning in graph_from_data_frame(.): In `d' `NA' elements were replaced with
## string "NA"
bigram_graph
## IGRAPH b95f01e DN-- 158 106 -- 
## + attr: name (v/c), n (e/n)
## + edges from b95f01e (vertex names):
##  [1] NA        ->NA          sexual    ->selection   vol       ->ii         
##  [4] natural   ->selection   lower     ->animals     sexual    ->differences
##  [7] breeding  ->season      secondary ->sexual      south     ->america    
## [10] sexual    ->characters  del       ->fuego       tierra    ->del        
## [13] vol       ->iii         de        ->la          bright    ->colours    
## [16] sexual    ->difference  distinct  ->species     tail      ->feathers   
## [19] closely   ->allied      vocal     ->organs      zoological->gardens    
## [22] natural   ->history     buenos    ->ayres       captain   ->fitz       
## + ... omitted several edges
library(ggraph)
set.seed(1234)

ggraph(bigram_graph, layout = "fr") +
  geom_edge_link() +
  geom_node_point() +
  geom_node_text(aes(label = name), vjust = 1, hjust = 1)
## Warning: Using the `size` aesthetic in this geom was deprecated in ggplot2 3.4.0.
## ℹ Please use `linewidth` in the `default_aes` field and elsewhere instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.

we can also add directionality to this network

set.seed (1234)

a <- grid::arrow(type = "closed", length = unit(.15, "inches"))

ggraph(bigram_graph, layout = "fr") +
  geom_edge_link(aes(edge_alpha = n), show.legend = FALSE, arrow = a, end_cap = circle(.07, 'inches')) +
  geom_node_point(color = "lightblue", size = 5) +
  geom_node_text(aes(label=name), vjust = 1, hjust = 1) +
  theme_void()

Word Frequencies

a central question in text mining is how to quantify what a document is about. we can do this but looking at words that make up that document, and measuring term frequency.

there are a lot of words that may not be important, these are the stop words.

one way to remedy this is to look at inverse document frequency words, which decreases the weight for commonly used words adn increases the weight for words that are not used very much

term frequency in Darwins works

library(dplyr)
library(tidytext)

book_words <- gutenberg_download(c(944, 1227, 1228, 2300), mirror = "https://gutenberg.nabasny.com/")
## Warning: ! Could not download a book at
##   https://gutenberg.nabasny.com//1/2/2/1228/1228.zip.
## ℹ The book may have been archived.
## ℹ Alternatively, You may need to select a different mirror.
## → See https://www.gutenberg.org/MIRRORS.ALL for options.
colnames(book_words)[1] <- "book"

book_words$book[book_words$book == 944] <- "The Voyage of the Beagle"
book_words$book[book_words$book == 1227] <- "The Expression of the Emotions in Man and Animals"
book_words$book[book_words$book == 1228] <- "On the Origin of Species By Means of Natural Selection"
book_words$book[book_words$book == 2300] <- "The Descent of Man, and Selection in Relation to Sex"

now lets disect

book_words <- book_words %>%
  unnest_tokens(word, text) %>%
  count(book, word, sort = TRUE)

book_words
## # A tibble: 35,579 × 3
##    book                                                 word      n
##    <chr>                                                <chr> <int>
##  1 The Descent of Man, and Selection in Relation to Sex the   25490
##  2 The Voyage of the Beagle                             the   16930
##  3 The Descent of Man, and Selection in Relation to Sex of    16762
##  4 The Voyage of the Beagle                             of     9438
##  5 The Descent of Man, and Selection in Relation to Sex in     8882
##  6 The Expression of the Emotions in Man and Animals    the    8045
##  7 The Descent of Man, and Selection in Relation to Sex and    7854
##  8 The Descent of Man, and Selection in Relation to Sex to     5901
##  9 The Voyage of the Beagle                             and    5768
## 10 The Voyage of the Beagle                             a      5328
## # ℹ 35,569 more rows
book_words$n <- as.numeric(book_words$n)

total_words <- book_words %>%
  group_by(book) %>%
  summarize(total = sum(n))

book_words
## # A tibble: 35,579 × 3
##    book                                                 word      n
##    <chr>                                                <chr> <dbl>
##  1 The Descent of Man, and Selection in Relation to Sex the   25490
##  2 The Voyage of the Beagle                             the   16930
##  3 The Descent of Man, and Selection in Relation to Sex of    16762
##  4 The Voyage of the Beagle                             of     9438
##  5 The Descent of Man, and Selection in Relation to Sex in     8882
##  6 The Expression of the Emotions in Man and Animals    the    8045
##  7 The Descent of Man, and Selection in Relation to Sex and    7854
##  8 The Descent of Man, and Selection in Relation to Sex to     5901
##  9 The Voyage of the Beagle                             and    5768
## 10 The Voyage of the Beagle                             a      5328
## # ℹ 35,569 more rows
book_words <- cross_join(book_words, total_words)

book_words
## # A tibble: 35,579 × 4
##    book                                                 word      n  total
##    <chr>                                                <chr> <dbl>  <dbl>
##  1 The Descent of Man, and Selection in Relation to Sex the   25490 629573
##  2 The Voyage of the Beagle                             the   16930 629573
##  3 The Descent of Man, and Selection in Relation to Sex of    16762 629573
##  4 The Voyage of the Beagle                             of     9438 629573
##  5 The Descent of Man, and Selection in Relation to Sex in     8882 629573
##  6 The Expression of the Emotions in Man and Animals    the    8045 629573
##  7 The Descent of Man, and Selection in Relation to Sex and    7854 629573
##  8 The Descent of Man, and Selection in Relation to Sex to     5901 629573
##  9 The Voyage of the Beagle                             and    5768 629573
## 10 The Voyage of the Beagle                             a      5328 629573
## # ℹ 35,569 more rows

you can see that the usual suspects are the most common words, but dont tell us anything about what the book topic is

library(ggplot2)

ggplot(book_words, aes(n/total, fill = book)) +
  geom_histogram(show.legend = FALSE) +
  xlim(NA, 0.0009) +
  facet_wrap(~book, ncol = 2, scales = "free_y")
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## Warning: Removed 125 rows containing non-finite values (`stat_bin()`).
## Warning: Removed 3 rows containing missing values (`geom_bar()`).

zipfs law

the frequency that a word appears in inversely proportional to its rank when predicting topics

lets apply zipfs law to Darwins work

freq_by_rank <- book_words %>%
  group_by(book) %>%
  dplyr::mutate(rank = row_number(),
         `term frequency` = n/total) %>%
  ungroup()

freq_by_rank
## # A tibble: 35,579 × 6
##    book                                word      n  total  rank `term frequency`
##    <chr>                               <chr> <dbl>  <dbl> <int>            <dbl>
##  1 The Descent of Man, and Selection … the   25490 629573     1          0.0405 
##  2 The Voyage of the Beagle            the   16930 629573     1          0.0269 
##  3 The Descent of Man, and Selection … of    16762 629573     2          0.0266 
##  4 The Voyage of the Beagle            of     9438 629573     2          0.0150 
##  5 The Descent of Man, and Selection … in     8882 629573     3          0.0141 
##  6 The Expression of the Emotions in … the    8045 629573     1          0.0128 
##  7 The Descent of Man, and Selection … and    7854 629573     4          0.0125 
##  8 The Descent of Man, and Selection … to     5901 629573     5          0.00937
##  9 The Voyage of the Beagle            and    5768 629573     3          0.00916
## 10 The Voyage of the Beagle            a      5328 629573     4          0.00846
## # ℹ 35,569 more rows
freq_by_rank %>%
  ggplot(aes(rank, `term frequency`, color = book)) +
  geom_line(size = 1.1, alpha = 0.8, show.legend = FALSE) +
  scale_x_log10() +
  scale_y_log10()

lets us TF - IDF to find words for each document by decreasing the weight for commonly used words and increasing the weight for words that are not used very much in a collection of documents

book_tf_idf <- book_words %>%
  bind_tf_idf(word, book, n)

book_tf_idf
## # A tibble: 35,579 × 7
##    book                                   word      n  total     tf   idf tf_idf
##    <chr>                                  <chr> <dbl>  <dbl>  <dbl> <dbl>  <dbl>
##  1 The Descent of Man, and Selection in … the   25490 629573 0.0820     0      0
##  2 The Voyage of the Beagle               the   16930 629573 0.0813     0      0
##  3 The Descent of Man, and Selection in … of    16762 629573 0.0539     0      0
##  4 The Voyage of the Beagle               of     9438 629573 0.0453     0      0
##  5 The Descent of Man, and Selection in … in     8882 629573 0.0286     0      0
##  6 The Expression of the Emotions in Man… the    8045 629573 0.0729     0      0
##  7 The Descent of Man, and Selection in … and    7854 629573 0.0253     0      0
##  8 The Descent of Man, and Selection in … to     5901 629573 0.0190     0      0
##  9 The Voyage of the Beagle               and    5768 629573 0.0277     0      0
## 10 The Voyage of the Beagle               a      5328 629573 0.0256     0      0
## # ℹ 35,569 more rows

lets look at terms with high tf-idf in darwin’s works

book_tf_idf %>%
  select(-total) %>%
  arrange(desc(tf_idf))
## # A tibble: 35,579 × 6
##    book                                        word      n      tf   idf  tf_idf
##    <chr>                                       <chr> <dbl>   <dbl> <dbl>   <dbl>
##  1 The Expression of the Emotions in Man and … tears   126 1.14e-3 1.10  1.25e-3
##  2 The Expression of the Emotions in Man and … blush   114 1.03e-3 1.10  1.13e-3
##  3 The Descent of Man, and Selection in Relat… sexu…   745 2.40e-3 0.405 9.71e-4
##  4 The Descent of Man, and Selection in Relat… sele…   621 2.00e-3 0.405 8.10e-4
##  5 The Voyage of the Beagle                    sea     351 1.69e-3 0.405 6.84e-4
##  6 The Voyage of the Beagle                    degs    117 5.62e-4 1.10  6.18e-4
##  7 The Expression of the Emotions in Man and … eyeb…   149 1.35e-3 0.405 5.47e-4
##  8 The Voyage of the Beagle                    isla…   271 1.30e-3 0.405 5.28e-4
##  9 The Descent of Man, and Selection in Relat… shewn   143 4.60e-4 1.10  5.05e-4
## 10 The Expression of the Emotions in Man and … frown    46 4.17e-4 1.10  4.58e-4
## # ℹ 35,569 more rows

lets look at the visualization for these tf - idf words

book_tf_idf %>%
  group_by(book) %>%
  slice_max(tf_idf, n = 15) %>%
  ungroup() %>%
  ggplot(aes(tf_idf, reorder(word, tf_idf), fill = book)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~book, ncol = 2, scales = "free") +
  labs(x="tf-idf", y = NULL)

Twitter

Lets load our data

##tweets20CA <- read.csv(“tweets20CA.csv”, row.names = 1) ##tweets21CA <- read.csv(“tweets21CA.csv”, row.names=1) ##tweets22CA <- read.csv(“tweets22CA.csv”, row.names=1)

##tweets20US <- read.csv(“tweets20US.csv”, row.names = 1) ##tweets21US <- read.csv(“tweets21US.csv”, row.names=1) ##tweets22US <- read.csv(“tweets22US.csv”, row.names=1)

##tweets20GB <- read.csv(“tweets20GB.csv”, row.names = 1) ##tweets21GB <- read.csv(“tweets21GB.csv”, row.names=1) ##tweets22GB <- read.csv(“tweets22GB.csv”, row.names=1)

##lets load our required libraries

##library(lubridate) ##library(ggplot2) ##library(dplyr) ##library(readr)

##lets combine our US tweets

##tweetsUS <- bind_rows(tweets21US %>% ## mutate(year = “y2021”), ## tweets22US %>% mutate(year = “y2022”), ##tweets20US %>% ## mutate(year = 2020)) %>% ## mutate(timestamp = ymd_hms(created_at))

##lets combine our GB tweets

##tweetsGB <- bind_rows(tweets21GB %>% ## mutate(year = “y2021”), ## tweets22GB %>% ## mutate(year = “y2022”), ## tweets20GB %>% ##mutate(year = 2020)) %>% ##mutate(timestamp = ymd_hms(created_at))

##Now lets combine our CA tweets

##tweetsCA <- bind_rows(tweets21CA %>% ## mutate(year = “y2021”), ## tweets22CA %>% ##mutate(year = “y2022”), ##tweets20CA %>% ## mutate(year = 2020)) %>% ##mutate(timestamp = ymd_hms(created_at))

##now lets clean our US tweets

##remove_reg <- “&|<>” ##tidy_tweetsUS <- tweetsUS %>% ## filter(!str_detect(text, “^RT”)) %>% ## mutate(text = str_remove_all(text, remove_reg)) %>% ## unnest_tokens(word, text, token = “tweets”) %>% ## filter(!word %in% stop_words\(word, ## !word %in% str_remove_all(stop_words\)word, ##““), ##str_detect(word,”[a-z”))

Now lets clean our CA tweets

##remove_reg <- “&|<>” ##tidy_tweetsCA <- tweetsCA %>% #3 filter(!str_detect(text, “^RT”)) %>% #3 mutate(text = str_remove_all(text, remove_reg)) %>% ## unnest_tokens(word, text, token = “tweets”) %>% ## filter(!word %in% stop_words\(word, !word %in% str_remove_all(stop_words\)word, ##““), ## str_detect(word,”[a-z”))

##Now lets clean our GB tweets

##remove_reg <- “&|<>” ##tidy_tweetsGB <- tweetsGB %>% ## filter(!str_detect(text, “^RT”)) %>% ## mutate(text = str_remove_all(text, remove_reg)) ##%>% ## unnest_tokens(word, text, token = “tweets”) %>% ## filter(!word %in% stop_words\(word, ## !word %in% str_remove_all(stop_words\)word, ##““), ## str_detect(word,”[a-z”))

##lets take a look at our data

##ggplot(tweetsUS, aes(x = timestamp, fill = year))+ ## geom_histogram(position = “identity”, bins = 20, ##show.legend = FALSE) + ## facet_wrap(~year, ncol = 1)

##lets looks at our GB data

##ggplot(tweetsGB, aes(x = timestamp, fill = year))+ ## geom_histogram(position = “identity”, bins = 20, show.legend = FALSE) + ## facet_wrap(~year, ncol = 1)

##Lets look at our CA tweets

##ggplot(tweetsCA, aes(x = timestamp, fill = year))+ ## geom_histogram(position = “identity”, bins = 20, ##show.legend = FALSE) + ##facet_wrap(~year, ncol = 1)

Lets look at the US frequencies for covid data

##frequencyUS <- tidy_tweetsUS %>% ## count(year, word, sort = TRUE) %>% ## left_join(tidy_tweetsUS %>% ## count(year, name = “total”)) %>% ## mutate(freq = n/total)

##frequencyUS

##Lets looks at the word frequencies of GB covid data

##frequencyGB <- tidy_tweetsGB %>% ## count(year, word, sort = TRUE) %>% ## left_join(tidy_tweetsGB %>% ## count(year, name = “total”)) %>% ## mutate(freq = n/total)

##frequencyGB

##Lets look at CA word frequencies for covid data

##frequencyCA <- tidy_tweetsCA %>% ##count(year, word, sort = TRUE) %>% ## left_join(tidy_tweetsCA %>% ## count(year, name = “total”)) %>% ## mutate(freq = n/total)

##frequencyCA

##lets look at percentrage of words for US

##library(tidyr)

##frequencyUS2 <- frequencyUS %>% ## select(year, word, freq) %>% ## pivot_wider(names_from = year, values_from = ##freq) %>% ## arrange(‘y2020’, ‘y2021’, ‘y2022’)

head(frequencyUS2)

lets look at percentgae of words for GB

library(tidyr)

frequencyGB2 <- frequencyGB %>% select(year, word, freq) %>% pivot_wider(names_from = year, values_from = freq) %>% arrange(‘y2020’, ‘y2021’, ‘y2022’)

head(frequencyGB2)

lets look at percentage of words for CA

library(tidyr)

frequencyCA2 <- frequencyCA %>% select(year, word, freq) %>% pivot_wider(names_from = year, values_from = freq) %>% arrange(‘y2020’, ‘y2021’, ‘y2022’)

head(frequencyCA2)

lets plot

ggplot(freqeuncyUS2, aes(2020, 2022)) + geom_jitter(alpha = 0.05, size = 2.5, width = 0.25, height = 0.25) + geom_text(aes(label = word), check_overlap = TRUE, vjust = 1.5) + scale_x_log10(labels = percent_format()) + scale_y_log10(labels = percent_format()) + geom_abline(color = “red”)

library(dplyr) library(stringr)

nrc_joy <- get_sentiments(“nrc”) %>% filter(sentiment == “joy”)

tidy_tweetsUS %>% filter(year == “y2020”) %>% inner_join(nrc_joy) %>% count(word, sort = TRUE)

nrc_anger <- nrc %>% filter(sentiment == “anger”)

tidy_tweetsGB %>% filter(year == “y2020”) %>% innter_join(nrc_anger) %>% count(word, sort = TRUE)

nrc_fear <- nrc %>% filter(sentiment == “fear”)

tidy_tweetsGB %>% filter(year == “y2022”) %>% innter_join(nrc_fear) %>% count(word, sort = TRUE)

GB_Covid_Sentiment <- tidy_tweetsGB %>% inner_join(get_sentiments(“bing”)) %>% count(year, index = id %% 80, sentiment) %>% pivot_wider(names_from = sentiment, values_from = n, values_fill = 0) %>% mutate(sentiment = positive - negative)

library(ggplot2)

ggplot(GB_Covid_Sentiment, aes(sentiment, fill = year)) + geom_histogram(show.legend = FALSE) + facet_wrap(~year, ncol = 2, scales = “free_x”)

GB_Covid_Sentiment_Means <- aggregate(x = GB_Covid_Sentiment\(sentiment, by = list(GB_Covid_Sentiment\)year), FUN = mean)

GB_Covid_Sentiment_Means

US_Covid_Sentiment <- tidy_tweetsUS %>% inner_join(get_sentiments(“bing”)) %>% count(year, index = id %% 80, sentiment) %>% pivot_wider(names_from = sentiment, values_from = n, values_fill = 0) %>% mutate(sentiment = positive - negative)

library(ggplot2)

ggplot(US_Covid_Sentiment, aes(sentiment, fill = year)) + geom_histogram(show.legend = FALSE) + facet_wrap(~year, ncol = 2, scales = “free_x”)

US_Covid_Sentiment_Means <- aggregate(x = US_Covid_Sentiment\(sentiment, by = list(US_Covid_Sentiment\)year), FUN = mean)

US_Covid_Sentiment_Means

CA_Covid_Sentiment <- tidy_tweetsCA %>% inner_join(get_sentiments(“bing”)) %>% count(year, index = id %% 80, sentiment) %>% pivot_wider(names_from = sentiment, values_from = n, values_fill = 0) %>% mutate(sentiment = positive - negative)

library(ggplot2)

ggplot(CA_Covid_Sentiment, aes(sentiment, fill = year)) + geom_histogram(show.legend = FALSE) + facet_wrap(~year, ncol = 2, scales = “free_x”)

CA_Covid_Sentiment_Means <- aggregate(x = CA_Covid_Sentiment\(sentiment, by = list(CA_Covid_Sentiment\)year), FUN = mean)

CA_Covid_Sentiment_Means

bing_word_counts_US <- tidy_tweetsUS %>% inner_join(get_sentiments(“bing”)) %>% count(word, sentiment, sort = TRUE) %>% ungroup()

bing_words_counts_US

bing_words_counts_US %>% group_by(sentiment) %>% slice_max(n, n = 10) %>% ungroup() %>% mutate(word = reorder(word, n)) %>% ggplot(aes(n, word, fill = sentiment)) + geom_col(show.legend = FALSE) + facet_wrap(~sentiment, scales = “free_y”) + labs(x = “contribution to sentiment”, y = NULL)

custom_stop_words <- bind_rows(tibble(word = c(“trump”, “positive”), lexicon = c(“custom”)), stop_words)

custom_stop_words

word clouds

library(wordcloud)

tidy_tweetsUS %>% anti_join(custom_stop_words) %>% inner_join(get_sentiments(“bing”)) %>% count(word, sentiment, sort = TRUE) %>% acast(word ~ sentiment, value.var = “n”, fill = 0) %>% comparison.cloud(colors = c(“gray20”, “gray80”), max.words = 100)

tidy_tweetsUS %>% anti_join(custom_stop_words) %>% count(word) %>% with(wordcloud(word, n, max.words = 100))

bingnegative <- get_sentiments(“bing”) %>% filter(sentiment == “negative”)

wordcounts <- tidy_tweetsCA %>% group_by(year) %>% summarize(words = n())

tidy_tweetsCA %>% semi_join(bingnegative) %>% group_by(year) %>% summarize(negativewords = n()) %>% left_join(wordcounts, by = “year”) %>% mutate(ratio = negativewords/words) %>% slice_max(ratio, n = 3) %>% ungroup()

Intro to shiny

library(shiny)

ui <- fluidPage( selectInput(“dataset”, label = “Dataset”, choices = ls(“packages:datasets”)), verbatimTextOutput(“summary”), tableOutput(“table”) ) server <- function(input, output,session) { output\(summary <- renderPrint({ dataset <- get(input\)dataset, “package::datasets”) summary(dataset) }) } shinyApp(ui, server)

Shiny Example 1-4

library(shiny) library(vroom) library(tidyverse) dir.create(“neiss”)

download <- funtcion(name) { url <- “https://raw.githubusercontent.com/hadley/mastering-shiny/main/neiss/” download.file(paste0(url, name), paste0(“neiss/”, name), quiet = TRUE) }

injuries <- vroom::vroom(“neiss/injuries.tsv.gz”) injuries

products <- vroom::vroom(“neiss/products.tsv”) products

population <- vroom::vroom(“neiss/population.tsv”) population

prod_codes <- setNames(products\(prod_code, products\)title)

injuries %>% mutate(diag = fct_lump(fct_infreq(diag), n = 5)) %>% group_by(diag) %>% summarise(n = as.integer(sum(weight)))

count_top <- function(df, var, n = 5) { df %>% mutate({{var }} := fct_lump(fct_infreq({{ var }}), n = n)) %>% group_by({{ var }}) %>% summarise(n = as.integer(sum(weight))) }

ui <- fluidPage( fluidRow( column(8, selectInput(“code”, “Product”, choices = setNames(products\(prod_code, products\)title), width = “100%” ) ), column(2, selectInput(“y”, “Y axis”, c(“rate”, “count”))) ),

#fluidRow( #column(6, #selectInput(“code”, “Product”, choices = prod_codes) #) #), fluidRow( column(4, tableOutput(“diag”)), column(4, tableOutput(“body_part”)), column(4, tableOutput(“location”)) ), fluidRow( column(2, actionButton(“story”, “Tell me a story”)), column(10, textOutput(“narrative”)) ), fluidRow( column(12, plotOutput(“age_sex”)) ) )

server <- function(input, output, session) { selected <- reactive(injuries %>% filter(prod_code == input$code))

output\(diag <- renderTable(count_top(selected(), diag), width = "100%") output\)body_part <- renderTable(count_top(selected(), body_part), width = “100%”) output$location <- renderTable(count_top(selected(), location), width = “100%”)

summary <- reactive({ selected() %>% count(age, sex, wt = weight) %>% left_join(population, by = c(“age”, “sex”)) %>% mutate(rate = n / population * 1e4) })

output\(age_sex <- renderPlot({ if (input\)y == “count”) { summary() %>% ggplot(aes(age, n, color = sex)) + geom_line()+ labs(y = “Estimated number of injuries”) } else { summary() %>% ggplot(aes(age, rate, color = sex)) + geom_line(na.rm = TRUE) + labs(y = “Injuries per 10,000 people”) } }, res = 96)

narrative_sample <- eventReactive( list(input\(story, selected()), selected() %>% pull(narrative) %>% sample(1) ) output\)narrative <- renderText(narrative_sample())

} shinyApp(ui, server)

Shiny Side bars

ui <- fluidPage( titlePanel(“Central limit theorem”), sidebarLayout( sidebarPanel( numericInput(“m”, “number of samples:”, 2, min = 1, max = 100) ), mainPanel( plotOutput(“hist”) ) ) )

server <- function(input, output, session) { output\(hist <- renderPlot({ means <- replicate(1e4, mean(runif(input\)m))) hist(means, breaks = 20) }, res = 96) } shinyApp(ui, server)

Shiny multipage

gglotex <- “yadayadayada” ui <- fluidPage( sidebarLayout( sidebarPanel( textOutput(“panel”) ), mainPanel( tabsetPanel( id = “tabset”, tabPanel(“panel 1”, ggplotex), tabPanel(“panel 2”, “two”), tabPanel(“panel 3”, “three”) ) ) ) )

server <- function(input, output, session) { output\(panel <- renderText({ paste("Current panel: ", input\)tabset) }) } shinyApp(ui, server)

Shiny navbars

ui <- fluidPage( navlistPanel( id = “tabset”, “Heading 1”, tabPanel(“panel 1”, “Panel one contents”), “Heading 2”, tabPanel(“panel 2”, “Panel two contents”), tabPanel(“panel 3”, “Panel three contents”) ) )

server <- function(input, output, session) { output\(panel <- renderText({ paste("Current panel: ", input\)tabset) }) } shinyApp(ui, server)

Shiny dropdown

ui <- navbarPage( “Page title”, tabPanel(“panel1”, “one”), tabPanel(“panel 2”, “two”), tabPanel(“pannel 3”, “three”), navbarMenu(“subpanels”, tabPanel(“panel 4a”, “four-a”), tabPanel(“panel 4b”, “four-b”), tabPanel(“panel 4c”, “four-c”) ) )

server <- function(input, output, session) { output\(panel <- renderText({ paste("Current panel:", input\)tabset) }) } shinyApp(ui, server)

Shiny Thematic

BiocManager::install(“thematic”, update = FALSE)

library(ggplot2) library(thematic) ui <- fluidPage( theme = bslib::bs_theme(bootswatch = “darkly”), titlePanel(“A themed plot”), plotOutput(“plot”),

)

server <- function(input, output, session) { thematic::thematic_shiny()

output$plot <- renderPlot({ ggplot(mtcars, aes(wt, mpg)) + geom_point() + geom_smooth() }, res = 96) } shinyApp(ui, server)

Shiny Brushed

ui <- fluidPage( plotOutput(“plot”, brush = “plot_brush”), tableOutput(“data”) )

server <- function(input, output, session) { output$plot <- renderPlot({ ggplot(mtcars, aes(wt, mpg)) + geom_point() }, res = 96)

output\(data <- renderTable({ brushedPoints(mtcars, input\)plot_brush) }) }

shinyApp(ui, server)

Shiny Brushed 2

ui <- fluidPage( plotOutput(“plot”, brush = “plot_brush”, dbclick = “plot_reset”)

)

server <- function(input, output, session) { selected <- reactiveVal(rep(FALSE, nrow(mtcars)))

observeEvent(input\(plot_brush, { brushed <- brushedPoints(mtcars, input\)plot_brush, allRows = TRUE)\(selected_ selected(brushed | selected()) }) observeEvent(input\)plot_reset, { selected(rep(FALSE, nrow(mtcars))) }) output\(plot <- renderPlot({ mtcars\)sel <- selected() ggplot(mtcars, aes(wt, mpg)) + geom_point(aes(colour = sel)) + scale_colour_discrete(limits = c(“TRUE”, “FALSE”)) }, res = 96) }

shinyApp(ui, server)

Shiny images

setwd(“~/Desktop/Classroom/Myfiles”)

puppies <- tibble::tribble( ~breed, ~id, ~author, “corgi”, “eoqnr8ikwFE”, “alvannee”, “labrador”, “KCdYn0xu2fU”, “shaneguymon”, “spaniel”, “TzjMd7i5WQI”, “redo” )

ui <- fluidPage( selectInput(“id”, “Pick a Breed”, choices = setNames(puppies\(id, puppies\)breed)), htmlOutput(“source”), imageOutput(“photo”) )

server <- function(input, output, session) { output\(photo <- renderImage({ list( src = file.path("puppy-photos", paste0(input\)id, “.jpg”)), concentType = “image/jpeg”, width = 500, height = 600, ) }, deleteFile = FALSE)

output\(source <- renderUI({ info <- puppies[puppies\)id == input\(id, , drop = FALSE] HTML(glue::glue("<p> <a href='https://unsplash.com/photos/{info\)id}’>original by {info$author}

“)) })

} shinyApp(ui, server)

Shiny interactive

ui <- fluidPage( plotOutput(“plot”, click = “plot_click”), verbatimTextOutput(“info”) ) server <- function(input, output) { output\(plot <- renderPlot({ plot(mtcars\)wt, mtcars\(mpg) }, res = 96) output\)info <- renderPrint({ req(input\(plot_click) x <- round(input\)plot_click\(x, 2) y <- round(input\)plot_click$y, 2) cat(“[”, x, ”, ”, y , ”]”, sep = ““) }) } shinyApp(ui, server)

Shiny Responsive Graphs

library(ggplot2)

ui <- fluidPage( selectInput(“x”, “x Variable”, choices = names(iris)), selectInput(“y”, “y Variable”, choices = names(iris)), selectInput(“geom”, “geom”, c(“point”, “smooth”, “jitter”)), plotOutput(“plot”) )

server <- function(input, output, session) { plot_geom <- reactive({ switch(input\(geom, point = geom_point(), smooth = geom_smooth(se = FALSE), jitter = geom_jitter() ) }) output\)plot <- renderPlot({ ggplot(iris, aes(.data[[input\(x]], .data[[input\)y]])) + plot_geom() }, res = 96) }

shinyApp(ui, server)

Shiny selected boxes

library(dplyr) setwd(“H:/Data Products/Shiny/Examples/SelectBoxes”) health <- vroom::vroom(“us_county_sociohealth_data.csv”, col_types = list(), na = ““) health %>% select(state, county, everything()) %>% arrange(county) ui <- fluidPage( selectInput(”state”, “state”, choices = unique(health\(state)), selectInput("county", "county", choices = NULL), tableOutput("data") ) server <- function(input, output, session) { state <- reactive({ filter(health, state == input\)state) })

observeEvent(state(), { choices <- unique(state()$county) updateSelectInput(inputId = “county”, choices = choices) })

county <- reactive({ req(input\(county) filter(state(), county == input\)county) }) output$data <- renderTable({

req(input\(county) county() %>% filter(county == input\)county) %>% select(2:9) }) }