This report covers 2017 data collection.

The script has been revised in response to reviewer comments in Winter 2023.

Data prep

Libraries.

library(tidyverse)

## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.1 ──

## ✔ ggplot2 3.4.0      ✔ purrr   1.0.0 
## ✔ tibble  3.1.8      ✔ dplyr   1.0.10
## ✔ tidyr   1.2.1      ✔ stringr 1.5.0 
## ✔ readr   2.1.3      ✔ forcats 0.5.1

## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()

library(lme4)

## Loading required package: Matrix

## 
## Attaching package: 'Matrix'

## The following objects are masked from 'package:tidyr':
## 
##     expand, pack, unpack

library(brms)

## Loading required package: Rcpp

## Loading 'brms' package (version 2.18.0). Useful instructions
## can be found by typing help('brms'). A more detailed introduction
## to the package is available through vignette('brms_overview').

## 
## Attaching package: 'brms'

## The following object is masked from 'package:lme4':
## 
##     ngrps

## The following object is masked from 'package:stats':
## 
##     ar

library(knitr)
library(BayesFactor)

## Loading required package: coda

## ************
## Welcome to BayesFactor 0.9.12-4.4. If you have questions, please contact Richard Morey (richarddmorey@gmail.com).
## 
## Type BFManual() to open the manual.
## ************

library(ggthemes)
theme_set(theme_few())

Read sheets.

raw_data <- read_csv("data/amazon_pragmatics_data.csv")

## Rows: 84 Columns: 17
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (14): Edad, Genero, Lugar de grabacion, Grabacion_date, Trial 1, Trial 2...
## dbl  (3): Age, Grabacion_order, Orden
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

orders <- read_csv("data/amazon_pragmatics_orders.csv")

## Rows: 40 Columns: 8
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (6): TrialClass, L, R, Target, Trial Type, Stimulus
## dbl (2): Order, Trial
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

Now merge in stimulus information.

Note, we previously excluded children > 10 years old. Now we leave them in.

d_raw <- raw_data |>
  mutate(Subid = 1:n()) |>
  rename(Order = Orden, 
         Gender = Genero) |>
  gather(Trial, Choice, `Trial 1`:`Trial 10`) |>
  mutate(Trial = as.numeric(str_sub(Trial,start = -2, -1))) |>
  left_join(orders) |>
  rename(TrialType = `Trial Type`) |>
  mutate(Correct = Choice == Target, 
         TargetStimulus = ifelse(Target == "L", L, R)) |>
  select(Subid, Age, Gender, Order, Trial, Choice, Target, TrialType, Stimulus, TargetStimulus, Correct) |>
  mutate(age_grp = case_when(
    Age > 4 & Age <= 6 ~ "4 - 6",
    Age > 6 & Age <= 8 ~ "6 - 8",
    Age > 8  ~ "8+"),
    TrialType = fct_recode(TrialType, 
                           `Control-Double` = "ControlDouble",
                           `Control-Single` = "ControlSingle",
                           `Warm-Up` = "Warm-up",
                           `Pragmatic Inference` = "Test"))

## Joining, by = c("Order", "Trial")

subs <- d_raw |>
  group_by(Subid, age_grp) |>
  summarise(Age = Age[1], 
            n_trials = sum(!is.na(Correct)), 
            correct = mean(Correct), 
            gender = Gender[1])

## `summarise()` has grouped output by 'Subid'. You can override using the
## `.groups` argument.

ggplot(subs, aes(x = Age)) + 
  geom_histogram(binwidth = .5, center = 0)

## Warning: Removed 6 rows containing non-finite values (`stat_bin()`).

subs |>
  group_by(age_grp) |>
  summarise(mean_age = mean(Age, na.rm=TRUE),
            sd_age = sd(Age, na.rm=TRUE),
            max_age = max(Age, na.rm=TRUE), 
            min_age = min(Age, na.rm=TRUE),
            n_trials = mean(n_trials), 
            n = n(), 
            missing_age = sum(is.na(Age)), 
            n_male = sum(gender == "M"), 
            prop_male = n_male / n) |>
  kable(digits = 2)

## Warning in max(Age, na.rm = TRUE): no non-missing arguments to max; returning
## -Inf

## Warning in min(Age, na.rm = TRUE): no non-missing arguments to min; returning
## Inf

age_grp	mean_age	sd_age	max_age	min_age	n_trials	n	missing_age	n_male	prop_male
4 - 6	5.43	0.49	6.0	4.3	10.00	11	0	3	0.27
6 - 8	7.09	0.50	8.0	6.1	9.93	30	0	16	0.53
8+	9.08	0.67	10.8	8.1	10.00	37	0	14	0.38
NA	NaN	NA	-Inf	Inf	10.00	6	6	2	0.33

Primary analysis

Start by filtering missing ages.

d <- d_raw |>
  mutate(TrialType = fct_relevel(TrialType, "Pragmatic Inference")) |>
  filter(!is.na(Age))

Now let’s make the primary plot.

mbs <- d |>
  group_by(TrialType, Age, age_grp, Subid) |>
  summarise(Correct = mean(Correct, na.rm=TRUE),
            n = n())

## `summarise()` has grouped output by 'TrialType', 'Age', 'age_grp'. You can
## override using the `.groups` argument.

ms <- mbs |>
  group_by(TrialType, age_grp) |>
  langcog::multi_boot_standard(col = "Correct", na.rm=TRUE) |>
  mutate(Age = case_when(age_grp == "4 - 6" ~ 5,
                         age_grp == "6 - 8" ~ 7,
                         age_grp == "8+" ~ 9))

ggplot(mbs,
       aes(x = Age, y = Correct, 
           col = TrialType, shape = TrialType)) + 
  geom_jitter(width = 0, height = .05, alpha = .5) +
  geom_pointrange(data = ms, 
                  aes(y = mean, ymin = ci_lower, ymax = ci_upper), 
                  position = position_dodge(width = .05)) + 
  geom_line(data = filter(ms, age_grp != "exclude"), 
            aes(y = mean, group = TrialType)) + 
  geom_hline(lty = 2, yintercept = .5) + 
  scale_color_colorblind(name = "Trial Type" ) + 
  scale_shape_few(name = "Trial Type" ) + 
  ylab("Proportion Correct") + 
  xlab("Age Group (years)") + 
  theme(legend.position = "bottom")

And by items.

mbs_items <- d |>
  filter(TrialType %in% c("Pragmatic Inference", 
                          "Control-Double")) |>
  group_by(TrialType, Age, age_grp, Stimulus, Subid) |>
  summarise(Correct = mean(Correct, na.rm=TRUE),
            n = n())

## `summarise()` has grouped output by 'TrialType', 'Age', 'age_grp', 'Stimulus'.
## You can override using the `.groups` argument.

ms_items <- mbs_items |>
  group_by(TrialType, Stimulus, age_grp) |>
  langcog::multi_boot_standard(col = "Correct", na.rm=TRUE) 

ggplot(mbs_items,
       aes(x = age_grp, y = Correct, 
           col = TrialType, shape = TrialType)) + 
  geom_pointrange(data = ms_items, 
                  aes(y = mean, ymin = ci_lower, ymax = ci_upper), 
                  position = position_dodge(width = .05)) + 
  geom_line(data = ms_items, 
            aes(y = mean, group = TrialType)) + 
  geom_hline(lty = 2, yintercept = .5) + 
  scale_color_manual(name = "Trial Type",
                         values = c("Pragmatic Inference" = "#000000", 
                                    "Control-Double" = "#E69F00") ) + 
  scale_shape_few(name = "Trial Type" ) + 
  ylab("Proportion Correct") + 
  xlab("Age Group (years)") + 
  theme(legend.position = "bottom") + 
  facet_wrap(~Stimulus)

Stats

Model

Maximal random effects structure. Note that this model is no longer included in the manuscript for reasons of parsimony.

bayes_mod <- brm(as.numeric(Correct) ~ TrialType * scale(Age) 
                 + (TrialType | Subid) 
                 + (TrialType * scale(Age) | Stimulus), 
                 family = "bernoulli", 
                 data = filter(d, age_grp != "exclude"))
kable(fixef(bayes_mod), digits = 2)

Bayesian hypothesis tests

Control-single all kids.

ttestBF(filter(mbs,TrialType=="Control-Single")$Correct, mu = .5)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 17799133 ±0%
## 
## Against denominator:
##   Null, mu = 0.5 
## ---
## Bayes factor type: BFoneSample, JZS

Control-single by age.

ttestBF(filter(mbs,TrialType=="Control-Single" & age_grp == "4 - 6")$Correct, mu = .5)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 0.51032 ±0.01%
## 
## Against denominator:
##   Null, mu = 0.5 
## ---
## Bayes factor type: BFoneSample, JZS

ttestBF(filter(mbs,TrialType=="Control-Single" & age_grp == "6 - 8")$Correct, mu = .5)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 1.339463 ±0.02%
## 
## Against denominator:
##   Null, mu = 0.5 
## ---
## Bayes factor type: BFoneSample, JZS

ttestBF(filter(mbs,TrialType=="Control-Single" & age_grp == "8+")$Correct, mu = .5)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 48133161716 ±0%
## 
## Against denominator:
##   Null, mu = 0.5 
## ---
## Bayes factor type: BFoneSample, JZS

Inference trials.

ttestBF(filter(mbs, TrialType=="Pragmatic Inference")$Correct, mu = .5)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 4.253915 ±0.01%
## 
## Against denominator:
##   Null, mu = 0.5 
## ---
## Bayes factor type: BFoneSample, JZS

By age.

ttestBF(filter(mbs,TrialType=="Pragmatic Inference" & age_grp == "4 - 6")$Correct, mu = .5)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 0.3400324 ±0.01%
## 
## Against denominator:
##   Null, mu = 0.5 
## ---
## Bayes factor type: BFoneSample, JZS

ttestBF(filter(mbs,TrialType=="Pragmatic Inference" & age_grp == "6 - 8")$Correct, mu = .5)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 0.1995884 ±0.03%
## 
## Against denominator:
##   Null, mu = 0.5 
## ---
## Bayes factor type: BFoneSample, JZS

ttestBF(filter(mbs,TrialType=="Pragmatic Inference" & age_grp == "8+")$Correct, mu = .5)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 58.75831 ±0%
## 
## Against denominator:
##   Null, mu = 0.5 
## ---
## Bayes factor type: BFoneSample, JZS

Frequentist t-tests

Control-single all kids.

t.test(filter(mbs,TrialType=="Control-Single")$Correct- .5)

## 
##  One Sample t-test
## 
## data:  filter(mbs, TrialType == "Control-Single")$Correct - 0.5
## t = 7.0826, df = 77, p-value = 5.797e-10
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.1843209 0.3284996
## sample estimates:
## mean of x 
## 0.2564103

By age.

t.test(filter(mbs,TrialType=="Control-Single" & age_grp == "4 - 6")$Correct - .5)

## 
##  One Sample t-test
## 
## data:  filter(mbs, TrialType == "Control-Single" & age_grp == "4 - 6")$Correct - 0.5
## t = 1.1504, df = 10, p-value = 0.2767
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.1277398  0.4004671
## sample estimates:
## mean of x 
## 0.1363636

t.test(filter(mbs,TrialType=="Control-Single" & age_grp == "6 - 8")$Correct - .5)

## 
##  One Sample t-test
## 
## data:  filter(mbs, TrialType == "Control-Single" & age_grp == "6 - 8")$Correct - 0.5
## t = 2.1122, df = 29, p-value = 0.0434
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.004229691 0.262436976
## sample estimates:
## mean of x 
## 0.1333333

t.test(filter(mbs,TrialType=="Control-Single" & age_grp == "8+")$Correct - .5)

## 
##  One Sample t-test
## 
## data:  filter(mbs, TrialType == "Control-Single" & age_grp == "8+")$Correct - 0.5
## t = 11.424, df = 36, p-value = 1.58e-13
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.3223176 0.4614662
## sample estimates:
## mean of x 
## 0.3918919

Inference trials.

t.test(filter(mbs, TrialType=="Pragmatic Inference")$Correct - .5)

## 
##  One Sample t-test
## 
## data:  filter(mbs, TrialType == "Pragmatic Inference")$Correct - 0.5
## t = 2.7658, df = 77, p-value = 0.007105
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.03051791 0.18743081
## sample estimates:
## mean of x 
## 0.1089744

By age.

t.test(filter(mbs,TrialType=="Pragmatic Inference" & age_grp == "4 - 6")$Correct - .5)

## 
##  One Sample t-test
## 
## data:  filter(mbs, TrialType == "Pragmatic Inference" & age_grp == "4 - 6")$Correct - 0.5
## t = 0.55902, df = 10, p-value = 0.5884
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.1357189  0.2266280
## sample estimates:
##  mean of x 
## 0.04545455

t.test(filter(mbs,TrialType=="Pragmatic Inference" & age_grp == "6 - 8")$Correct - .5)

## 
##  One Sample t-test
## 
## data:  filter(mbs, TrialType == "Pragmatic Inference" & age_grp == "6 - 8")$Correct - 0.5
## t = 0.23869, df = 29, p-value = 0.813
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.1261407  0.1594741
## sample estimates:
##  mean of x 
## 0.01666667

t.test(filter(mbs,TrialType=="Pragmatic Inference" & age_grp == "8+")$Correct - .5)

## 
##  One Sample t-test
## 
## data:  filter(mbs, TrialType == "Pragmatic Inference" & age_grp == "8+")$Correct - 0.5
## t = 3.8307, df = 36, p-value = 0.0004927
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  0.09538419 0.31002122
## sample estimates:
## mean of x 
## 0.2027027

Side bias

Look at side biase effects. Not any huge evidence of side bias, though overall test performance higher when target is on the R.

mbs_side <- d |>
  group_by(TrialType, Target, Subid) |>
  summarise(Correct = mean(Correct, na.rm=TRUE),
            n = n())

## `summarise()` has grouped output by 'TrialType', 'Target'. You can override
## using the `.groups` argument.

ms_side <- mbs_side |>
  group_by(TrialType, Target) |>
  langcog::multi_boot_standard(col = "Correct", na.rm=TRUE) 


ggplot(mbs_side, aes(x = Target, y = Correct, col = TrialType)) + 
  geom_pointrange(data = ms_side, 
                  aes(y = mean, ymin = ci_lower, ymax = ci_upper), 
                  position = position_dodge(width = .05)) + 
  geom_hline(lty = 2, yintercept = .5) + 
  scale_color_solarized() + 
  ylab("Proportion correct") + 
  xlab("Age (years)") + 
  ylim(0,1) + 
  theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

Analysis of Amazon Stiller Task Data

Martin Fortier, Danielle Kellier, and Mike Frank

2023-02-27

Data prep

Primary analysis

Stats

Model

Bayesian hypothesis tests

Frequentist t-tests

Side bias