Setup

library(tidyverse)
library(lubridate)
library(ggthemes)
library(assertthat)
library(langcog) # for CIs
library(mirt)
library(ggpubr)
library(knitr)
library(dplyr)
library(here)
library(lme4)
library(lmerTest)

Load data from multiafc task

First load all metadata we’ll need for the items

test_corpus <- read_csv(here::here("item_generation/4_create_multi_AFC_stimuli/new_test.csv")) %>%
  filter(!Word1 %in% c('honey','scrabble')) # manually excluded
test_clip_cor <- read_csv(here::here("item_generation/4_create_multi_AFC_stimuli/exp1_all_trials2023-04-11.csv")) %>%
  select(Word1, Word2, trial_type, cor, AoA_Est_Word2, AoA_Est_Word1) %>%
  rename(wordPairing = trial_type) %>%
  rename(clip_cor = cor) %>%
  filter(Word1 %in% unique(test_corpus$Word1)) %>%
  filter(!Word1 %in% c('honey','scrabble')) # manually excluded for symmetry across blocks

Create data structure and merge that has clip co =1 for target selection

test_clip_cor_identity <- test_clip_cor %>%
  select(Word1, AoA_Est_Word1) %>%
  distinct(Word1, AoA_Est_Word1) %>%
  mutate(Word2 = Word1) %>%
  mutate(AoA_Est_Word2 = AoA_Est_Word1) %>%
  mutate(clip_cor = 1) %>%
  mutate(wordPairing = 'target')

test_clip_cor <- test_clip_cor %>%
  full_join(test_clip_cor_identity) %>%
  arrange(Word1)

Rename meta adata structures so that they merge with trial data

clip_to_join <- test_clip_cor %>%
  rename(targetWord = Word1) %>%
  rename(answerWord = Word2)
test_corpus <- test_corpus %>%
  left_join(test_clip_cor)

sum(is.na(test_corpus$clip_cor))
## [1] 0

Read in bing data

bing = read_csv(here::here('data/validation_data/preprocessed_data/all_bing_data_for_item_info.csv'))  %>%
  mutate(age_group = as.numeric(floor(age_in_months/12))) %>%
  select(-block)

Read in multi-afc adult data Set age to “25” so we can plot it together

adults <- read_csv(here::here("data/data_multiafc/multi-afc-april28.csv"))  %>%
  mutate(age_group = as.numeric(25)) %>% # set 
  select(-block)

Construct trial structure

Make trial-levels data structure across all ages

trial_data <- read_csv(here::here("data/rocketship_data/multi-afc-with-meta.csv")) %>% 
  select(-block) %>%
  mutate(age_group = as.numeric(age_group)) %>%
  filter(age_group<12) %>% # older kids just don't have enough trials
  full_join(bing) %>% # 3-5 year olds
  full_join(adults) %>%
  filter(studyId %in% c("school-multiAFC", "prolific-multiAFC", "validation-multiAFC"), completed == TRUE, task =="test_response") %>% 
  mutate(schoolId = ifelse(studyId == "school-multiAFC", "school", schoolId)) %>%
  mutate(options_clean = str_replace_all(options, "'", "\"")) # necessary for json parsing

Create functions for parsing options from nested json

parse_json_column <- function(json_col) {
  map(json_col, ~as.data.frame(t(unlist(jsonlite::fromJSON(.x, flatten = TRUE)))))
}

Parse out all of the target/distractor pairing

parsed_list <- parse_json_column(trial_data$options_clean)

# Combine the list of data frames into a single data frame
parsed_df <- bind_rows(parsed_list)

# Combine the parsed columns with the original data frame
final_df <- cbind(trial_data %>% select(-options), parsed_df)

Join together and clean datastructures

df.multiAFC.trials <- final_df %>% 
  pivot_longer(cols = c("0", "1", "2", "3")) %>% 
  filter(answerWord == value) %>% 
  select(pid, runId, schoolId, answerWord, targetWord, numAFC, value, correct, rt, age_group) %>% 
  left_join(test_corpus %>% 
              rename(targetWord = "Word1", answerWord = "Word2")) %>%
  filter(is.na(itemGroup) | (itemGroup == "test")) %>% 
  mutate(wordPairing = ifelse(is.na(wordPairing), "target", wordPairing)) %>%
  filter(!targetWord %in%  c("ant","ball","bear","scrabble","honey")) %>% 
  select(pid, runId, schoolId, answerWord, targetWord, numAFC, value, correct, rt, wordPairing, age_group, clip_cor)

Create age/group and number of participants for filtering for plots / age-based analyses

age_group_pid <- df.multiAFC.trials %>%
  group_by(age_group) %>%
  summarize(num_pids = length(unique(pid)))

We have 1446 children aged 3-11 years, and 205 adults.

Now write code to fill out response pattern for all possible target/distractor responses on each age for each age…for use later.

full_item_structure_4AFC <- clip_to_join %>%
  mutate(numAFC = 4)

full_item_structure_3AFC <- clip_to_join %>%
  filter(wordPairing != 'distal') %>%
  mutate(numAFC = 3)

full_item_structure_2AFC <- clip_to_join %>%
  filter(!wordPairing %in% c('distal','easy')) %>%
  mutate(numAFC = 2)

full_item_structure <- full_item_structure_4AFC %>%
  full_join(full_item_structure_3AFC) %>%
  full_join(full_item_structure_2AFC)

Now fill this out by all ages in the dataset, oof this was annoying

full_item_structure_by_age = map_df(age_group_pid$age_group, ~full_item_structure %>% mutate(age_group = .x))

Calculate proportion of times participants chose an image

First do this across all age groups – for basic item plots.

df.multiAFC.totalAttempts <- df.multiAFC.trials %>% 
  group_by(targetWord, numAFC) %>% 
  tally() %>% 
  dplyr::rename(totalAttempts = n)

df.multiAFC.distractor.summary <- df.multiAFC.trials %>% 
  group_by(targetWord,  answerWord, numAFC, wordPairing) %>% 
  tally() %>%
  left_join(df.multiAFC.totalAttempts) %>% 
  mutate(perc = n/totalAttempts) %>%
  full_join(full_item_structure) %>% # need to fill out item structure (but not by age)
  mutate(perc = replace_na(perc, replace=0))

Look at histogram of how often an item was attempted overal

hist(df.multiAFC.totalAttempts$totalAttempts)

Look at histograms of how often teh target was chosen

target_pc <- df.multiAFC.distractor.summary %>%
  filter(wordPairing=='target')

hist(target_pc$perc)

Low accuracy items

target_pc %>%
  filter(perc<.5) %>%
  kable()
targetWord answerWord numAFC wordPairing n totalAttempts perc clip_cor AoA_Est_Word2 AoA_Est_Word1
bobsled bobsled 3 target 117 292 0.4006849 1 9.38 9.38
bobsled bobsled 4 target 111 278 0.3992806 1 9.38 9.38
candlestick candlestick 2 target 98 254 0.3858268 1 5.61 5.61
candlestick candlestick 3 target 118 275 0.4290909 1 5.61 5.61
candlestick candlestick 4 target 128 293 0.4368601 1 5.61 5.61
grate grate 2 target 138 284 0.4859155 1 8.90 8.90
grate grate 4 target 125 257 0.4863813 1 8.90 8.90
mulch mulch 2 target 111 281 0.3950178 1 9.22 9.22
mulch mulch 3 target 112 284 0.3943662 1 9.22 9.22
mulch mulch 4 target 100 259 0.3861004 1 9.22 9.22

High accuracy items

target_pc %>%
  filter(perc>.95) %>%
  kable()
targetWord answerWord numAFC wordPairing n totalAttempts perc clip_cor AoA_Est_Word2 AoA_Est_Word1
acorn acorn 2 target 243 254 0.9566929 1 5.95 5.95
cake cake 2 target 262 273 0.9597070 1 3.26 3.26
carrot carrot 2 target 283 288 0.9826389 1 2.74 2.74
elbow elbow 2 target 286 300 0.9533333 1 4.78 4.78
fan fan 2 target 265 277 0.9566787 1 5.63 5.63
footbath footbath 2 target 277 286 0.9685315 1 8.53 8.53
footbath footbath 4 target 253 266 0.9511278 1 8.53 8.53
hamster hamster 2 target 261 272 0.9595588 1 4.37 4.37
hedgehog hedgehog 2 target 257 267 0.9625468 1 8.63 8.63
lollipop lollipop 2 target 248 255 0.9725490 1 3.89 3.89
lollipop lollipop 3 target 269 281 0.9572954 1 3.89 3.89
map map 2 target 250 261 0.9578544 1 5.60 5.60
map map 3 target 213 224 0.9508929 1 5.60 5.60
marshmallow marshmallow 2 target 254 263 0.9657795 1 3.80 3.80
pie pie 2 target 284 292 0.9726027 1 3.67 3.67
potato potato 2 target 243 253 0.9604743 1 4.84 4.84
rice rice 2 target 257 266 0.9661654 1 3.72 3.72
rice rice 4 target 263 273 0.9633700 1 3.72 3.72
shower shower 2 target 263 275 0.9563636 1 4.72 4.72
shower shower 3 target 238 249 0.9558233 1 4.72 4.72
squirrel squirrel 2 target 249 259 0.9613900 1 4.44 4.44
sunflower sunflower 2 target 240 246 0.9756098 1 6.00 6.00
sunflower sunflower 3 target 238 245 0.9714286 1 6.00 6.00
sunflower sunflower 4 target 268 277 0.9675090 1 6.00 6.00
turkey turkey 2 target 257 264 0.9734848 1 3.95 3.95
watermelon watermelon 2 target 235 245 0.9591837 1 4.22 4.22
watermelon watermelon 3 target 253 264 0.9583333 1 4.22 4.22
watermelon watermelon 4 target 265 271 0.9778598 1 4.22 4.22

Construct response dataframe now by age group

df.multiAFC.totalAttempts.byAge <- df.multiAFC.trials %>% 
  group_by(targetWord, numAFC, age_group) %>% 
  tally() %>% 
  dplyr::rename(totalAttempts = n)

df.multiAFC.distractor.summary.byAge <- df.multiAFC.trials %>% 
  group_by(targetWord,  answerWord, numAFC, wordPairing, age_group) %>% 
  tally() %>% 
  left_join(df.multiAFC.totalAttempts.byAge) %>% 
  mutate(perc = n/totalAttempts) %>%
  full_join(full_item_structure_by_age)

couple of sanity checks for missing values that shuold be zero

assert_that(sum(is.na(df.multiAFC.distractor.summary.byAge$age_group))==0)
## [1] TRUE
assert_that(sum(is.na(df.multiAFC.distractor.summary.byAge$clip_cor))==0)
## [1] TRUE

There are some missing observations because these items were never chosen on a given trial/age group – need to fill those out

But we also have some target/distractor/age group pairings that are VERY sparsely populated – that makes sense especially as kids get more accurate and don’t choose distal items. We also have relatively few younger kids.

hist(df.multiAFC.distractor.summary.byAge$totalAttempts)

None of these are actually zero, just hcekcing.

sum(df.multiAFC.distractor.summary.byAge$totalAttempts==0, na.rm=TRUE)
## [1] 0

We need to populate “0” value in the percent chosen category for these distractor items that were never chosen and so not in the data structure

sum(is.na(df.multiAFC.distractor.summary.byAge$perc))
## [1] 2502
df.multiAFC.distractor.summary.byAge <- df.multiAFC.distractor.summary.byAge %>%
  mutate(perc = replace_na(perc, replace=0))

Summary by distractor type and numAFC by age, group, use confidence intervals here (langcog package)

df.multiAFC.distractor.summary.perc <- df.multiAFC.distractor.summary.byAge %>% 
  ungroup() %>%
  mutate(wordPairing = factor(wordPairing, levels = c('target','hard','easy','distal'), labels = c("Target word", "High sim. dist", "Med sim. dist.", "Low sim. dist.")))  %>% # just reordering
  group_by(age_group, numAFC, wordPairing) %>% 
  multi_boot_standard(col = 'perc')  %>% # compute CIs
  mutate(number_afc = as.factor(paste0(numAFC,' AFC'))) #cosmetic for plots

Plot gradient in selection across target type within age

First visualize within each age group – can plots nAFCs by different colors

Looks like gradient is getting sharper with age, as expected

ggplot(data = df.multiAFC.distractor.summary.perc, aes(x=wordPairing, y=mean, col=number_afc)) +
  geom_point() +
  geom_linerange(aes(y=mean, ymax = ci_upper, ymin = ci_lower)) +
  geom_line(aes(group=number_afc)) +
  theme(aspect.ratio=.75) +
  theme_few() +
  ylab('Proportion images chosen') +
  facet_wrap(~age_group,nrow=2) +
  theme(axis.text.x = element_text(size=6, angle = 45, hjust = 1))

Plot gradient in selection across age within afc

Now plot within afc, with age groups as colors Make adults grey here (lots of extra code) so that the kid age gradient isn’t too compressed

ggplot(data = df.multiAFC.distractor.summary.perc %>% filter(age_group<25), aes(x=wordPairing, y=mean, col=age_group)) +
  geom_point(alpha=.8) +
  geom_point(data = df.multiAFC.distractor.summary.perc %>% filter(age_group==25), color='grey', alpha=.8) +
  geom_linerange(aes(y=mean, ymax = ci_upper, ymin = ci_lower), alpha=.3) +
  geom_linerange(data = df.multiAFC.distractor.summary.perc %>% filter(age_group==25), aes(y=mean, ymax = ci_upper, ymin = ci_lower), alpha=.3, color='grey') +
  geom_line(data = df.multiAFC.distractor.summary.perc %>% filter(age_group==25), color='grey',aes(group=age_group)) +
  geom_line(aes(group=age_group)) +
  facet_wrap(~number_afc) +
  theme(aspect.ratio=.75) +
  scale_color_viridis_c(name='Age (in years)') +
  xlab('') +
  ylab('Proportion chosen') +
  theme_few() +
  ylim(0,1) +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) 

ggsave(filename = 'figures/prop_chosen_by_type_by_age.pdf', width=6, height=5, units='in')

Let’s try binning by clip similarity instead of wordPairing – look similar

clip_cor_distractors <- df.multiAFC.distractor.summary.byAge  %>%
  ungroup() %>%
  mutate(clip_quantile = ntile(clip_cor, 4)) %>%
  group_by(age_group, numAFC, clip_quantile) %>% 
  multi_boot_standard(col = 'perc')  
ggplot(data = clip_cor_distractors %>% filter(age_group<25), aes(x=clip_quantile, y=mean, col=age_group)) +
  geom_point(alpha=.8) +
  geom_point(data = clip_cor_distractors %>% filter(age_group==25), color='grey', alpha=.8) +
  geom_linerange(aes(y=mean, ymax = ci_upper, ymin = ci_lower), alpha=.3) +
  geom_linerange(data = clip_cor_distractors %>% filter(age_group==25), aes(y=mean, ymax = ci_upper, ymin = ci_lower), alpha=.3, color='grey') +
  geom_line(data = clip_cor_distractors %>% filter(age_group==25), color='grey',aes(group=age_group)) +
  geom_line(aes(group=age_group)) +
  facet_wrap(~numAFC) +
  theme(aspect.ratio=.75) +
  scale_color_viridis_c(name='Age (in years)') +
  xlab('CLIP similarity quantiles') +
  ylab('Proportion chosen') +
  theme_few() +
  ylim(0,1) +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) 

# ggsave(filename = 'figures/prop_chosen_by_type_by_age.pdf', width=6, height=5, units='in')

Third way of visaulizing this data – which kinds of items were chosen as a function of age in each NAFC

ggplot(data = df.multiAFC.distractor.summary.perc, aes(x=age_group, y=mean, col=wordPairing)) +
  geom_point() +
  geom_linerange(aes(y=mean, ymax = ci_upper, ymin = ci_lower), alpha=.8) +
  geom_smooth(aes(group=wordPairing), span=10) +
  facet_wrap(~number_afc) +
  theme(aspect.ratio=.75) +
  # scale_color_viridis_c(name='Age (in years)') +
  xlab('') +
  ylab('Proportion chosen') +
  theme_few() +
  ylim(0,1) +
  ggtitle('Proportion images chosen by # of distractors')

ggsave(filename = 'figures/prop_chosen_by_age_by_type.pdf', width=5, height=5, units='in')

Item effects

First in order over all kids

df.multiAFC.distractor.summary.word <- df.multiAFC.distractor.summary %>% 
  group_by(numAFC, wordPairing, targetWord) %>% 
  summarise(percentile = mean(perc)) %>%  # averaging across age
  arrange(targetWord, numAFC) %>%
  ungroup() %>%
  mutate(targetWord = fct_reorder(targetWord, percentile))

We have a big range of item effects here – some near ceiling, some near floor, and we span the whole range

ggplot(data = df.multiAFC.distractor.summary.word, aes(x=targetWord, y=percentile, color=wordPairing)) +
  geom_point(alpha=.8, size=2) +
  theme_few(base_size=12) + 
  theme(axis.text.x = element_text(size = 4, angle = 90, hjust = 1)) +
  facet_wrap(~numAFC, nrow=3) +
  theme(legend.position = 'bottom')

ggsave(filename = 'figures/item_effects.pdf', width=5, height=5, units='in')

Plotting individual item effects for the younger age groups is tricky…just not enough data from the youngest/oldest kids.

Plot which items do vs. don’t follow the expected gradient, from Anya’s prior analysis

df.flag.pair <- df.multiAFC.distractor.summary.word %>% 
  pivot_wider(names_from = wordPairing, values_from = percentile) %>%
  filter((hard > target) | (easy > hard) | (easy > target)) %>% 
  add_column(flag = 1) %>% 
  select(numAFC, targetWord, flag)

df.multiAFC.distractor.summary.word.updated <- df.multiAFC.distractor.summary.word %>% 
  left_join(df.flag.pair) %>% 
  mutate(flag = ifelse(is.na(flag), "expected", "flag"))

Lets look also at which items were hardest

hardest_items <- df.multiAFC.distractor.summary.word %>%
  group_by(targetWord) %>%
  filter(wordPairing=='target') %>%
  filter(numAFC==4) %>%
  arrange(percentile) %>%
  ungroup() %>%
  slice_max(order_by=-percentile, n=20)

Red items are the ones that don’t follow the gradient – note that the axes is reversed here

ggplot(data=df.multiAFC.distractor.summary.word.updated , aes(x=wordPairing, y=percentile)) +
  geom_point(size = 1, alpha = 0.6) + 
  geom_line(aes(group = targetWord, color = flag), alpha=.4) +
  facet_wrap(~numAFC) + 
  scale_color_manual(values=c( "#8F993E", "#E05A1D")) + 
  labs(x = "item distractor type",
       y = "percentage of responses")

Let’s manually inspect these items that are off the gradient

Make wide data structure for 4afc only

df.multiAFC.distractor.summary.wide <- df.multiAFC.distractor.summary %>%
  ungroup() %>% 
  filter(numAFC == 4) %>% 
  select(-c(n, answerWord, clip_cor,AoA_Est_Word1, AoA_Est_Word2)) %>% 
  pivot_wider(names_from = wordPairing, values_from = perc)

Look at specific items that are off (hard chosen less often than easy distractor)

df.multiAFC.distractor.summary.wide %>% 
  filter((hard + 0.1)< easy)
## # A tibble: 3 × 7
##   targetWord numAFC totalAttempts target distal   hard  easy
##   <chr>       <dbl>         <int>  <dbl>  <dbl>  <dbl> <dbl>
## 1 aloe            4           298  0.540 0.111  0.0906 0.258
## 2 coaster         4           252  0.591 0.0913 0.0437 0.274
## 3 cymbal          4           276  0.638 0.0435 0.0797 0.239

Look at specific items that are off for 3afc (hard chosen less often than easy distractor)

df.multiAFC.distractor.summary %>%
  ungroup() %>% 
  filter(numAFC == 3) %>% 
  select(-c(n, answerWord, clip_cor,AoA_Est_Word1, AoA_Est_Word2)) %>% 
  pivot_wider(names_from = wordPairing, values_from = perc) %>%
  filter((hard + 0.1)< easy)
## # A tibble: 4 × 6
##   targetWord numAFC totalAttempts target   hard  easy
##   <chr>       <dbl>         <int>  <dbl>  <dbl> <dbl>
## 1 aloe            3           279  0.642 0.0609 0.297
## 2 coaster         3           275  0.575 0.0836 0.342
## 3 cymbal          3           297  0.690 0.0640 0.246
## 4 omelet          3           267  0.749 0.0637 0.187

Look at specific items that are off for 2afc (hard chosen less often than easy distractor)

df.multiAFC.distractor.summary %>%
  ungroup() %>% 
  filter(numAFC == 2) %>% 
  select(-c(n, answerWord, clip_cor,AoA_Est_Word1, AoA_Est_Word2)) %>% 
  pivot_wider(names_from = wordPairing, values_from = perc) %>%
  arrange(target)
## # A tibble: 108 × 5
##    targetWord  numAFC totalAttempts target  hard
##    <chr>        <dbl>         <int>  <dbl> <dbl>
##  1 candlestick      2           254  0.386 0.614
##  2 mulch            2           281  0.395 0.605
##  3 grate            2           284  0.486 0.514
##  4 bobsled          2           259  0.533 0.467
##  5 cheese           2           282  0.596 0.404
##  6 freezer          2           245  0.604 0.396
##  7 thermos          2           296  0.608 0.392
##  8 sauerkraut       2           261  0.636 0.364
##  9 tuxedo           2           261  0.640 0.360
## 10 corset           2           266  0.647 0.353
## # ℹ 98 more rows

Correlate clip with distractor error patterns within each age group/nafc

Can clearly see some of the data sparsity issues here at the younger ages

ggplot(data=df.multiAFC.distractor.summary.byAge %>% filter(wordPairing != 'target'), aes(x=clip_cor, y=perc)) +
  geom_point(alpha=.05) +
  geom_smooth(method='lm') +
  theme_few() +
  ylab('Prop distractor chosen') +
  xlab('Similarity in CLIP space') +
  facet_grid(~age_group) +
  theme(axis.text.x = element_text(size=6)) +
  scale_x_continuous(breaks = c(.5, .7, .9)) +
  stat_cor(method = "pearson", aes(label = ..r.label..), size=2) +
  ylim(0,1)

clip_correlation_by_age_by_afc <- df.multiAFC.distractor.summary.byAge %>%
  filter(wordPairing != 'target') %>%
  group_by(age_group) %>%
  summarize(rvalue = cor(clip_cor, perc))

Correlation between adults/kids oddly isn’t that high, maybe some data sparisty here

# this is 962 possibilities across all 108 items x 234afc combinations

adult_patterns <- df.multiAFC.distractor.summary.byAge %>%
  filter(age_group==25)

Just doing within all afc because grouping is tricky but oddly not that high relative to clip correlations, adult data does correlate with itself at r=1 within the pipe suggesting no indexing issues

adult_correlation_by_age <- df.multiAFC.distractor.summary.byAge %>%
  group_by(age_group) %>%
  summarize(r_adults = cor(perc, adult_patterns$perc))

Here’s a take home plot that’s easy – CLIP- behavior plot across age on error patterns

Have to think about how much of this is about reliability, ugh.

ggplot(data = clip_correlation_by_age_by_afc, aes(x=age_group, y=rvalue)) +
  geom_point() +
  geom_smooth(span=1, alpha=.2) +
  theme_few() +
  ylab('CLIP-Behavior correlation') +
  xlab('Age in years')

s

Inferential stats

Modeling image choice data for a given trial - what we’re modeling is the proportion of times an image was chosen on a trial by children in a certain age group, as a function of the (1) clip correlation between the target word and the distractor word, (clip_cor) (2) the age in years of the children/adults participating (age_group) (3) the number of other distrators on that trial (numAFC)

With random effects for (1) Random slopes of the clip_correlation on each targetWord, (2) the number of times this trial was seen by kids in this age group, which varies a lot given the different recruitment strategies and we want to account for this

We’re looking for way to test for that gradient items children choose as a function of CLIP simialrity

Add numafc as a covariate and random effects for the clip | target_word

summary(lmer(data = df.multiAFC.distractor.summary.byAge, perc ~ clip_cor*age_group + numAFC + (clip_cor|targetWord) + (1|totalAttempts)))
## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: perc ~ clip_cor * age_group + numAFC + (clip_cor | targetWord) +  
##     (1 | totalAttempts)
##    Data: df.multiAFC.distractor.summary.byAge
## 
## REML criterion at convergence: -2697.3
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.6979 -0.7537  0.0265  0.7530  4.5649 
## 
## Random effects:
##  Groups        Name        Variance Std.Dev. Corr 
##  targetWord    (Intercept) 0.679090 0.82407       
##                clip_cor    0.827115 0.90946  -1.00
##  totalAttempts (Intercept) 0.004081 0.06388       
##  Residual                  0.036562 0.19121       
## Number of obs: 7218, groups:  targetWord, 108; totalAttempts, 86
## 
## Fixed effects:
##                      Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)        -6.206e-01  8.763e-02  1.406e+02  -7.082 6.24e-11 ***
## clip_cor            1.177e+00  9.521e-02  1.327e+02  12.365  < 2e-16 ***
## age_group          -1.040e-01  3.101e-03  6.998e+03 -33.528  < 2e-16 ***
## numAFC             -5.117e-03  2.994e-03  6.989e+03  -1.709   0.0875 .  
## clip_cor:age_group  1.183e-01  3.406e-03  6.962e+03  34.737  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) clp_cr ag_grp numAFC
## clip_cor    -0.986                     
## age_group   -0.318  0.312              
## numAFC      -0.169  0.065  0.009       
## clp_cr:g_gr  0.313 -0.321 -0.982 -0.008

Add in estimated aoa of word1/2 as covariates, same results – still see interaction

summary(lmer(data = df.multiAFC.distractor.summary.byAge, perc ~ clip_cor*age_group + numAFC + AoA_Est_Word2 + AoA_Est_Word1 + (clip_cor|targetWord) + (1|totalAttempts)))
## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: 
## perc ~ clip_cor * age_group + numAFC + AoA_Est_Word2 + AoA_Est_Word1 +  
##     (clip_cor | targetWord) + (1 | totalAttempts)
##    Data: df.multiAFC.distractor.summary.byAge
## 
## REML criterion at convergence: -2983
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.8656 -0.7194  0.0291  0.7229  4.4815 
## 
## Random effects:
##  Groups        Name        Variance Std.Dev. Corr 
##  targetWord    (Intercept) 0.808789 0.89933       
##                clip_cor    0.994509 0.99725  -1.00
##  totalAttempts (Intercept) 0.003988 0.06315       
##  Residual                  0.034852 0.18669       
## Number of obs: 7218, groups:  targetWord, 108; totalAttempts, 86
## 
## Fixed effects:
##                      Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)        -6.693e-01  9.571e-02  1.401e+02  -6.992 1.01e-10 ***
## clip_cor            1.178e+00  1.027e-01  1.255e+02  11.462  < 2e-16 ***
## age_group          -1.035e-01  3.028e-03  6.990e+03 -34.187  < 2e-16 ***
## numAFC             -9.288e-03  2.933e-03  6.985e+03  -3.166  0.00155 ** 
## AoA_Est_Word2       4.401e-02  2.484e-03  6.025e+03  17.720  < 2e-16 ***
## AoA_Est_Word1      -3.431e-02  3.200e-03  3.107e+02 -10.721  < 2e-16 ***
## clip_cor:age_group  1.178e-01  3.326e-03  6.954e+03  35.430  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) clp_cr ag_grp numAFC AA_E_W2 AA_E_W1
## clip_cor    -0.973                                     
## age_group   -0.285  0.282                              
## numAFC      -0.153  0.059  0.008                       
## AA_Est_Wrd2 -0.009 -0.003  0.009 -0.082                
## AA_Est_Wrd1 -0.137  0.015 -0.004  0.066 -0.674         
## clp_cr:g_gr  0.280 -0.291 -0.982 -0.008 -0.009   0.002

But this goes away if we exclude kids choosing the target word – only looking at errors

We still see many effects of age and clip similarity, but no interaction

summary(lmer(data = df.multiAFC.distractor.summary.byAge %>% filter(wordPairing != 'target'), perc ~ clip_cor*age_group + numAFC  + (clip_cor|targetWord) + (1|totalAttempts)))
## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: perc ~ clip_cor * age_group + numAFC + (clip_cor | targetWord) +  
##     (1 | totalAttempts)
##    Data: df.multiAFC.distractor.summary.byAge %>% filter(wordPairing !=  
##     "target")
## 
## REML criterion at convergence: -5792.1
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.8065 -0.5349 -0.1407  0.3882  5.7615 
## 
## Random effects:
##  Groups        Name        Variance Std.Dev. Corr 
##  targetWord    (Intercept) 0.133263 0.3651        
##                clip_cor    0.286672 0.5354   -0.99
##  totalAttempts (Intercept) 0.009721 0.0986        
##  Residual                  0.011993 0.1095        
## Number of obs: 4139, groups:  targetWord, 108; totalAttempts, 85
## 
## Fixed effects:
##                      Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)        -3.731e-03  5.327e-02  3.038e+02  -0.070   0.9442    
## clip_cor            3.988e-01  6.799e-02  2.048e+02   5.865 1.78e-08 ***
## age_group          -6.796e-03  3.153e-03  3.930e+03  -2.156   0.0312 *  
## numAFC             -2.265e-02  2.526e-03  3.937e+03  -8.967  < 2e-16 ***
## clip_cor:age_group -1.332e-03  3.893e-03  3.924e+03  -0.342   0.7323    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) clp_cr ag_grp numAFC
## clip_cor    -0.951                     
## age_group   -0.529  0.504              
## numAFC      -0.328  0.174  0.006       
## clp_cr:g_gr  0.524 -0.513 -0.989 -0.006

What if we exclude the hardest items that the youngest kids don’t know? This is about half of the words 57/108 words – and we still see the interaction

summary(lmer(data = df.multiAFC.distractor.summary.byAge %>% filter(AoA_Est_Word1<8), perc ~ clip_cor*age_group + numAFC  + (clip_cor|targetWord) + (1|totalAttempts)))
## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: perc ~ clip_cor * age_group + numAFC + (clip_cor | targetWord) +  
##     (1 | totalAttempts)
##    Data: df.multiAFC.distractor.summary.byAge %>% filter(AoA_Est_Word1 <  
##     8)
## 
## REML criterion at convergence: -1477.3
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.6965 -0.7820  0.0727  0.7526  4.5566 
## 
## Random effects:
##  Groups        Name        Variance Std.Dev. Corr 
##  totalAttempts (Intercept) 0.003589 0.0599        
##  targetWord    (Intercept) 0.567541 0.7534        
##                clip_cor    0.682392 0.8261   -1.00
##  Residual                  0.038239 0.1955        
## Number of obs: 4537, groups:  totalAttempts, 85; targetWord, 70
## 
## Fixed effects:
##                      Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)        -1.006e+00  1.040e-01  1.029e+02  -9.677 3.92e-16 ***
## clip_cor            1.588e+00  1.120e-01  9.645e+01  14.172  < 2e-16 ***
## age_group          -1.051e-01  4.501e-03  4.390e+03 -23.344  < 2e-16 ***
## numAFC              5.216e-03  3.870e-03  4.393e+03   1.348    0.178    
## clip_cor:age_group  1.173e-01  4.894e-03  4.368e+03  23.968  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) clp_cr ag_grp numAFC
## clip_cor    -0.986                     
## age_group   -0.387  0.382              
## numAFC      -0.186  0.073  0.004       
## clp_cr:g_gr  0.383 -0.391 -0.987 -0.006

Bizarrely the interaction is back for distractors (though p=.04) for these words with lower AoA ? Not sure why

summary(lmer(data = df.multiAFC.distractor.summary.byAge %>% filter(wordPairing != 'target' & numAFC>2), perc ~ clip_cor*age_group + numAFC  + (clip_cor|targetWord) + (1|totalAttempts)))
## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: perc ~ clip_cor * age_group + numAFC + (clip_cor | targetWord) +  
##     (1 | totalAttempts)
##    Data: df.multiAFC.distractor.summary.byAge %>% filter(wordPairing !=  
##     "target" & numAFC > 2)
## 
## REML criterion at convergence: -5000.8
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.4138 -0.5266 -0.1420  0.3545  6.0255 
## 
## Random effects:
##  Groups        Name        Variance Std.Dev. Corr 
##  targetWord    (Intercept) 0.12760  0.3572        
##                clip_cor    0.26537  0.5151   -0.99
##  totalAttempts (Intercept) 0.01059  0.1029        
##  Residual                  0.01033  0.1016        
## Number of obs: 3312, groups:  targetWord, 108; totalAttempts, 83
## 
## Fixed effects:
##                      Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)        -5.826e-02  5.316e-02  3.175e+02  -1.096   0.2740    
## clip_cor            3.913e-01  6.599e-02  2.020e+02   5.930 1.29e-08 ***
## age_group          -6.615e-03  3.123e-03  3.086e+03  -2.118   0.0342 *  
## numAFC             -9.301e-03  3.818e-03  3.084e+03  -2.436   0.0149 *  
## clip_cor:age_group -1.793e-04  3.905e-03  3.080e+03  -0.046   0.9634    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) clp_cr ag_grp numAFC
## clip_cor    -0.928                     
## age_group   -0.527  0.520              
## numAFC      -0.353  0.102  0.008       
## clp_cr:g_gr  0.521 -0.532 -0.987 -0.009