XTMEM

Meta-analysis of experiments

---

---

Reproduce SPSS Fig. 3

(plotting proportion basic level)

Read in all previous data and bind to current dataset

files = dir("../data/")
all.d = data.frame()
for (i in 1:length(files)[1]) {
    s <- read.csv(paste0("../data/", files[i]))
    all.d = bind_rows(all.d, data.frame(s))
}

exp_key = read.csv("experiment_key.csv")

Get means and plot

all.d$condition = plyr::mapvalues(all.d$condition,
                               from = c("one", "3bas", "3sub", "3sup"), 
                               to = c("one", "three_basic", 
                                        "three_subordinate",
                                        "three_superordinate"))
all.d$condition = as.factor(all.d$condition)

all.ms = all.d %>%
  gather(variable, value, c(prop_sub, prop_bas, prop_sup)) %>%
  group_by(condition,variable, exp, subids) %>%
  mutate(value = as.numeric(value)) %>%
  summarize(value = mean(value)) %>%
  group_by(condition,variable, exp) %>%
  multi_boot_standard(column = "value")  %>% 
  ungroup() %>%
  mutate(variable = as.factor(variable)) %>%
  filter(condition == "one" | condition == "three_subordinate") %>%
  filter(variable == "prop_bas") %>%
  mutate(exp = as.factor(exp)) %>%
  left_join(exp_key) 
  
ggplot(all.ms, aes(x = exp, y = mean, group = condition, fill = condition)) +
  geom_bar(position = "dodge", stat = "identity") +
  geom_linerange(aes(ymin = summary_ci_lower, 
                     ymax = summary_ci_upper), 
                 position=position_dodge(width = .9)) +
  facet_grid(order * one_3sub_label ~ timing) +
  ylim(0,1)+
  ylab("Proportion basic-level choices ") +
  xlab("Experiment") +
  theme_bw() +
  theme(legend.title = element_blank())

same vs. different refers to whether the one and 3 subordinate trials recieved the same lable. 3–1 and 1–3 refers to the relative order of the one and 3 subordinate trials.

Plot effect size

Get effect sizes of our experiments.

all.ms.subj = all.d %>%
  gather(variable, value, c(prop_sub, prop_bas, prop_sup)) %>%
  group_by(condition,variable, exp, subids) %>%
  mutate(value = as.numeric(value)) %>%
  summarize(value = mean(value)) %>%
  filter(condition == "one" | condition == "three_subordinate", variable == 'prop_bas') %>%
  spread(condition, value) %>%
  ungroup () %>%
  select(-variable)

effect.sizes.npaired  = all.ms.subj %>%
  group_by(exp) %>%
  summarize(m.one = mean(one),
            sd.one = sd(one),
            m.3sub = mean(three_subordinate),
            sd.3sub = sd(three_subordinate),
            n= n()) %>%
  do(data.frame(d = mes(.$m.one, .$m.3sub, .$sd.one,
       .$sd.3sub, .$n, .$n, verbose = F)$d,
       var.d = mes(.$m.one, .$m.3sub, .$sd.one,
       .$sd.3sub, .$n, .$n, verbose = F)$var.d))%>%
  mutate(high = d + (1.96*var.d),
         low = d - (1.96*var.d),
         es.type = "nonpaired",
         exp = as.numeric(c(1:10)))  # fix this

effect.sizes = effect.sizes.npaired %>%
  mutate(n = 50, 
         exp = as.factor(exp)) 

Calculate effect sizes from previous experiments. Here are the means and sd based on SPSS table 1.

original.effect.sizes = data.frame(exp = c("XT_adults_e1", "XT_children_e2",
                        "SPSS_e1",
                        "SPSS_e2",
                        "SPSS_e3",
                        "SPSS_eS1",
                        "SPSS_eS2"),
                        one_means = c(76, 40, 48.24, 30.83, 39.91, 24.56, 15.79), 
                        three_means = c(9, 6, 10.53, 53.33, 51.75, 16.67, 11.40),
                        one_sd = c(40.40, 40.40, 40.40,37.18,35.20,32.09,24.51),
                        three_sd = c(24.97, 24.97, 24.97,36.11,42.63,25.45,24.25),
                        n = c(22, 36, 19, 20, 19,19,19))
kable(original.effect.sizes)

exp	one_means	three_means	one_sd	three_sd	n
XT_adults_e1	76.00	9.00	40.40	24.97	22
XT_children_e2	40.00	6.00	40.40	24.97	36
SPSS_e1	48.24	10.53	40.40	24.97	19
SPSS_e2	30.83	53.33	37.18	36.11	20
SPSS_e3	39.91	51.75	35.20	42.63	19
SPSS_eS1	24.56	16.67	32.09	25.45	19
SPSS_eS2	15.79	11.40	24.51	24.25	19

Calculate previous effect sizes

original.effect.sizes$d = mes(original.effect.sizes$one_means/100,
                              original.effect.sizes$three_means/100, 
                              original.effect.sizes$one_sd/100,
                              original.effect.sizes$three_sd/100,
                              original.effect.sizes$n,
                              original.effect.sizes$n, verbose = F)$d

original.effect.sizes$d_var = mes(original.effect.sizes$one_means/100,
                                  original.effect.sizes$three_means/100,
                                  original.effect.sizes$one_sd/100,
                                  original.effect.sizes$three_sd/100,
                                  original.effect.sizes$n, 
                                  original.effect.sizes$n,
                                  verbose = F)$var.d

original.effect.sizes = original.effect.sizes %>%
                            mutate(high = d + (1.96*d_var),
                                     low = d - (1.96*d_var)) %>%
                            select(-one_means, -three_means, -one_sd, -three_sd) %>%
                            mutate(es.type = "nonpaired")

Bind previous and current effect sizes together and plot

all.es =  original.effect.sizes %>%
            bind_rows(effect.sizes) %>%
            left_join(exp_key)


ggplot(all.es, aes(x = exp, y = d)) +
  geom_point(color = "red", size = 2)+
  geom_linerange(aes(ymax =high, ymin=low),
                 position = position_dodge(width = 0), color = "red") +
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  facet_grid(order * one_3sub_label ~ timing, drop = TRUE, scales = "free_y")+
  coord_flip() +
  geom_hline(yintercept = 0, linetype = 2) +
  ggtitle("Effect size") +
  xlab("Experiment")

same vs. different refers to whether the one and 3 subordinate trials recieved the same lable. 3–1 and 1–3 refers to the relative order of the one and 3 subordinate trials.

Misc

By category

Post-task questions

Task time

Duplicates

TO DO: look at ES excluide duplicates (about 10%)