Xu and Tenenbaum 2007b Replication

Supplementary Information

---

---

TABLE OF CONTENTS
     (1) Original Xu and Tenenabum (2007b)
     (2) Exp. 1 - online
     (3) Exp. 2 - online
     (4) Exp. 3 - in person
     (5) Exp. 4 - online
     (6) Exp. 5 - online
     (7) All
     (8) Power Analysis for Exp. 5

This document was created from an R Markdown file. The R Markdown file can be found here. All analyses and plots can be reproduced from the raw data with the code in this file. This document also contains links to the experimental tasks.

---

---

There are different possible criteria for categorizing a participant as a subordinate or basic-level generalizer. X&T find that participants respond consistently across questions as either a subordinate or a basic generalizer for each of the two trials. In contrast, we observed varied responses across questions by participants in our sample. Thus, we adopt two criteria in analyzing our data: a “strict” criteria where we only include participants if they responded consistently (“yes” or “no” to all basic-level questions and “yes” or “no” to all subordinate level questions), and a “liberal” criteria where we counted a particpant as a basic generalizer if they responded “yes” to any of the basic level questions. The results of the liberal criteria are reported in the Main Text, and the results of the strict criteria are reported in Appendix A.

## [1] "Strict criteria:  TRUE"

Original Xu and Tenenbaum (2007b)

X&T 2007b data - adults

N = 14
N_per_condition = N/2  # 2 conditions (teacher + learner)
# Note: The proportions described in the original report were obtained by aggregating across *trials* not participants (2 trials/participant). We correct for this here by taking the proportion of participants, which results in slightly different proportions than those reported in the original paper.

# make raw data
prop_sub_teacher = .928 
n_sub_teacher = round(prop_sub_teacher * N_per_condition)
n_basic_teacher = N_per_condition - n_sub_teacher 
teacher_xt_data = as.factor(c(rep("sub", n_sub_teacher),
                              rep("basic", n_basic_teacher)))

prop_sub_learner = .357
n_sub_learner = round(prop_sub_learner * N_per_condition)
n_basic_learner = N_per_condition - n_sub_learner
learner_xt_data = as.factor(c(rep("sub", n_sub_learner), 
                              rep("basic", n_basic_learner)))

xt_data.adults = data.frame(response = c(as.character(teacher_xt_data),
                                       as.character(learner_xt_data)))
xt_data.adults$Answer.sample[1:N_per_condition] = "teacher"
xt_data.adults$Answer.sample[(N_per_condition + 1):
                              (N_per_condition * 2)] = "learner"

# get props
xt.a.props = get_bootstrapped_props(xt_data.adults, "response", "X&T (2007b) adults")

X&T 2007b data - children

N = 24
N_per_condition = N/2 # 2 conditions

# make raw data
prop_sub_teacher = .71
n_sub_teacher = round(prop_sub_teacher * N_per_condition)
n_basic_teacher = N_per_condition - n_sub_teacher 
teacher_xt_data = as.factor(c(rep("sub", n_sub_teacher),
                              rep("basic", n_basic_teacher)))

prop_sub_learner = .29
n_sub_learner = round(prop_sub_learner * N_per_condition)
n_basic_learner = N_per_condition - n_sub_learner
learner_xt_data = as.factor(c(rep("sub", n_sub_learner), 
                              rep("basic", n_basic_learner)))

xt_data.children = data.frame(response = c(as.character(teacher_xt_data),
                                       as.character(learner_xt_data)))
xt_data.children$Answer.sample[1:N_per_condition] = "teacher"
xt_data.children$Answer.sample[(N_per_condition + 1):
                               (N_per_condition * 2)] = "learner"

# get props
xt.c.props = get_bootstrapped_props(xt_data.children, "response", "X&T (2007b) children")

Experiment 1

Read in data and pre-process.

t1 = read.csv("../data/anonymized/turk_replication_1_A.csv")

# make factors
t1$Answer.sample <- factor(t1$Answer.sample, labels=c('teacher','learner')) # sample0 = teacher, sample1 = learner
t1$Answer.label <- factor(t1$Answer.label, labels=c('nolabel','label')) # 0 = nolabel, 1 = label
t1 <- colwise(as.factor)(t1)

Filter.

# get number of exclusions by category
t1.exclusion.ns = t1 %>%
              summarise(n_noLabelParticipants = length(which(Answer.label != 'label')),
                        n_repeatWorkers = length(which(duplicated(workerids))),
                        n_badTraining = length(which(Answer.click1 == "\"false\"" | 
                                                       Answer.click2 == "\"false\"")),
                        n_badGeneralize = length(which(Answer.Qwcheck != 0)),
                        n_badFilter = length(which(Answer.question1 == 'FALSE')))

t1.exclusion.ns

##   n_noLabelParticipants n_repeatWorkers n_badTraining n_badGeneralize
## 1                    48               8            21               8
##   n_badFilter
## 1           0

# subset data
t1.f = t1 %>%
       filter(Answer.label == 'label') %>% #remove nolabel participants (run 4)
       filter(!duplicated(workerids)) %>% #participants who completed multiple runs
       filter(Answer.click1 == "\"correct\"" & 
                Answer.click2 == "\"correct\"") %>% #correct training items (learning only)
       filter(Answer.Qwcheck == 0)  %>% #check generalization question
       filter(Answer.question1 == 'TRUE') #filter question

dim(t1)[1] # total

## [1] 350

dim(t1)[1] - t1.exclusion.ns[1,"n_noLabelParticipants"] - t1.exclusion.ns[1,"n_repeatWorkers"] # actual total

## [1] 294

dim(t1.f)[1] # total with exclusions

## [1] 274

Categorize response patterns based on criterion.

sub <- t1.f$Answer.Qwsmm1 == 0 & t1.f$Answer.Qwsmm2 == 0 & 
  t1.f$Answer.Qwsm1 == 1 & t1.f$Answer.Qwsm2 == 1
if (strict){
  basic <- (t1.f$Answer.Qwsmm1 == 1 & t1.f$Answer.Qwsmm2 == 1) &
      t1.f$Answer.Qwsm1 == 1 & t1.f$Answer.Qwsm2 == 1 
} else {
  basic <- (t1.f$Answer.Qwsmm1 == 1 | t1.f$Answer.Qwsmm2 == 1) &
       t1.f$Answer.Qwsm1 == 1 & t1.f$Answer.Qwsm2 == 1 
}

t1.f$w.ans.cat <- "other"
t1.f$w.ans.cat[sub] <- "sub"
t1.f$w.ans.cat[basic] <- "basic"
t1.f$w.ans.cat = factor(t1.f$w.ans.cat, levels = c("sub", "basic", "other"))

# filter out other responses
length(which(t1.f$w.ans.cat == "other"))

## [1] 94

t1.f = t1.f[t1.f$w.ans.cat != "other",]
t1.f$w.ans.cat = droplevels(t1.f$w.ans.cat)

# get props
t1.f.props = get_bootstrapped_props(t1.f, "w.ans.cat", "Turk #1")

Stats

t1_tab = table(t1.f$w.ans.cat, t1.f$Answer.sample)[c("sub", "basic"),]
chisq.test(t1_tab)

## 
##  Pearson's Chi-squared test with Yates' continuity correction
## 
## data:  t1_tab
## X-squared = 0.075348, df = 1, p-value = 0.7837

Experiment 2

Read in data and pre-process

t2 <- read.csv("../data/anonymized/turk_replication_2_A.csv", header = T)
t2  <- t2 [,c(-1:-19,-21:-32,-36,-41:-43,-54:-55,-56)] # remove unnecessary columns

# make factors
t2$Answer.sample <- factor(t2$Answer.sample,
                           labels = c('teacher','learner')) # sample0 = teacher, sample1 = learner
t2 <- colwise(as.factor)(t2)

Filter.

# get number of exclusions by category
t2.exclusion.ns = t2 %>%
              summarise(n_badTraining = length(which(Answer.click1 != "\"correct\"" |
                                                       Answer.click2 != "\"correct\"")),
                        n_badGeneralize = length(which(Answer.Qcheck1 != 0 | Answer.Qcheck2 != 0)),
                        n_badFilter = length(which(Answer.question1 != 'true' | 
                                                     Answer.question2 != 'true' | 
                                                     Answer.question3 != 'true' | 
                                                     Answer.question4 !='true')))
t2.exclusion.ns

##   n_badTraining n_badGeneralize n_badFilter
## 1            22              15           9

# subset data
t2.f = t2 %>%
       filter(Answer.click1 == "\"correct\"" &
                Answer.click2 == "\"correct\"") %>% # take out those who click on wrong training items
      filter(Answer.Qcheck1 == 0 & Answer.Qcheck2 == 0) %>% # take out if missed check generalization question
      filter(Answer.question1 != 'true' | Answer.question2 != 'true' | 
               Answer.question3 == 'true' & Answer.question4 == 'true')  # take out those who missed attention check questions

dim(t2)[1] # total

## [1] 150

dim(t2.f)[1] # total with exclusions

## [1] 118

Categorize response patterns based on criterion.

sub <-  t2.f$Answer.Qproper1 == 1 & t2.f$Answer.Qproper2 == 1 & 
  t2.f$Answer.Qproper3 == 1 & t2.f$Answer.Qsub1 == 1 &
  t2.f$Answer.Qsub2 == 1 & t2.f$Answer.Qbasic1 == 0 &
  t2.f$Answer.Qbasic2 == 0 & t2.f$Answer.Qbasic3 == 0 

if (strict){
  basic <- t2.f$Answer.Qproper1 == 1 & t2.f$Answer.Qproper2 == 1 & 
    t2.f$Answer.Qproper3 == 1 & t2.f$Answer.Qsub1 == 1 &
    t2.f$Answer.Qsub2 == 1 & t2.f$Answer.Qbasic1 == 1 &
    t2.f$Answer.Qbasic2 == 1 & t2.f$Answer.Qbasic3 == 1 
} else {
  basic <- t2.f$Answer.Qproper1 == 1 & t2.f$Answer.Qproper2 == 1 & 
    t2.f$Answer.Qproper3 == 1 & t2.f$Answer.Qsub1 == 1 &
    t2.f$Answer.Qsub2 == 1 & (t2.f$Answer.Qbasic1 == 1 |
    t2.f$Answer.Qbasic2 == 1 | t2.f$Answer.Qbasic3 == 1)
}

t2.f$w.ans.cat <- "other"
t2.f$w.ans.cat[sub] <- "sub"
t2.f$w.ans.cat[basic] <- "basic"
t2.f$w.ans.cat = factor(t2.f$w.ans.cat, levels = c("sub", "basic", "other"))

# filter out "other" responses
length(which(t2.f$w.ans.cat == "other"))

## [1] 36

t2.f = t2.f[t2.f$w.ans.cat != "other",]
t2.f$w.ans.cat = droplevels(t2.f$w.ans.cat)

# get props
t2.f.props = get_bootstrapped_props(t2.f, "w.ans.cat", "Turk #2")

Stats

t2_tab = table(t2.f$w.ans.cat, t2.f$Answer.sample)[c("sub", "basic"),]
chisq.test(t2_tab)

## 
##  Pearson's Chi-squared test with Yates' continuity correction
## 
## data:  t2_tab
## X-squared = 0.06999, df = 1, p-value = 0.7914

Experiment 3 - In Person Replication

l1 <- read.csv("../data/anonymized/inlab_replication.csv")

# make factors
l1 <- colwise(as.factor)(l1)
names(l1)[names(l1) == "sample"] = "Answer.sample"

Filter.

l1.exclusion.ns = l1 %>%
                  summarise(n_badTraining = length(
                    which(t2_a == 0 | t3_a == 0 |t2_b == 0 | t2_b == 0)))
l1.exclusion.ns

##   n_badTraining
## 1             1

# subset data to those with correct training
l1.f = l1 %>%
       filter(t2_a == 1 & t3_a == 1 & t2_b == 1 & t2_b == 1)

dim(l1)[1] # total

## [1] 41

dim(l1.f)[1] # total with exclusions

## [1] 40

Categorize response patterns based on criterion.

sub <- l1.f$basic1_a == 0 & l1.f$basic2_a == 0 & 
       l1.f$basic1_b == 0 & l1.f$basic2_b == 0 & 
       (l1.f$sub1_a == 1 & l1.f$sub2_a == 1 & 
          l1.f$sub1_b == 1 & l1.f$sub2_b == 1)
if (strict){
    basic <- (l1.f$basic1_a == 1 & l1.f$basic2_a == 1 &
            l1.f$basic1_b == 1 & l1.f$basic2_b == 1) & 
           (l1.f$sub1_a == 1 & l1.f$sub2_a == 1 & 
              l1.f$sub1_b == 1 & l1.f$sub2_b == 1)
  
} else {
  basic <- (l1.f$basic1_a == 1 | l1.f$basic2_a == 1 | 
            l1.f$basic1_b == 1 | l1.f$basic2_b == 1) & 
           (l1.f$sub1_a == 1 & l1.f$sub2_a == 1 & 
              l1.f$sub1_b == 1 & l1.f$sub2_b == 1)
}

l1.f$w.ans.cat <- "other"
l1.f$w.ans.cat[sub] <- "sub"
l1.f$w.ans.cat[basic] <- "basic"
l1.f$w.ans.cat <- factor(l1.f$w.ans.cat, c("sub","basic","other"))

# filter out "other"" responses
length(which(l1.f$w.ans.cat  == "other"))

## [1] 14

l1.f = l1.f[l1.f$w.ans.cat != "other",]
l1.f$w.ans.cat = droplevels(l1.f$w.ans.cat)

# get props
l1.f.props = get_bootstrapped_props(l1.f, "w.ans.cat", "In person")

Stats

l1_tab = table(l1.f$w.ans.cat, l1.f$sample)[c("sub", "basic"),]
chisq.test(l1_tab)

## 
##  Pearson's Chi-squared test with Yates' continuity correction
## 
## data:  l1_tab
## X-squared = 0.6303, df = 1, p-value = 0.4272

Experiment 4 - Turk Replication #3

Read in data and pre-process

t3 <- read.csv("../data/anonymized/turk_replication_3_A.csv")

# make factors
t3$Answer.sample <- factor(t3$Answer.sample, 
                           labels=c('teacher','learner')) # sample0 = teacher, sample1 = learner
t3 <- colwise(as.factor)(t3)

Filter.

t3.exclusion.ns = t3 %>%
              summarise(n_badTraining = length(which(Answer.click1 != "\"correct\"" | 
                                                       Answer.click2 != "\"correct\"")),
                        n_badGeneralize = length(which(Answer.Qcheck1 != 0 | Answer.Qcheck2 != 0)),
                        n_badFilter = length(which(Answer.question1 != 'TRUE' | Answer.question2 != 'TRUE'| 
                                Answer.question3 != 'TRUE' | Answer.question4 != 'TRUE')))
t3.exclusion.ns

##   n_badTraining n_badGeneralize n_badFilter
## 1            27              17          10

# subset data
t3.f = t3 %>%
      filter(Answer.click1 == "\"correct\"" & 
                Answer.click2 == "\"correct\"") %>% # take out those who click on wrong training items
      filter(Answer.Qcheck1 == 0 & Answer.Qcheck2 == 0) %>% # take out if missed check generalization question
      filter(Answer.question1 == 'TRUE' & Answer.question2 == 'TRUE' & 
            Answer.question3 == 'TRUE' & Answer.question4 == 'TRUE') # take out those who missed filter question

dim(t3)[1] # total

## [1] 200

dim(t3.f)[1] # total with exclusions

## [1] 161

Categorize response patterns based on criterion.

sub <-  t3.f$Answer.Qproper1 == 1 & t3.f$Answer.Qproper2 == 1 & 
    t3.f$Answer.Qproper3 == 1 & t3.f$Answer.Qsub1 == 1 &
    t3.f$Answer.Qsub2 == 1 & t3.f$Answer.Qbasic1 == 0 &
    t3.f$Answer.Qbasic2 == 0 & t3.f$Answer.Qbasic3 == 0 

if (strict){
  basic <- t3.f$Answer.Qproper1 == 1 & t3.f$Answer.Qproper2 == 1 & 
    t3.f$Answer.Qproper3 == 1 & t3.f$Answer.Qsub1 == 1 &
    t3.f$Answer.Qsub2 == 1 & t3.f$Answer.Qbasic1 == 1 &
    t3.f$Answer.Qbasic2 == 1 & t3.f$Answer.Qbasic3 == 1 
} else {
  basic <- t3.f$Answer.Qproper1 == 1 & t3.f$Answer.Qproper2 == 1 & 
      t3.f$Answer.Qproper3 == 1 & t3.f$Answer.Qsub1 == 1 &
      t3.f$Answer.Qsub2 == 1 & (t3.f$Answer.Qbasic1 == 1 |
      t3.f$Answer.Qbasic2 == 1 | t3.f$Answer.Qbasic3 == 1)
}

t3.f$w.ans.cat <- "other"
t3.f$w.ans.cat[sub] <- "sub"
t3.f$w.ans.cat[basic] <- "basic"
t3.f$w.ans.cat = factor(t3.f$w.ans.cat, levels = c("sub", "basic", "other"))

# filter out "other"" responses
length(which(t3.f$w.ans.cat == "other"))

## [1] 31

t3.f = t3.f[t3.f$w.ans.cat != "other",]
t3.f$w.ans.cat = droplevels(t3.f$w.ans.cat)

# get props
t3.f.props = get_bootstrapped_props(t3.f, "w.ans.cat", "Turk #3")

Stats

t3_tab = table(t3.f$w.ans.cat, t3.f$Answer.sample)[c("sub", "basic"),]
chisq.test(t3_tab)

## 
##  Pearson's Chi-squared test with Yates' continuity correction
## 
## data:  t3_tab
## X-squared = 4.1256, df = 1, p-value = 0.04224

Experiment 5 - Turk Replication #4

Read in data and pre-process

t4 <- read.csv("../data/anonymized/turk_replication_4_A.csv")

# make factors
t4$Answer.sample <- factor(t4$Answer.sample, 
                           labels = c('teacher','learner')) # sample0 = teacher, sample1 = learner
t4 <- colwise(as.factor)(t4)

Filter.

t4.exclusion.ns = t4 %>%
              summarise(n_badTraining = length(which(Answer.click1 != "\"correct\"" | 
                                                       Answer.click2 != "\"correct\"")),
                        n_badGeneralize = length(which(Answer.Qcheck1 != 0 | Answer.Qcheck2 != 0)),
                        n_badFilter = length(which(Answer.question1 != 'TRUE' | Answer.question2 !='TRUE'| 
                                Answer.question3 != 'TRUE' | Answer.question4 !='TRUE')))
t4.exclusion.ns

##   n_badTraining n_badGeneralize n_badFilter
## 1            26              15          10

# subset data
t4.f = t4 %>%
      filter(Answer.click1 == "\"correct\"" & 
                Answer.click2 == "\"correct\"") %>% # take out those who click on wrong training items
      filter(Answer.Qcheck1 == 0 & Answer.Qcheck2 == 0) %>% # take out if missed check generalization question
      filter(Answer.question1 == 'TRUE' & Answer.question2 == 'TRUE' & 
            Answer.question3 == 'TRUE' & Answer.question4 == 'TRUE')# take out those who missed filter question

dim(t4)[1] # total

## [1] 200

dim(t4.f)[1] # total with exclusions

## [1] 163

Categorize response patterns based on criterion.

sub <-  t4.f$Answer.Qproper1 == 1 & t4.f$Answer.Qproper2 == 1 & 
    t4.f$Answer.Qproper3 == 1 & t4.f$Answer.Qsub1 == 1 &
    t4.f$Answer.Qsub2 == 1 & t4.f$Answer.Qbasic1 == 0 &
    t4.f$Answer.Qbasic2 == 0 & t4.f$Answer.Qbasic3 == 0 

if (strict){
  basic <- t4.f$Answer.Qproper1 == 1 & t4.f$Answer.Qproper2 == 1 & 
    t4.f$Answer.Qproper3 == 1 & t4.f$Answer.Qsub1 == 1 &
    t4.f$Answer.Qsub2 == 1 & t4.f$Answer.Qbasic1 == 1 &
    t4.f$Answer.Qbasic2 == 1 & t4.f$Answer.Qbasic3 == 1 
} else {
  basic <- t4.f$Answer.Qproper1 == 1 & t4.f$Answer.Qproper2 == 1 & 
      t4.f$Answer.Qproper3 == 1 & t4.f$Answer.Qsub1 == 1 &
      t4.f$Answer.Qsub2 == 1 & (t4.f$Answer.Qbasic1 == 1 |
      t4.f$Answer.Qbasic2 == 1 | t4.f$Answer.Qbasic3 == 1)
}

t4.f$w.ans.cat <- "other"
t4.f$w.ans.cat[sub] <- "sub"
t4.f$w.ans.cat[basic] <- "basic"
t4.f$w.ans.cat = factor(t4.f$w.ans.cat, levels = c("sub", "basic", "other"))

# filter out "other"" responses
length(which(t4.f$w.ans.cat == "other"))

## [1] 30

t4.f = t4.f[t4.f$w.ans.cat != "other",]
t4.f$w.ans.cat = droplevels(t4.f$w.ans.cat)

# get props
t4.f.props = get_bootstrapped_props(t4.f, "w.ans.cat", "Turk #4")

Stats

t4_tab = table(t4.f$w.ans.cat, t4.f$Answer.sample)[c("sub", "basic"),]
chisq.test(t4_tab)

## 
##  Pearson's Chi-squared test with Yates' continuity correction
## 
## data:  t4_tab
## X-squared = 1.0112, df = 1, p-value = 0.3146

All experiments

All proportions

# merge together all raw data 
xt_data.adults$exp = "X&T adults"
xt_data.children$exp = "X&T children"
t1.f$exp = "Exp. 1"
t2.f$exp = "Exp. 2"
l1.f$exp = "Exp. 3"
t3.f$exp = "Exp. 4"
t4.f$exp = "Exp. 5"
xt_data.adults$w.ans.cat = xt_data.adults$response
xt_data.children$w.ans.cat = xt_data.children$response

all.data.f = rbind(xt_data.children[,c("Answer.sample", "w.ans.cat", "exp")],
                  xt_data.adults[,c("Answer.sample", "w.ans.cat", "exp")],
                   t1.f[,c("Answer.sample", "w.ans.cat", "exp")],
                   t2.f[,c("Answer.sample", "w.ans.cat", "exp")],
                   l1.f[,c("Answer.sample", "w.ans.cat", "exp")],
                   t3.f[,c("Answer.sample", "w.ans.cat", "exp")],
                   t4.f[,c("Answer.sample", "w.ans.cat", "exp")])
all.data.f <- colwise(as.factor)(all.data.f)

# get props by experiment
all.data.f.props = all.data.f %>%
                    group_by(exp) %>%
                    do(get_bootstrapped_props(.,"w.ans.cat", .$exp[1])) 

# re-order experiments
all.data.f.props$exp = factor(all.data.f.props$exp,
                           levels(all.data.f.props$exp)[c(7, 6, 1:5)]) 

All effect sizes

# spread props and n_cond
all.data.f.props.basic = all.data.f.props %>%
                        filter(response.cat == "basic") %>%
                        select(-cill, -ciul, -response.cat, -prop, -n) %>%
                        spread(Answer.sample, n_cond, fill = F) %>%
                        rename(teacher_n = teacher, learner_n = learner)

all.data.f.props.basic = all.data.f.props  %>% 
                        filter(response.cat == "basic") %>%
                        select(-cill, -ciul, -response.cat, -n_cond, -n) %>%
                        spread(Answer.sample, prop, fill = F) %>%
                        rename(teacher_prop = teacher, learner_prop = learner) %>%
                        left_join(all.data.f.props.basic)

# get all effect sizes (compute.es package)
all.es = propes(all.data.f.props.basic$learner_prop, 
           all.data.f.props.basic$teacher_prop, 
           all.data.f.props.basic$learner_n,
           all.data.f.props.basic$teacher_n, 
           verbose = F)

all.es$exp = all.data.f.props.basic$exp # add exp ids

# fixed effects (metafor package)
fixed.effects.d = rma(d, var.d, data = all.es, method = "FE")
fixed.effects.d

## 
## Fixed-Effects Model (k = 7)
## 
## Test for Heterogeneity: 
## Q(df = 6) = 7.7215, p-val = 0.2592
## 
## Model Results:
## 
## estimate       se     zval     pval    ci.lb    ci.ub          
##   0.3596   0.1151   3.1247   0.0018   0.1340   0.5852       ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# random effects
random.effects.d = rma(d, var.d, data = all.es)
random.effects.d

## 
## Random-Effects Model (k = 7; tau^2 estimator: REML)
## 
## tau^2 (estimated amount of total heterogeneity): 0.0133 (SE = 0.0591)
## tau (square root of estimated tau^2 value):      0.1154
## I^2 (total heterogeneity / total variability):   11.62%
## H^2 (total variability / sampling variability):  1.13
## 
## Test for Heterogeneity: 
## Q(df = 6) = 7.7215, p-val = 0.2592
## 
## Model Results:
## 
## estimate       se     zval     pval    ci.lb    ci.ub          
##   0.3784   0.1266   2.9897   0.0028   0.1303   0.6264       ** 
## 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Power analysis for Exp. 5

based on exp 4

## METHOD 1
t3_tab

##        
##         teacher learner
##   sub        63      40
##   basic      10      17

chisq.test(t3_tab)$statistic

## X-squared 
##  4.125626

sum(t3_tab)

## [1] 130

w1 =  sqrt(chisq.test(t3_tab)$statistic/sum(t3_tab))
power.analysis = pwr.chisq.test(w1, df = 1, 
                                sig.level = 0.05, power = .95)
power.analysis

## 
##      Chi squared power calculation 
## 
##               w = 0.1781449
##               N = 409.468
##              df = 1
##       sig.level = 0.05
##           power = 0.95
## 
## NOTE: N is the number of observations

print(paste("N for .95 power:", power.analysis$N))

## [1] "N for .95 power: 409.468047848553"

print(paste("Extra: ", .35 * power.analysis$N))

## [1] "Extra:  143.313816746994"

## METHOD 2
w2 =  sqrt(sum(((prop.table(t3_tab, 2)-.5)^2)/.5))
power.analysis = pwr.chisq.test(w2, df = 1, 
                                sig.level = 0.05, power = .95)
power.analysis

## 
##      Chi squared power calculation 
## 
##               w = 0.8306233
##               N = 18.83469
##              df = 1
##       sig.level = 0.05
##           power = 0.95
## 
## NOTE: N is the number of observations

print(paste("N for .95 power:", power.analysis$N))

## [1] "N for .95 power: 18.8346859569469"

print(paste("Extra: ", .35 * power.analysis$N))

## [1] "Extra:  6.5921400849314"

#OLD
# fixed effects
fixed.effects = rma(d, var.d, data = all.es[2:6,], method = "FE")
power.analysis = pwr.chisq.test(w = fixed.effects$b, df = 1, 
                                sig.level = 0.05, power = .95)
power.analysis
print(paste("N for .95 power (fixed effects):", power.analysis$N))

# random effects
random.effects = rma(d , var.d, data = all.es[2:6,])
power.analysis = pwr.chisq.test(w = random.effects$b, df = 1, 
                                sig.level = 0.05, power = .95)
power.analysis
print(paste("N for .95 power (random effects):", power.analysis$N))