Person-Level Statistical Cognition - Researchers

As pre-registered, we sought to collect data to attain 280 analyzable responses in total (i.e., 140 participants who passed attention checks for each between-subjects condition).

After two waves of data collection, we were only able to recruit N = 256. Applying the pre-registered exclusion criterion led to N = 244 (n = 123 for the “simple effect” and n = 121 for the “complex effect”). However, we did not resample to reach our pre-registered Ns because these analyzable Ns still yielded more than 90% power for our focal inferential tests.

Data Cleaning

Before data were loaded into R (below), the following changes were made:

1. Raw variable names were slightly adjusted to create more intuitive labels than what was originally specified in Qualtrics.

2. Participants were removed if they did not finish the entire survey.

3. All other identifying information was removed (e.g., IP addresses, longitude/latitude, etc.).

Loading Data/Packages

Before running this chunk, please load “Stat_Cog_Researchers.csv” into the R environment.

options(scipen = 99) # turns off scientific notation
# packages should be loaded in the following order to avoid function conflicts
library(psych) # for describing data
library(effsize) # for mean difference effect sizes
library(sjstats) # for eta-squared effect sizes
library(correlation) # for cleaner correlation test output
library(rmcorr) # for repeated-measures correlation tests
library(cocor) # for comparing dependent correlation coefficients
library(tidyverse) # for data manipulation and plotting
library(rstatix) # for wilcoxon test effect sizes

Data Manipulation

Data Separation/Recombining

Data were separated into two distinct data sets (for each between-subjects condition). Then, a between-subjects variable was created within each dataset. Last, both datasets were recombined.

# creates dataset w/ people who read about simple psych experiment
simple_stat <- Stat_Cog_Researchers_OSF %>%
  filter(Simple_Attn_Ck != "NA")

# creates dataset w/ people who read about complex psych experiment
complex_stat <- Stat_Cog_Researchers_OSF %>%
  filter(Complex_Attn_Ck != "NA")

# create between-subjects condition variable
simple_stat$BSs_cond <- rep("Simple", nrow(simple_stat))
complex_stat$BSs_cond <- rep("Complex", nrow(complex_stat))

# recombine between-subjects data
stat_res_all <- rbind(simple_stat, complex_stat)

stat_res_all$BSs_cond <- as.factor(stat_res_all$BSs_cond)
stat_res_all$BSs_cond <- ordered(stat_res_all$BSs_cond, levels = c("Simple", "Complex"))

Cleaning Data

stat_res_clean <- stat_res_all %>%
  filter(Finished == 1) %>% # finished whole survey
  filter(Complex_Attn_Ck == 1 | Simple_Attn_Ck == 1) # passed attn/comp check

Get Demographics for Clean Data

# gender
stat_res_clean %>% 
  group_by(Gender) %>% 
  dplyr::summarize(gender = n()) # 0 = female; 1 = male; 2 = non-binary; 3 = other

# ethnicity
stat_res_clean %>% 
  group_by(Ethnic) %>% 
  dplyr::summarize(ethnicity = n()) # 0 = white; 2 = black; 2 = american indian / alaskan native; 3 = asian; 4 = native hawaiian / pacific islander; 5 = other

# age
mean(stat_res_clean$Age)

[1] 33.09426

sd(stat_res_clean$Age)

[1] 11.34074

Creating Analysis Variables

# Main DVs
# create intuitive var names that collapse across between-subj conditions

# e.g., Simple_Q_Prop_0_Response = Simple Design / Empirical Q Prop Estimate / Anchor started at 0

stat_res_clean$Emp_Prop_Est  <- rowSums(stat_res_clean[, c("Simple_Q_Prop_0_Response", 
                                                           "Simple_Q_Prop_50_Response",
                                                           "Simple_Q_Prop_100_Response",
                                                           "Complex_Q_Prop_0_Response",
                                                           "Complex_Q_Prop_50_Response",
                                                           "Complex_Q_Prop_100_Response")], na.rm = T)
stat_res_clean$Theory_Prop_Est  <- rowSums(stat_res_clean[, c("Simple_T_Prop_0_Response", 
                                                           "Simple_T_Prop_50_Response",
                                                           "Simple_T_Prop_100_Response",
                                                           "Complex_T_Prop_0_Response",
                                                           "Complex_T_Prop_50_Response",
                                                           "Complex_T_Prop_100_Response")], na.rm = T)

stat_res_clean$Emp_TF_Count  <- rowSums(stat_res_clean[, c("Simple_Q_TF_Response_1",
                                                       "Simple_Q_TF_Response_2",
                                                       "Simple_Q_TF_Response_3",
                                                       "Simple_Q_TF_Response_4",
                                                       "Simple_Q_TF_Response_5",
                                                       "Simple_Q_TF_Response_6",
                                                       "Complex_Q_TF_Response_1",
                                                       "Complex_Q_TF_Response_2",
                                                       "Complex_Q_TF_Response_3",
                                                       "Complex_Q_TF_Response_4",
                                                       "Complex_Q_TF_Response_5",
                                                       "Complex_Q_TF_Response_6")], na.rm = T)
stat_res_clean$Theory_TF_Count  <- rowSums(stat_res_clean[, c("Simple_T_TF_Response_1",
                                                       "Simple_T_TF_Response_2",
                                                       "Simple_T_TF_Response_3",
                                                       "Simple_T_TF_Response_4",
                                                       "Simple_T_TF_Response_5",
                                                       "Simple_T_TF_Response_6",
                                                       "Complex_T_TF_Response_1",
                                                       "Complex_T_TF_Response_2",
                                                       "Complex_T_TF_Response_3",
                                                       "Complex_T_TF_Response_4",
                                                       "Complex_T_TF_Response_5",
                                                       "Complex_T_TF_Response_6")], na.rm = T)


# Pre-registered Cut-off Scores
# i.e., whether prop estimates are at 51% or higher
stat_res_clean <- stat_res_clean %>%
  mutate(Emp_Prop_Cutoff = case_when(
    Emp_Prop_Est >= 51 ~ "Simple Majority or Greater",
    Emp_Prop_Est < 51 ~ "Less than Simple Majority")) %>%
  mutate(Theory_Prop_Cutoff = case_when(
    Theory_Prop_Est >= 51 ~ "Simple Majority or Greater",
    Theory_Prop_Est < 51 ~ "Less than Simple Majority"))

# select only relevant data
stat_res_clean <- stat_res_clean %>%
  select(ResponseId,
         BSs_cond,
         Emp_Prop_Est:Theory_Prop_Cutoff,
         Design_Inf,
         Academia:`Political Beliefs_3`)

Descriptive Stats

Proportion Estimates

Empirical Distribution

describeBy(stat_res_clean$Emp_Prop_Est, list(stat_res_clean$BSs_cond), mat = T)

Empirical Cut-Off

table(stat_res_clean$BSs_cond, stat_res_clean$Emp_Prop_Cutoff)

         
          Less than Simple Majority Simple Majority or Greater
  Simple                         33                         90
  Complex                        24                         97

Theoretical Distribution

describeBy(stat_res_clean$Theory_Prop_Est, list(stat_res_clean$BSs_cond), mat = T)

Theoretical Cut-Off

table(stat_res_clean$BSs_cond, stat_res_clean$Theory_Prop_Cutoff)

         
          Less than Simple Majority Simple Majority or Greater
  Simple                         24                         99
  Complex                        12                        109

T/F Counts

Empirical T/F Counts

table(stat_res_clean$BSs_cond, stat_res_clean$Emp_TF_Count) # number of times t/f was marked true by a participant (out of 6 t/f statements where all statements were technically false; see OSF Qualtrics file)

         
           0  1  2  3  4  5  6
  Simple  34 37 29  8  9  6  0
  Complex 28 22 42 17  8  3  1

Theoretical T/F Counts

table(stat_res_clean$BSs_cond, stat_res_clean$Theory_TF_Count) # number of times t/f was marked true by a participant (out of 6 t/f statements where all statements were technically false; see OSF Qualtrics file)

         
           0  1  2  3  4  5  6
  Simple  13 12 36 29 22  9  2
  Complex 11 14 32 36 16  9  3

Plotting Proportion Estimates

Empirical Estimates

print(emp_prop_plot <- ggplot(stat_res_clean, aes(Emp_Prop_Est)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Empirical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Theoretical Estimates

print(theory_prop_plot <- ggplot(stat_res_clean, aes(Theory_Prop_Est)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Theoretical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Binomial Tests

Simple

Empirical Proportion

# using estimates from descriptive stats section
binom.test(x = 90, n = 123, p = 0.5, alternative = "greater", conf.level = 0.90)

    Exact binomial test

data:  90 and 123
number of successes = 90, number of trials = 123, p-value = 0.0000001372
alternative hypothesis: true probability of success is greater than 0.5
90 percent confidence interval:
 0.673807 1.000000
sample estimates:
probability of success 
             0.7317073 

Theoretical Proportion

# using estimates from descriptive stats section
binom.test(x = 99, n = 123, p = 0.5, alternative = "greater", conf.level = 0.90)

    Exact binomial test

data:  99 and 123
number of successes = 99, number of trials = 123, p-value = 0.000000000002583
alternative hypothesis: true probability of success is greater than 0.5
90 percent confidence interval:
 0.7513548 1.0000000
sample estimates:
probability of success 
              0.804878 

Complex

Empirical Proportion

# using estimates from descriptive stats section
binom.test(x = 97, n = 121, p = 0.5, alternative = "greater", conf.level = 0.90)

    Exact binomial test

data:  97 and 121
number of successes = 97, number of trials = 121, p-value = 0.000000000006721
alternative hypothesis: true probability of success is greater than 0.5
90 percent confidence interval:
 0.7473858 1.0000000
sample estimates:
probability of success 
             0.8016529 

Theoretical Proportion

# using estimates from descriptive stats section
binom.test(x = 109, n = 121, p = 0.5, alternative = "greater", conf.level = 0.90)

    Exact binomial test

data:  109 and 121
number of successes = 109, number of trials = 121, p-value < 0.00000000000000022
alternative hypothesis: true probability of success is greater than 0.5
90 percent confidence interval:
 0.8565791 1.0000000
sample estimates:
probability of success 
             0.9008264 

Additional Analyses

Design Inference

table(stat_res_clean$BSs_cond, stat_res_clean$Design_Inf)

         
           1  2  3
  Simple  23 38 62
  Complex 22 35 64

# "In your opinion, what kinds of experimental designs are necessary to make inferences about person-level psychological effects?
# 1 = between-subj; #2 = within-subj; #3 = either

Research Experience

Breakdown

stat_res_clean <- stat_res_clean %>%
  mutate(Research_Role = case_when(
    Research_Exp == 4 ~ "Undergrad",
    Research_Exp == 5 ~ "Post-bac/RA/Lab manager",
    Research_Exp == 6 ~ "Master's student",
    Research_Exp == 7 ~ "PhD student",
    Research_Exp == 8 ~ "Post-doc",
    Research_Exp == 9 ~ "Professor",
    TRUE ~ "Other"))

stat_res_clean$Research_Role <- as.factor(stat_res_clean$Research_Role)
stat_res_clean$Research_Role <- ordered(stat_res_clean$Research_Role, 
                                        levels = c("Other",
                                                   "Undergrad",
                                                   "Post-bac/RA/Lab manager",
                                                   "Master's student",
                                                   "PhD student",
                                                   "Post-doc",
                                                   "Professor"))

table(stat_res_clean$BSs_cond, stat_res_clean$Research_Role)

         
          Other Undergrad Post-bac/RA/Lab manager Master's student PhD student Post-doc Professor
  Simple      5         2                       7               10          53        6        40
  Complex     8         5                      10                8          43       11        36

Empirical Proportion Plot

print(emp_prop_plot_researchexp <- ggplot(stat_res_clean, aes(Emp_Prop_Est,
                                                              fill = Research_Role)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        theme(legend.position = "right") +
        scale_fill_grey() +
        theme_classic() +
        xlab("Empirical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Theoretical Proportion Plot

print(theory_prop_plot_researchexp <- ggplot(stat_res_clean, aes(Theory_Prop_Est,
                                                              fill = Research_Role)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        scale_fill_grey() +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Theoertical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Teaching Experience

Breakdown

stat_res_clean <- stat_res_clean %>%
  mutate(Teaching_Role = case_when(
    `Stats/Meth` == 0 ~ "None",
    `Stats/Meth` == 1 ~ "Research Methods",
    `Stats/Meth` == 2 ~ "Statistics",
    `Stats/Meth` == 3 ~ "Both Research Methods & Statistics"))

stat_res_clean$Teaching_Role <- as.factor(stat_res_clean$Teaching_Role)
stat_res_clean$Teaching_Role <- ordered(stat_res_clean$Teaching_Role, 
                                        levels = c("None",
                                                   "Research Methods",
                                                   "Statistics",
                                                   "Both Research Methods & Statistics"))

table(stat_res_clean$BSs_cond, stat_res_clean$Teaching_Role)

         
          None Research Methods Statistics Both Research Methods & Statistics
  Simple    53               27         11                                 32
  Complex   71               20         11                                 19

Empirical Proportion Plot

print(emp_prop_plot_teachingexp <- ggplot(stat_res_clean, aes(Emp_Prop_Est,
                                                              fill = Teaching_Role)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        scale_fill_grey() +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Empirical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Theoretical Proportion Plot

print(theory_prop_plot_teachingexp <- ggplot(stat_res_clean, aes(Theory_Prop_Est,
                                                              fill = Teaching_Role)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        scale_fill_grey() +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Theoretical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Academic Experience (Generally)

Breakdown

table(stat_res_clean$BSs_cond, stat_res_clean$Academia)

         
            0   1
  Simple    5 118
  Complex   8 113

Empirical Proportion Scatterplot

print(emp_prop_ac_scatter <- ggplot(data = stat_res_clean, aes(x = Academia_Length, 
                                                               y = Emp_Prop_Est)) +
        geom_jitter(alpha = 0.5) +
        geom_smooth(method = 'lm') +
        facet_wrap(BSs_cond~., nrow = 2) +
        scale_x_continuous(limits = c(-1,55), breaks = c(0,10,20,30,40,50)) +
        scale_y_continuous(limits = c(-1,101), breaks = c(0,20,40,60,80,100)) +
        geom_hline(yintercept = 50, linetype = "dashed", color = "black") +
        theme_classic() +
        xlab("Academic Experience (in years)") +
        ylab("Empirical Proportion Estimate") +
          theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Theoretical Proportion Scatterplot

print(theory_prop_ac_scatter <- ggplot(data = stat_res_clean, aes(x = Academia_Length, 
                                                               y = Theory_Prop_Est)) +
        geom_jitter(alpha = 0.5) +
        geom_smooth(method = 'lm') +
        facet_wrap(BSs_cond~., nrow = 2) +
        scale_x_continuous(limits = c(-1,55), breaks = c(0,10,20,30,40,50)) +
        scale_y_continuous(limits = c(-1,101), breaks = c(0,20,40,60,80,100)) +
        geom_hline(yintercept = 50, linetype = "dashed", color = "black") +
        theme_classic() +
        xlab("Academic Experience (in years)") +
        ylab("Theoretical Proportion Estimate") +
          theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))