Person-Level Statistical Cognition - Laypeople

As pre-registered, we sought to collect data to attain 642 analyzable responses in total (i.e., 321 participants who passed attention checks for each between-subjects condition).

On the first wave of data collection (N = 706), applying the pre-registered exclusion criteria led to inadequate samples (i.e., Ns < 321) for both between-subjects datasets (Ns = 303 for one condition, 285 for another condition, totaling 588). However, we did not resample to reach our pre-registered Ns because these analyzable Ns (of 303 and 285) still yielded more than 90% power for our focal inferential tests.

Data Cleaning

Before data were loaded into R (below), the following changes were made:

1. Raw variable names were slightly adjusted to create more intuitive labels than what was originally specified in Qualtrics.

2. Participants were removed if they did not finish the entire survey.

3. Duplicate IP addresses were removed. There was only 1 instance of a duplicate IP address, leading to an N = 705.

4. All other identifying information was removed (e.g., IP addresses, longitude/latitude, etc.).

Loading Data/Packages

Before running this chunk, please load “Stat_Cog_Laypeople_OSF.csv” into the R environment.

options(scipen = 99) # turns off scientific notation
# packages should be loaded in the following order to avoid function conflicts
library(psych) # for describing data
library(effsize) # for mean difference effect sizes
library(sjstats) # for eta-squared effect sizes
library(correlation) # for cleaner correlation test output
library(rmcorr) # for repeated-measures correlation tests
library(cocor) # for comparing dependent correlation coefficients
library(tidyverse) # for data manipulation and plotting
library(rstatix) # for wilcoxon test effect sizes

Data Manipulation

Data Separation/Recombining

Data were separated into two distinct data sets (for each between-subjects condition). Then, a between-subjects variable was created within each dataset. Last, both datasets were recombined.

# creates dataset w/ people who read about simple psych experiment
simple_stat <- Stat_Cog_Laypeople_OSF %>%
  filter(Simple_Attn_Ck != "NA")

# creates dataset w/ people who read about complex psych experiment
complex_stat <- Stat_Cog_Laypeople_OSF %>%
  filter(Complex_Attn_Ck != "NA")

# create between-subjects condition variable
simple_stat$BSs_cond <- rep("Simple", nrow(simple_stat))
complex_stat$BSs_cond <- rep("Complex", nrow(complex_stat))

# recombine between-subjects data
stat_lay_all <- rbind(simple_stat, complex_stat)

stat_lay_all$BSs_cond <- as.factor(stat_lay_all$BSs_cond)
stat_lay_all$BSs_cond <- ordered(stat_lay_all$BSs_cond, levels = c("Simple", "Complex"))

Cleaning Data

stat_lay_clean <- stat_lay_all %>%
  filter(Finished == 1) %>% # finished whole survey
  filter(Complex_Attn_Ck == 1 | Simple_Attn_Ck == 1) # passed attn/comp check

Get Demographics for Clean Data

# gender
stat_lay_clean %>% 
  group_by(Gender) %>% 
  dplyr::summarize(gender = n()) # 0 = female; 1 = male; 2 = non-binary; 3 = other

# ethnicity
stat_lay_clean %>% 
  group_by(Ethnic) %>% 
  dplyr::summarize(ethnicity = n()) # 0 = white; 2 = black; 2 = american indian / alaskan native; 3 = asian; 4 = native hawaiian / pacific islander; 5 = other

# age
mean(stat_lay_clean$Age)

[1] 38.69218

sd(stat_lay_clean$Age)

[1] 11.2912

Creating Analysis Variables

# Main DVs
# create intuitive var names that collapse across between-subj conditions

# e.g., Simple_Q_Prop_0_Response = Simple Design / Empirical Q Prop Estimate / Anchor started at 0

stat_lay_clean$Emp_Prop_Est  <- rowSums(stat_lay_clean[, c("Simple_Q_Prop_0_Response", 
                                                           "Simple_Q_Prop_50_Response",
                                                           "Simple_Q_Prop_100_Response",
                                                           "Complex_Q_Prop_0_Response",
                                                           "Complex_Q_Prop_50_Response",
                                                           "Complex_Q_Prop_100_Response")], na.rm = T)
stat_lay_clean$Theory_Prop_Est  <- rowSums(stat_lay_clean[, c("Simple_T_Prop_0_Response", 
                                                           "Simple_T_Prop_50_Response",
                                                           "Simple_T_Prop_100_Response",
                                                           "Complex_T_Prop_0_Response",
                                                           "Complex_T_Prop_50_Response",
                                                           "Complex_T_Prop_100_Response")], na.rm = T)

stat_lay_clean$Emp_TF_Count  <- rowSums(stat_lay_clean[, c("Simple_Q_TF_Response_1",
                                                       "Simple_Q_TF_Response_2",
                                                       "Simple_Q_TF_Response_3",
                                                       "Simple_Q_TF_Response_4",
                                                       "Simple_Q_TF_Response_5",
                                                       "Simple_Q_TF_Response_6",
                                                       "Complex_Q_TF_Response_1",
                                                       "Complex_Q_TF_Response_2",
                                                       "Complex_Q_TF_Response_3",
                                                       "Complex_Q_TF_Response_4",
                                                       "Complex_Q_TF_Response_5",
                                                       "Complex_Q_TF_Response_6")], na.rm = T)
stat_lay_clean$Theory_TF_Count  <- rowSums(stat_lay_clean[, c("Simple_T_TF_Response_1",
                                                       "Simple_T_TF_Response_2",
                                                       "Simple_T_TF_Response_3",
                                                       "Simple_T_TF_Response_4",
                                                       "Simple_T_TF_Response_5",
                                                       "Simple_T_TF_Response_6",
                                                       "Complex_T_TF_Response_1",
                                                       "Complex_T_TF_Response_2",
                                                       "Complex_T_TF_Response_3",
                                                       "Complex_T_TF_Response_4",
                                                       "Complex_T_TF_Response_5",
                                                       "Complex_T_TF_Response_6")], na.rm = T)


# Pre-registered Cut-off Scores
# i.e., whether prop estimates are at 51% or higher
stat_lay_clean <- stat_lay_clean %>%
  mutate(Emp_Prop_Cutoff = case_when(
    Emp_Prop_Est >= 51 ~ "Simple Majority or Greater",
    Emp_Prop_Est < 51 ~ "Less than Simple Majority")) %>%
  mutate(Theory_Prop_Cutoff = case_when(
    Theory_Prop_Est >= 51 ~ "Simple Majority or Greater",
    Theory_Prop_Est < 51 ~ "Less than Simple Majority"))

# select only relevant data
stat_lay_clean <- stat_lay_clean %>%
  select(ResponseId,
         BSs_cond,
         Emp_Prop_Est:Theory_Prop_Cutoff,
         Data:Data_Length)

Descriptive Stats

Proportion Estimates

Empirical Distribution

describeBy(stat_lay_clean$Emp_Prop_Est, list(stat_lay_clean$BSs_cond), mat = T)

Empirical Cut-Off

table(stat_lay_clean$BSs_cond, stat_lay_clean$Emp_Prop_Cutoff)

         
          Less than Simple Majority Simple Majority or Greater
  Simple                         59                        244
  Complex                        33                        252

Theoretical Distribution

describeBy(stat_lay_clean$Theory_Prop_Est, list(stat_lay_clean$BSs_cond), mat = T)

Theoretical Cut-Off

table(stat_lay_clean$BSs_cond, stat_lay_clean$Theory_Prop_Cutoff)

         
          Less than Simple Majority Simple Majority or Greater
  Simple                         21                        282
  Complex                        22                        263

T/F Counts

Empirical T/F Counts

table(stat_lay_clean$BSs_cond, stat_lay_clean$Emp_TF_Count) # number of times t/f was marked true by a participant (out of 6 t/f statements where all statements were technically false; see OSF Qualtrics file)

         
           0  1  2  3  4  5  6
  Simple  13 26 41 70 71 76  6
  Complex  4 19 35 62 67 80 18

Theoretical T/F Counts

table(stat_lay_clean$BSs_cond, stat_lay_clean$Theory_TF_Count) # number of times t/f was marked true by a participant (out of 6 t/f statements where all statements were technically false; see OSF Qualtrics file)

         
           0  1  2  3  4  5  6
  Simple   8  7 23 48 74 83 60
  Complex  3 13 14 49 55 88 63

Plotting Proportion Estimates

Empirical Estimates

print(emp_prop_plot <- ggplot(stat_lay_clean, aes(Emp_Prop_Est)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Empirical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Theoretical Estimates

print(theory_prop_plot <- ggplot(stat_lay_clean, aes(Theory_Prop_Est)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Theoretical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Binomial Tests

Simple

Empirical Proportion

# using estimates from descriptive stats section
binom.test(x = 244, n = 303, p = 0.5, alternative = "greater", conf.level = 0.90)

    Exact binomial test

data:  244 and 303
number of successes = 244, number of trials = 303, p-value < 0.00000000000000022
alternative hypothesis: true probability of success is greater than 0.5
90 percent confidence interval:
 0.7729972 1.0000000
sample estimates:
probability of success 
             0.8052805 

Theoretical Proportion

# using estimates from descriptive stats section
binom.test(x = 282, n = 303, p = 0.5, alternative = "greater", conf.level = .90)

    Exact binomial test

data:  282 and 303
number of successes = 282, number of trials = 303, p-value < 0.00000000000000022
alternative hypothesis: true probability of success is greater than 0.5
90 percent confidence interval:
 0.9080944 1.0000000
sample estimates:
probability of success 
             0.9306931 

Complex

Empirical Proportion

# using estimates from descriptive stats section
binom.test(x = 252, n = 285, p = 0.5, alternative = "greater", conf.level = 0.90)

    Exact binomial test

data:  252 and 285
number of successes = 252, number of trials = 285, p-value < 0.00000000000000022
alternative hypothesis: true probability of success is greater than 0.5
90 percent confidence interval:
 0.8561042 1.0000000
sample estimates:
probability of success 
             0.8842105 

Theoretical Proportion

# using estimates from descriptive stats section
binom.test(x = 263, n = 285, p = 0.5, alternative = "greater", conf.level = 0.90)

    Exact binomial test

data:  263 and 285
number of successes = 263, number of trials = 285, p-value < 0.00000000000000022
alternative hypothesis: true probability of success is greater than 0.5
90 percent confidence interval:
 0.8984556 1.0000000
sample estimates:
probability of success 
              0.922807 

Additional Analyses

Data Experience

Breakdown

stat_lay_clean <- stat_lay_clean %>%
  mutate(Data_Exp = case_when(
    Data == 1 ~ "No",
    Data == 2 ~ "Yes",
    TRUE ~ "Other"))

stat_lay_clean$Data_Exp <- as.factor(stat_lay_clean$Data_Exp)
stat_lay_clean$Data_Exp <- ordered(stat_lay_clean$Data_Exp, 
                                        levels = c("No",
                                                   "Yes"))

table(stat_lay_clean$BSs_cond, stat_lay_clean$Data_Exp)

         
           No Yes
  Simple  198 105
  Complex 183 102

Empirical Proportion Plot

print(emp_prop_plot_dataexp <- ggplot(stat_lay_clean, aes(Emp_Prop_Est,
                                                              fill = Data_Exp)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
        scale_fill_grey() +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Empirical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))

Theoretical Proportion Plot

print(theory_prop_plot_dataexp <- ggplot(stat_lay_clean, aes(Theory_Prop_Est,
                                                              fill = Data_Exp)) +
  geom_histogram(binwidth = 1) +
  coord_cartesian(ylim = c(0,50), xlim = c(0,100)) +
        facet_wrap(~BSs_cond, nrow = 2) +
        geom_vline(xintercept = 50, linetype = "dashed", color = "black") +
       scale_fill_grey() +
        theme(legend.position = "right") +
        theme_classic() +
        xlab("Theoertical Proportion Estimates") +
        ylab("Participant Count") + 
        theme(axis.title.x = element_text(size = 14), 
              axis.title.y = element_text(size = 14),
              axis.text.x = element_text(color = "black", size = 12), 
              axis.text.y = element_text(color = "black", size = 12),
              legend.position = "right",
              legend.title = element_text(color = "black", size = 14),
              legend.text = element_text(color = "black", size = 12)))