Pilot B Description

This is my submission for progress check-in 3 using pilot B data (Prolific participants) in regards to replicating Muenks et al. (2020) Study 1. I have modified the paradigm to include an audio check, and made the demographic questions at the end to include a “prefer not to answer” option.

Qualtrics survey link: https://stanforduniversity.qualtrics.com/jfe/form/SV_7PNJTKIHxk8Gk7k (current estimated time is 8-10 minutes)

For Pilot B, I have conducted the survey on Prolific with 3 participants (who were randomly assigned to one of the three conditions).

Using G*power, apriori analysis with 80% power suggests that I need 122 participants total in order to see the same effect sizes mentioned in the paper (ANCOVA test with 3 groups and 3 covariates).

##libraries

## Load all packages needed for reshaping data
library(tidyverse) # for piping, useful packages
library(ltm) # for Cronbach's alpha
library(effects) # for predicted data for plotting
library(ggplot2) # for graphs
library(sjPlot) # correlation & model output
library(car) # for comparing coefficients
library(emmeans) # for unpacking interactions
library(ggpubr) # for significance on plots
library(psych)
library(effectsize)

pilotb <- read.csv("pilotb_data.csv")
#renaming columns and values
df.pilotb <- pilotb %>% 
  rename(race_native= race_1,
         race_asian = race_2,
         race_black = race_3,
         race_hispanic = race_4,
         race_white = race_5,
         race_other = race_6) %>% 
  mutate(condition = recode_factor(condition,
                                   `0` = "control instructor",
                                   `1` = "growth instructor", 
                                   `2` = "fixed instructor"))

#exclusion critera (for actual data)
#gender
df.pilotb <- df.pilotb %>% 
  mutate (dc_gender = case_when(
    gender == 1 ~ 0,
    gender == 2 ~ 1,
    gender == 3 ~ NA,
    gender == 4 ~ NA,
    TRUE ~ NA
  ))

#urm

df.pilotb <- df.pilotb %>% 
  mutate (dc_urm = case_when(
    race_native == 1 ~ 1,
    race_asian == 1 ~ 0,
    race_black == 1 ~ 1,
    race_hispanic == 1 ~ 1,
    race_white == 1 ~ 0,
    race_other == 1 ~ 1,
    TRUE ~ NA
  ))

#personal mindset
df.pilotb <- df.pilotb %>%
  mutate(personalmindset = rowMeans(dplyr::select(., starts_with("personal_mindset_")), na.rm = TRUE))

#factorization
df.pilotb <- df.pilotb %>%
  mutate(
    condition = factor(condition),
    dc_gender = factor(dc_gender),
    dc_urm = factor(dc_urm))
#mean calculations
df.pilotb <- df.pilotb %>%
  mutate(
    instructormindset = rowMeans(dplyr::select(., starts_with("instructor_mindset_")), na.rm = TRUE),
    belonging = rowMeans(dplyr::select(., starts_with("belonging_")), na.rm = TRUE),
    evaluation = rowMeans(dplyr::select(., starts_with("evaluation_")), na.rm = TRUE),
    engagement = rowMeans(dplyr::select(., starts_with("engagement_")), na.rm = TRUE),
    interest = rowMeans(dplyr::select(., starts_with("interest_")), na.rm = TRUE),
    preformance = rowMeans(dplyr::select(., starts_with("preformance_")), na.rm = TRUE))

The ANCOVA cannot be ran with 3 participants, so I have removed the code from this report. But the code will be the same as pilot A (and will work with more participants).

#data reshaping

df.graph <- df.pilotb %>%
  dplyr::select(ResponseId, condition, belonging, evaluation, engagement, interest, preformance) %>% 
  pivot_longer(cols = -c(condition, ResponseId), 
               names_to = "variable", 
               values_to = "value") %>%
  group_by(condition, variable) %>%
  summarise(
    Mean = mean(value),
    SE = sd(value) / sqrt(n()),
    CI_lower = Mean - qt(0.975, df = n() - 1) * SE,
    CI_upper = Mean + qt(0.975, df = n() - 1) * SE,
    .groups = "drop"
  )
## Warning: There were 30 warnings in `summarise()`.
## The first warning was:
## ℹ In argument: `CI_lower = Mean - qt(0.975, df = n() - 1) * SE`.
## ℹ In group 1: `condition = control instructor` and `variable = "belonging"`.
## Caused by warning in `qt()`:
## ! NaNs produced
## ℹ Run `dplyr::last_dplyr_warnings()` to see the 29 remaining warnings.
df.graph <- df.graph %>%
  mutate(variable = case_when(
    variable == "belonging" ~ "Belonging",
    variable == "evaluation" ~ "Evaluative Concerns",
    variable == "engagement" ~ "Course Engagement",
    variable == "interest" ~ "Course Interest",
    variable == "preformance" ~ "Course Performance",
    TRUE ~ variable
  ))
#plotting

graph.p <- ggplot(df.graph, aes(x = variable, y = Mean, fill = condition)) +
  geom_bar(stat = "identity", position = position_dodge(width = 0.9), width = 0.8) +
  geom_errorbar(aes(ymin = CI_lower, ymax = CI_upper), 
                width = 0.25, position = position_dodge(width = 0.9)) +
  scale_fill_manual(values = c("grey20", "grey95", "grey60")) +
  scale_y_continuous(limits = c(0, 7), breaks = 1:7) +
  labs(y = "", x = "", fill = "") +
  theme_minimal() +
  theme(
    legend.position = "bottom",
    panel.grid.major.x = element_blank(),
    panel.grid.minor.y = element_blank()
  ) +
  coord_cartesian(ylim = c(1, 7)) +
  geom_hline(yintercept = 0, color = "black")

print(graph.p)