0. Loading, setting up
0A. Loading packages
library(tidyverse)
library(lme4)
library(corrr)
library(forcats)
library(jmRtools)
0B. Loading data
esm <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-esm.csv")
pre_survey_data_processed <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-pre-survey.csv")
post_survey_data_partially_processed <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-post-survey.csv")
video <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-video.csv")
pqa <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-pqa.csv")
attendance <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-attendance.csv")
class_data <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-class-video.csv")
demographics <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-demographics.csv")
# demographics$participant_ID <- ifelse(demographics$participant_ID == 7187, NA, demographics$participant_ID)
# write_csv(demographics, "/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-demographics.csv")
parent <- read_csv("/Users/joshuarosenberg/Google Drive/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-parent.csv")
parent <- rename(parent, participant_ID = ParticipantID)
pm <- read_csv("~/Google Drive/1_Research/STEM IE - JJP/STEM-IE/data/final/program_match.csv")
0C. Joining
Note that parent and future_goals_plans had to be processed first for merging later on; can change.
parent <- mutate(parent,
HS_or_above = ifelse(q07a == 1, 1,
ifelse(q07a == 0, 0, NA)))
demographics <- left_join(demographics, parent)
## Joining, by = "participant_ID"
df <- left_join(esm, pre_survey_data_processed, by = "participant_ID") # esm & pre-survey
post_survey_data_partially_processed$post_future_goals_plans <- jmRtools::composite_mean_maker(post_survey_data_partially_processed,
post_future_job_become_STEM,
post_future_job_use_science_math,
post_future_job_work_science_computer)
pre_survey_data_processed$pre_future_goals_plans <- jmRtools::composite_mean_maker(pre_survey_data_processed,
pre_future_job_become_STEM,
pre_future_job_use_science_math,
pre_future_job_work_science_computer)
df <- left_join(df, post_survey_data_partially_processed, by = "participant_ID") # df & post-survey
df <- left_join(df, video, by = c("program_ID", "response_date", "sociedad_class", "signal_number")) # df & video
df <- left_join(df, demographics, by = c("participant_ID", "program_ID")) # df and demographics
0D. Processing CLASS video data
class_data$sociedad_class <- ifelse(class_data$eighth_math == 1, "8th Math",
ifelse(class_data$seventh_math == 1, "7th Math",
ifelse(class_data$sixth_math == 1, "6th Math",
ifelse(class_data$robotics == 1, "Robotics",
ifelse(class_data$dance == 1, "Dance", NA)))))
class_data <- rename(class_data,
response_date = Responsedate,
signal_number = r_signal_number,
program_ID = SiteIDNumeric)
class_data$CLASS_comp <- jmRtools::composite_mean_maker(class_data, ID, P, AI, ILF, QF, CU)
class_data <- select(class_data,
program_ID,
response_date,
signal_number,
sociedad_class,
CLASS_comp,
CLASS_EmotionalSupportEncouragement = EmotionalSupportEncouragement,
CLASS_InstructionalSupport = InstructionalSupport,
CLASS_Autonomy = Autonomy,
CLASS_STEMConceptualDevelopment = STEMConceptualDevelopment,
CLASS_ActivityLeaderEnthusiasm = ActivityLeaderEnthusiasm)
class_data$response_date <- as.character(class_data$response_date)
df <- mutate(df, response_date = as.character(response_date))
df <- left_join(df, class_data, by = c("program_ID", "response_date", "signal_number", "sociedad_class"))
OE. Further processing
df$participant_ID <- as.factor(df$participant_ID)
df$program_ID <- as.factor(df$program_ID)
df$beep_ID <- as.factor(df$beep_ID)
df$beep_ID_new <- as.factor(df$beep_ID_new)
# Recode problem solving, off task, student presentation, and showing video as other
df$youth_activity_rc <- ifelse(df$youth_activity == "Off Task", "Not Focused", df$youth_activity)
df$youth_activity_rc <- ifelse(df$youth_activity_rc == "Student Presentation" | df$youth_activity_rc == "Problem Solving", "Creating Product", df$youth_activity_rc)
df$youth_activity_rc <- ifelse(df$youth_activity_rc == "Showing Video", "Program Staff Led", df$youth_activity_rc)
df$youth_activity_rc <- as.factor(df$youth_activity_rc)
df$youth_activity_rc <- forcats::fct_relevel(df$youth_activity_rc, "Not Focused")
df$relevance <- jmRtools::composite_mean_maker(df, use_outside, future_goals, important)
# need to move up
video$youth_activity_rc <- ifelse(video$youth_activity == "Off Task", "Not Focused", video$youth_activity)
video$youth_activity_rc <- ifelse(video$youth_activity_rc == "Student Presentation" | video$youth_activity_rc == "Problem Solving", "Creating Product", video$youth_activity_rc)
video$youth_activity_rc <- ifelse(video$youth_activity_rc == "Showing Video", "Program Staff Led", video$youth_activity_rc)
ncol(df)
## [1] 161
0F: CLASS correlations and a plot of the composite
df %>%
select(CLASS_comp, CLASS_EmotionalSupportEncouragement, CLASS_InstructionalSupport, CLASS_STEMConceptualDevelopment, CLASS_ActivityLeaderEnthusiasm, CLASS_Autonomy,
challenge, relevance, learning, positive_affect) %>%
correlate() %>%
shave() %>%
fashion() %>%
knitr::kable()
| CLASS_comp |
|
|
|
|
|
|
|
|
|
|
| CLASS_EmotionalSupportEncouragement |
.35 |
|
|
|
|
|
|
|
|
|
| CLASS_InstructionalSupport |
.98 |
.39 |
|
|
|
|
|
|
|
|
| CLASS_STEMConceptualDevelopment |
.96 |
.28 |
.89 |
|
|
|
|
|
|
|
| CLASS_ActivityLeaderEnthusiasm |
.74 |
.63 |
.77 |
.63 |
|
|
|
|
|
|
| CLASS_Autonomy |
.50 |
.29 |
.50 |
.51 |
.52 |
|
|
|
|
|
| challenge |
.06 |
.03 |
.06 |
.04 |
.07 |
.05 |
|
|
|
|
| relevance |
.05 |
-.01 |
.05 |
.04 |
.04 |
.02 |
.39 |
|
|
|
| learning |
.07 |
.00 |
.07 |
.07 |
.05 |
.02 |
.30 |
.65 |
|
|
| positive_affect |
.06 |
.01 |
.06 |
.05 |
.09 |
.02 |
.27 |
.52 |
.48 |
|
ggplot(df, aes(x = CLASS_comp)) +
geom_histogram()
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## Warning: Removed 172 rows containing non-finite values (stat_bin).
## 0G. Processing demographics
demographics %>% count(race)
## # A tibble: 6 x 2
## race n
## <chr> <int>
## 1 Asian 14
## 2 Black 72
## 3 Hispanic 97
## 4 Multiracial 6
## 5 White 13
## 6 <NA> 3
df$urm <- ifelse(df$race %in% c("White", "Asian"), 0, 1)
df$race <- as.factor(df$race)
df$race <- fct_lump(df$race, n = 2)
df$race_other <- fct_relevel(df$race, "Other")
df$gender_female <- as.factor(df$gender) # female is comparison_group
df$gender_female <- ifelse(df$gender_female == "F", 1,
ifelse(df$gender_female == "M", 0, NA))
1. QUESTION 1
How are instructional practices in STEM summer programs related to perceptions of challenge, relevance, learning, and affect for participating youth?
1A. Null models
Note that our normal display presently doesn’t work if there are no predictor variables: it should be fixed in a bit, but for now, we’re just printing the ICCs.
m0i <- lmer(challenge ~ 1 +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjstats::icc(m0i)
## Linear mixed model
## Family: gaussian (identity)
## Formula: challenge ~ 1 + (1 | program_ID) + (1 | participant_ID) + (1 | beep_ID_new)
##
## ICC (beep_ID_new): 0.066428
## ICC (participant_ID): 0.371089
## ICC (program_ID): 0.034016
m0ii <- lmer(relevance ~ 1 +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjstats::icc(m0ii)
## Linear mixed model
## Family: gaussian (identity)
## Formula: relevance ~ 1 + (1 | program_ID) + (1 | participant_ID) + (1 | beep_ID_new)
##
## ICC (beep_ID_new): 0.019134
## ICC (participant_ID): 0.515087
## ICC (program_ID): 0.009082
m0iii <- lmer(learning ~ 1 +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjstats::icc(m0iii)
## Linear mixed model
## Family: gaussian (identity)
## Formula: learning ~ 1 + (1 | program_ID) + (1 | participant_ID) + (1 | beep_ID_new)
##
## ICC (beep_ID_new): 0.023511
## ICC (participant_ID): 0.349134
## ICC (program_ID): 0.000000
df$positive_affect <- jmRtools::composite_mean_maker(df,
happy, excited)
m0iv <- lmer(positive_affect ~ 1 +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjstats::icc(m0iv)
## Linear mixed model
## Family: gaussian (identity)
## Formula: positive_affect ~ 1 + (1 | program_ID) + (1 | participant_ID) + (1 | beep_ID_new)
##
## ICC (beep_ID_new): 0.020685
## ICC (participant_ID): 0.425042
## ICC (program_ID): 0.090985
2C. Models for youth activity (with CLASS composite)
m1i <- lmer(challenge ~ 1 +
youth_activity_rc +
gender_female +
CLASS_comp +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m1i, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
|
|
|
challenge
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
2.21
|
0.14
|
<.001
|
|
youth_activity_rc (Basic Skills Activity)
|
|
0.06
|
0.07
|
.399
|
|
youth_activity_rc (Creating Product)
|
|
0.29
|
0.08
|
<.001
|
|
youth_activity_rc (Field Trip Speaker)
|
|
-0.12
|
0.13
|
.359
|
|
youth_activity_rc (Lab Activity)
|
|
0.15
|
0.13
|
.245
|
|
youth_activity_rc (Program Staff Led)
|
|
-0.13
|
0.08
|
.112
|
|
gender_female
|
|
-0.25
|
0.11
|
.016
|
|
CLASS_comp
|
|
0.04
|
0.03
|
.155
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
228
|
|
Nparticipant_ID
|
|
201
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.046
|
|
ICCparticipant_ID
|
|
0.376
|
|
ICCprogram_ID
|
|
0.032
|
|
Observations
|
|
2693
|
|
R2 / Ω02
|
|
.527 / .520
|
m1ii <- lmer(relevance ~ 1 +
youth_activity_rc +
gender_female +
CLASS_comp +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m1ii, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
|
|
|
relevance
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
2.49
|
0.10
|
<.001
|
|
youth_activity_rc (Basic Skills Activity)
|
|
0.12
|
0.04
|
.004
|
|
youth_activity_rc (Creating Product)
|
|
0.18
|
0.05
|
<.001
|
|
youth_activity_rc (Field Trip Speaker)
|
|
0.27
|
0.07
|
<.001
|
|
youth_activity_rc (Lab Activity)
|
|
0.10
|
0.08
|
.183
|
|
youth_activity_rc (Program Staff Led)
|
|
0.11
|
0.05
|
.015
|
|
gender_female
|
|
-0.22
|
0.10
|
.032
|
|
CLASS_comp
|
|
0.03
|
0.02
|
.081
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
228
|
|
Nparticipant_ID
|
|
201
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.006
|
|
ICCparticipant_ID
|
|
0.518
|
|
ICCprogram_ID
|
|
0.017
|
|
Observations
|
|
2693
|
|
R2 / Ω02
|
|
.589 / .586
|
m1iii <- lmer(learning ~ 1 +
youth_activity_rc +
gender_female +
CLASS_comp +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m1iii, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
|
|
|
learning
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
2.53
|
0.10
|
<.001
|
|
youth_activity_rc (Basic Skills Activity)
|
|
0.17
|
0.05
|
.002
|
|
youth_activity_rc (Creating Product)
|
|
0.04
|
0.06
|
.499
|
|
youth_activity_rc (Field Trip Speaker)
|
|
0.06
|
0.10
|
.502
|
|
youth_activity_rc (Lab Activity)
|
|
0.09
|
0.10
|
.347
|
|
youth_activity_rc (Program Staff Led)
|
|
0.04
|
0.06
|
.490
|
|
gender_female
|
|
-0.06
|
0.10
|
.549
|
|
CLASS_comp
|
|
0.06
|
0.02
|
.005
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
228
|
|
Nparticipant_ID
|
|
201
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.008
|
|
ICCparticipant_ID
|
|
0.355
|
|
ICCprogram_ID
|
|
0.006
|
|
Observations
|
|
2692
|
|
R2 / Ω02
|
|
.427 / .421
|
df$positive_affect <- jmRtools::composite_mean_maker(df,
happy, excited)
m1iv <- lmer(positive_affect ~ 1 +
youth_activity_rc +
gender_female +
CLASS_comp +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
na.action = "na.omit",
data = df)
sjPlot::sjt.lmer(m1iv, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
|
|
|
positive_affect
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
2.75
|
0.15
|
<.001
|
|
youth_activity_rc (Basic Skills Activity)
|
|
0.01
|
0.06
|
.880
|
|
youth_activity_rc (Creating Product)
|
|
-0.02
|
0.06
|
.718
|
|
youth_activity_rc (Field Trip Speaker)
|
|
-0.02
|
0.11
|
.875
|
|
youth_activity_rc (Lab Activity)
|
|
0.05
|
0.11
|
.657
|
|
youth_activity_rc (Program Staff Led)
|
|
-0.07
|
0.06
|
.269
|
|
gender_female
|
|
-0.18
|
0.11
|
.099
|
|
CLASS_comp
|
|
0.01
|
0.02
|
.543
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
228
|
|
Nparticipant_ID
|
|
201
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.025
|
|
ICCparticipant_ID
|
|
0.419
|
|
ICCprogram_ID
|
|
0.084
|
|
Observations
|
|
2693
|
|
R2 / Ω02
|
|
.582 / .578
|
2. QUESTION 2
How is the relationship between STEM practices and perceptions of challenge, relevance, learning, and affect moderated by: (a) youth characteristics at program entry (L2); (b) activity leader perceptions of effective instructional practice (L3 – limited power); (c) the type of activity(L1 – how different from STEM practices?); and (d) classroom versus field-based settings? STEM practices = CLASS items, value statements, etc. stuff from video
2A. Descriptives and correlations for youth activities
video %>%
left_join(pm) %>%
count(program_name, youth_activity_rc) %>%
filter(!is.na(youth_activity_rc)) %>%
spread(youth_activity_rc, n, fill = 0) %>%
gather(youth_activity_rc, frequency, -program_name) %>%
group_by(program_name) %>%
mutate(frequency_prop = frequency / sum(frequency)) %>%
ggplot(aes(x = reorder(youth_activity_rc, frequency_prop), y = frequency_prop)) +
facet_wrap( ~ program_name) +
geom_col() +
theme(axis.text.x = element_text(angle = 45, hjust = 1)) +
ylab("Frequency Proportion") +
xlab(NULL) +
ggtitle("Frequency of Youth Activity (Recoded) Codes by Program")
df$youth_activity_rc_fac <- as.factor(df$youth_activity_rc)
dc <- as.tibble(psych::dummy.code(df$youth_activity_rc_fac))
df_ss <- bind_cols(df, dc)
df_ss %>%
select(challenge, relevance, learning, positive_affect,
`Not Focused`, `Basic Skills Activity`, `Creating Product`,
`Field Trip Speaker`, `Lab Activity`, `Program Staff Led`) %>%
correlate() %>%
shave() %>%
fashion() %>%
knitr::kable()
| challenge |
|
|
|
|
|
|
|
|
|
|
| relevance |
.39 |
|
|
|
|
|
|
|
|
|
| learning |
.30 |
.65 |
|
|
|
|
|
|
|
|
| positive_affect |
.27 |
.52 |
.48 |
|
|
|
|
|
|
|
| Not Focused |
-.02 |
-.06 |
-.05 |
.04 |
|
|
|
|
|
|
| Basic Skills Activity |
-.01 |
-.01 |
.04 |
-.06 |
-.35 |
|
|
|
|
|
| Creating Product |
.12 |
.08 |
.04 |
.05 |
-.33 |
-.25 |
|
|
|
|
| Field Trip Speaker |
-.05 |
.01 |
-.02 |
.01 |
-.15 |
-.11 |
-.11 |
|
|
|
| Lab Activity |
.00 |
-.03 |
.01 |
.02 |
-.14 |
-.11 |
-.10 |
-.05 |
|
|
| Program Staff Led |
-.06 |
.01 |
-.00 |
-.07 |
-.27 |
-.21 |
-.20 |
-.09 |
-.09 |
|
2B. Correlations between situational interest and pre-survey measures
df %>%
select(overall_pre_competence_beliefs, overall_pre_interest, overall_pre_utility_value,
challenge, relevance, learning, positive_affect) %>%
correlate() %>%
shave() %>%
fashion() %>%
knitr::kable()
| overall_pre_competence_beliefs |
|
|
|
|
|
|
|
| overall_pre_interest |
.73 |
|
|
|
|
|
|
| overall_pre_utility_value |
.61 |
.64 |
|
|
|
|
|
| challenge |
-.12 |
.00 |
-.03 |
|
|
|
|
| relevance |
.04 |
.10 |
.11 |
.39 |
|
|
|
| learning |
.09 |
.08 |
.09 |
.30 |
.65 |
|
|
| positive_affect |
.08 |
.20 |
.04 |
.27 |
.52 |
.48 |
|
3. QUESTION 3
How do in-the-moment experiences of youth (i.e., challenge, relevance, learning, and affect) cultivate situational interest and engagement in STEM activities?
3A. Correlations for situational experiences
df$overall_engagement <- jmRtools::composite_mean_maker(df, hard_working, concentrating, enjoy)
df %>%
select(overall_engagement, interest, challenge, relevance, learning, positive_affect) %>%
correlate() %>%
shave() %>%
fashion() %>%
knitr::kable()
| overall_engagement |
|
|
|
|
|
|
| interest |
.69 |
|
|
|
|
|
| challenge |
.30 |
.28 |
|
|
|
|
| relevance |
.65 |
.61 |
.39 |
|
|
|
| learning |
.68 |
.56 |
.30 |
.65 |
|
|
| positive_affect |
.65 |
.56 |
.27 |
.52 |
.48 |
|
3B. Models for situational experiences (with CLASS composite)
m3i0 <- lmer(interest ~ 1 +
overall_pre_interest +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m3i0, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
## Computing p-values via Wald-statistics approximation (treating t as Wald z).
|
|
|
interest
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
2.44
|
0.19
|
<.001
|
|
overall_pre_interest
|
|
0.14
|
0.06
|
.018
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
248
|
|
Nparticipant_ID
|
|
181
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.040
|
|
ICCparticipant_ID
|
|
0.328
|
|
ICCprogram_ID
|
|
0.017
|
|
Observations
|
|
2738
|
|
R2 / Ω02
|
|
.448 / .438
|
m3ii0 <- lmer(overall_engagement ~ 1 +
overall_pre_interest +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m3ii0, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
## Computing p-values via Wald-statistics approximation (treating t as Wald z).
|
|
|
overall_engagement
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
2.50
|
0.16
|
<.001
|
|
overall_pre_interest
|
|
0.11
|
0.05
|
.030
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
248
|
|
Nparticipant_ID
|
|
181
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.034
|
|
ICCparticipant_ID
|
|
0.429
|
|
ICCprogram_ID
|
|
0.000
|
|
Observations
|
|
2738
|
|
R2 / Ω02
|
|
.523 / .518
|
3B. Models for situational experiences (with CLASS composite)
m3i <- lmer(interest ~ 1 +
challenge + relevance + learning + positive_affect +
gender_female +
classroom_versus_field_enrichment +
CLASS_comp +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m3i, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
|
|
|
interest
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
0.62
|
0.10
|
<.001
|
|
challenge
|
|
0.02
|
0.02
|
.150
|
|
relevance
|
|
0.35
|
0.02
|
<.001
|
|
learning
|
|
0.17
|
0.02
|
<.001
|
|
positive_affect
|
|
0.28
|
0.02
|
<.001
|
|
gender_female
|
|
0.04
|
0.05
|
.404
|
|
classroom_versus_field_enrichment
|
|
0.06
|
0.04
|
.130
|
|
CLASS_comp
|
|
-0.00
|
0.02
|
.829
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
231
|
|
Nparticipant_ID
|
|
201
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.034
|
|
ICCparticipant_ID
|
|
0.103
|
|
ICCprogram_ID
|
|
0.014
|
|
Observations
|
|
2727
|
|
R2 / Ω02
|
|
.585 / .583
|
m3ii <- lmer(overall_engagement ~ 1 +
challenge + relevance + learning + positive_affect +
gender_female +
classroom_versus_field_enrichment +
CLASS_comp +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m3ii, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
|
|
|
overall_engagement
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
0.59
|
0.07
|
<.001
|
|
challenge
|
|
0.03
|
0.01
|
.008
|
|
relevance
|
|
0.23
|
0.02
|
<.001
|
|
learning
|
|
0.26
|
0.01
|
<.001
|
|
positive_affect
|
|
0.29
|
0.01
|
<.001
|
|
gender_female
|
|
0.07
|
0.04
|
.105
|
|
classroom_versus_field_enrichment
|
|
0.09
|
0.03
|
.001
|
|
CLASS_comp
|
|
0.01
|
0.01
|
.276
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
231
|
|
Nparticipant_ID
|
|
201
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.025
|
|
ICCparticipant_ID
|
|
0.213
|
|
ICCprogram_ID
|
|
0.000
|
|
Observations
|
|
2727
|
|
R2 / Ω02
|
|
.738 / .737
|
m3iii <- lmer(interest ~ 1 +
challenge + relevance + learning + positive_affect +
overall_pre_interest +
classroom_versus_field_enrichment +
gender_female +
CLASS_comp +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m3iii, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
|
|
|
interest
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
0.55
|
0.13
|
<.001
|
|
challenge
|
|
0.02
|
0.02
|
.207
|
|
relevance
|
|
0.35
|
0.02
|
<.001
|
|
learning
|
|
0.17
|
0.02
|
<.001
|
|
positive_affect
|
|
0.28
|
0.02
|
<.001
|
|
overall_pre_interest
|
|
0.02
|
0.03
|
.494
|
|
classroom_versus_field_enrichment
|
|
0.09
|
0.04
|
.046
|
|
gender_female
|
|
0.04
|
0.05
|
.441
|
|
CLASS_comp
|
|
-0.00
|
0.02
|
.852
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
231
|
|
Nparticipant_ID
|
|
180
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.037
|
|
ICCparticipant_ID
|
|
0.104
|
|
ICCprogram_ID
|
|
0.013
|
|
Observations
|
|
2513
|
|
R2 / Ω02
|
|
.582 / .580
|
m3iv <- lmer(overall_engagement ~ 1 +
challenge + relevance + learning + positive_affect +
overall_pre_interest +
classroom_versus_field_enrichment +
gender_female +
CLASS_comp +
(1|program_ID) + (1|participant_ID) + (1|beep_ID_new),
data = df)
sjPlot::sjt.lmer(m3iv, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
|
|
|
overall_engagement
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
0.50
|
0.10
|
<.001
|
|
challenge
|
|
0.03
|
0.01
|
.013
|
|
relevance
|
|
0.24
|
0.02
|
<.001
|
|
learning
|
|
0.25
|
0.01
|
<.001
|
|
positive_affect
|
|
0.29
|
0.01
|
<.001
|
|
overall_pre_interest
|
|
0.02
|
0.02
|
.374
|
|
classroom_versus_field_enrichment
|
|
0.09
|
0.03
|
<.001
|
|
gender_female
|
|
0.07
|
0.04
|
.105
|
|
CLASS_comp
|
|
0.01
|
0.01
|
.343
|
|
Random Parts
|
|
Nbeep_ID_new
|
|
231
|
|
Nparticipant_ID
|
|
180
|
|
Nprogram_ID
|
|
9
|
|
ICCbeep_ID_new
|
|
0.029
|
|
ICCparticipant_ID
|
|
0.215
|
|
ICCprogram_ID
|
|
0.001
|
|
Observations
|
|
2513
|
|
R2 / Ω02
|
|
.741 / .740
|
4. QUESTION 4
Are situational (momentary) interest and engagement in STEM activities across several weeks associated with changes in: (a) individual (sustained) interest in STEM; (b) a STEM self-concept; and (c) future goals and aspirations related to STEM?
A step we don’t yet take is to use output from mixed-effects models for each of the situational experiences variables to predict these outcomes; not yet added.
4A. Descriptives for situational experiences and future goals and plans
participant_df <- df %>%
select(participant_ID, challenge, relevance, learning, positive_affect, good_at, post_future_goals_plans) %>%
group_by(participant_ID) %>%
mutate_at(vars(challenge, relevance, learning, positive_affect, good_at), funs(mean, sd)) %>%
select(participant_ID, contains("mean"), contains("sd"), post_future_goals_plans) %>%
distinct()
df_ss <- left_join(df, participant_df)
df_ss <- select(df_ss,
participant_ID, program_ID,
challenge_mean, relevance_mean, learning_mean, positive_affect_mean, good_at_mean,
challenge_sd, relevance_sd, learning_sd, positive_affect_sd, good_at_sd,
overall_post_interest, overall_pre_interest, post_future_goals_plans,
future_goals)
df_ss <- distinct(df_ss)
df_ss$program_ID <- as.integer(df_ss$program_ID)
df_ss <- left_join(df_ss, pm)
4B. Using means and standard deviations to predict post-interest and future goals and aspirations
m4i <- lmer(overall_post_interest ~ 1 +
challenge_mean + challenge_sd +
learning_mean + learning_sd +
relevance_mean + relevance_sd +
positive_affect_mean + positive_affect_sd +
overall_pre_interest +
(1|program_ID),
data = df_ss)
sjPlot::sjt.lmer(m4i, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
## Computing p-values via Wald-statistics approximation (treating t as Wald z).
|
|
|
overall_post_interest
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
0.69
|
0.27
|
.011
|
|
challenge_mean
|
|
-0.07
|
0.06
|
.275
|
|
challenge_sd
|
|
-0.50
|
0.13
|
<.001
|
|
learning_mean
|
|
0.51
|
0.11
|
<.001
|
|
learning_sd
|
|
0.05
|
0.13
|
.701
|
|
relevance_mean
|
|
-0.25
|
0.12
|
.037
|
|
relevance_sd
|
|
0.31
|
0.18
|
.087
|
|
positive_affect_mean
|
|
0.16
|
0.07
|
.022
|
|
positive_affect_sd
|
|
0.07
|
0.13
|
.570
|
|
overall_pre_interest
|
|
0.47
|
0.05
|
<.001
|
|
Random Parts
|
|
Nprogram_ID
|
|
9
|
|
ICCprogram_ID
|
|
0.110
|
|
Observations
|
|
425
|
|
R2 / Ω02
|
|
.492 / .492
|
m4ii <- lmer(post_future_goals_plans ~ 1 +
challenge_mean + challenge_sd +
learning_mean + learning_sd +
relevance_mean + relevance_sd +
positive_affect_mean + positive_affect_sd +
overall_pre_interest +
(1|program_ID),
data = df_ss)
sjPlot::sjt.lmer(m4ii, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
## Computing p-values via Wald-statistics approximation (treating t as Wald z).
|
|
|
post_future_goals_plans
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
1.51
|
0.33
|
<.001
|
|
challenge_mean
|
|
-0.15
|
0.08
|
.058
|
|
challenge_sd
|
|
-0.04
|
0.17
|
.797
|
|
learning_mean
|
|
-0.09
|
0.14
|
.516
|
|
learning_sd
|
|
-0.28
|
0.16
|
.091
|
|
relevance_mean
|
|
0.54
|
0.15
|
<.001
|
|
relevance_sd
|
|
0.22
|
0.23
|
.338
|
|
positive_affect_mean
|
|
-0.04
|
0.08
|
.592
|
|
positive_affect_sd
|
|
-0.04
|
0.16
|
.816
|
|
overall_pre_interest
|
|
0.24
|
0.06
|
<.001
|
|
Random Parts
|
|
Nprogram_ID
|
|
9
|
|
ICCprogram_ID
|
|
0.015
|
|
Observations
|
|
412
|
|
R2 / Ω02
|
|
.169 / .169
|
4C. Predicting individual-level effects
library(magrittr)
##
## Attaching package: 'magrittr'
## The following object is masked from 'package:purrr':
##
## set_names
## The following object is masked from 'package:tidyr':
##
## extract
# write a function at some point
# extract_ranef <- function(df, var, random_effect) {
#
# }
chall_df <- lmer(challenge ~ (1|program_ID) + (1|participant_ID) + (1|beep_ID_new), data = df) %>%
ranef() %>%
extract2("participant_ID") %>%
rownames_to_column(var = "participant_ID") %>%
rename(pred_challenge = `(Intercept)`) %>%
tbl_df()
learning_df <- lmer(learning ~ (1|program_ID) + (1|participant_ID) + (1|beep_ID_new), data = df) %>%
ranef() %>%
extract2("participant_ID") %>%
rownames_to_column(var = "participant_ID") %>%
rename(pred_learning = `(Intercept)`) %>%
tbl_df()
positive_affect_df <- lmer(positive_affect ~ (1|program_ID) + (1|participant_ID) + (1|beep_ID_new), data = df) %>%
ranef() %>%
extract2("participant_ID") %>%
rownames_to_column(var = "participant_ID") %>%
rename(pred_positive_affect = `(Intercept)`) %>%
tbl_df()
relevance_df <- lmer(relevance ~ (1|program_ID) + (1|participant_ID) + (1|beep_ID_new), data = df) %>%
ranef() %>%
extract2("participant_ID") %>%
rownames_to_column(var = "participant_ID") %>%
rename(pred_relevance = `(Intercept)`) %>%
tbl_df()
# there's actually no variability at participant level:
mtest <- lmer(relevance ~ (1|program_ID) + (1|participant_ID) + (1|beep_ID_new), data = df)
mtest
## Linear mixed model fit by REML ['lmerMod']
## Formula:
## relevance ~ (1 | program_ID) + (1 | participant_ID) + (1 | beep_ID_new)
## Data: df
## REML criterion at convergence: 6554
## Random effects:
## Groups Name Std.Dev.
## beep_ID_new (Intercept) 0.13326
## participant_ID (Intercept) 0.69141
## program_ID (Intercept) 0.09181
## Residual 0.65104
## Number of obs: 2978, groups:
## beep_ID_new, 248; participant_ID, 203; program_ID, 9
## Fixed Effects:
## (Intercept)
## 2.576
pred_var_df <- chall_df %>%
left_join(learning_df, by = "participant_ID") %>%
left_join(positive_affect_df, by = "participant_ID") %>%
left_join(relevance_df, by = "participant_ID")
pre_survey_data_processed$participant_ID <- as.character(pre_survey_data_processed$participant_ID)
post_survey_data_partially_processed$participant_ID <- as.character(post_survey_data_partially_processed$participant_ID)
demographics$participant_ID <- as.character(demographics$participant_ID)
mod_df <- left_join(pred_var_df, pre_survey_data_processed, by = "participant_ID") %>%
left_join(post_survey_data_partially_processed, by = "participant_ID") %>%
left_join(demographics, by = "participant_ID")
4D. Using individual-level effects to predict post-interest and future goals and aspirations
m4bi <- lmer(overall_post_interest ~
pred_challenge + pred_positive_affect + pred_learning +
overall_pre_interest + (1|program_ID),
data = mod_df)
sjPlot::sjt.lmer(m4bi, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
## Computing p-values via Wald-statistics approximation (treating t as Wald z).
|
|
|
overall_post_interest
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
1.48
|
0.25
|
<.001
|
|
pred_challenge
|
|
-0.20
|
0.10
|
.054
|
|
pred_positive_affect
|
|
0.10
|
0.13
|
.420
|
|
pred_learning
|
|
0.35
|
0.15
|
.019
|
|
overall_pre_interest
|
|
0.52
|
0.08
|
<.001
|
|
Random Parts
|
|
Nprogram_ID
|
|
9
|
|
ICCprogram_ID
|
|
0.063
|
|
Observations
|
|
142
|
|
R2 / Ω02
|
|
.448 / .447
|
m4bii <- lmer(post_future_goals_plans ~
pred_challenge + pred_positive_affect + pred_learning +
pre_future_goals_plans + (1|program_ID),
data = mod_df)
sjPlot::sjt.lmer(m4bii, p.kr = F, show.re.var = F, show.ci = F, show.se = T)
## Computing p-values via Wald-statistics approximation (treating t as Wald z).
|
|
|
post_future_goals_plans
|
|
|
|
B
|
std. Error
|
p
|
|
Fixed Parts
|
|
(Intercept)
|
|
1.59
|
0.25
|
<.001
|
|
pred_challenge
|
|
-0.02
|
0.12
|
.845
|
|
pred_positive_affect
|
|
0.04
|
0.16
|
.779
|
|
pred_learning
|
|
0.28
|
0.18
|
.131
|
|
pre_future_goals_plans
|
|
0.44
|
0.09
|
<.001
|
|
Random Parts
|
|
Nprogram_ID
|
|
9
|
|
ICCprogram_ID
|
|
0.000
|
|
Observations
|
|
134
|
|
R2 / Ω02
|
|
.252 / .252
|