RQ_1 - sans momentary random effects

This is analysis associated with the first RQ for the STEM-IE project.

RQ #1. How are instructional practices in STEM summer programs related to perceptions of challenge, relevance, learning, and affect for participating youth?

• This is more about what are the activities that are challenging and relevant in order to understand what program-level characteristics predict these experience.

• We think it could be useful for evaluation and feedback purposes - this is more a background question that we want to be able to use to inform programs in terms of what activities may be challenging and relevant
  • this focuses should be on things that programs can control.

This is what we know:

• Basic skills predicts:
  • challenge (not very robustly)
  • learning (robustly)
• Creating products predicts:
  • challenge (fairly robustly)
  • relevance
  • learning (not very robustly)
• Support for agency predicts:
  • positive affect
• Support for STEM skill-building predicts:
  • relevance
  • learning
• Minor findings:
  • HO thinking seems to negatively predict challenge

This is what we want to figure out:

• How reliably was the PQA used videos?
• Can we use dummy codes instead of counts for PQA measures?

How this document is organized

• The outcomes are numbered 1 - 4 (challenge, relevance, learning, and positive affect)
• Within the section for each outcome, four models are specified (just youth activity, PQA counts, PQA dummy codes, and select activities with select PQA counts)

library(tidyverse)
library(lmerTest)
library(lme4)
library(corrr)
library(jmRtools)

esm <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-esm.csv")
pre_survey_data_processed <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-pre-survey.csv")
post_survey_data_partially_processed <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-post-survey.csv")
video <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-video.csv")
pqa <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-pqa.csv")
attendance <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-attendance.csv")
class_data <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-class-video.csv")
demographics <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-demographics.csv")
pm <- read_csv("/Volumes/SCHMIDTLAB/PSE/Data/STEM-IE/STEM-IE-program-match.csv")

attendance <- rename(attendance, participant_ID = ParticipantID)
attendance <- mutate(attendance, prop_attend = DaysAttended / DaysScheduled, 
                     participant_ID = as.integer(participant_ID))
attendance <- select(attendance, participant_ID, prop_attend)

demographics <- filter(demographics, participant_ID!= 7187)
demographics <- left_join(demographics, attendance)

esm$overall_engagement <- jmRtools::composite_mean_maker(esm, hard_working, concentrating, enjoy, interest)

df <- left_join(esm, pre_survey_data_processed, by = "participant_ID") # df & post-survey
df <- left_join(df, video, by = c("program_ID", "response_date", "sociedad_class", "signal_number")) # df & video
df <- left_join(df, demographics, by = c("participant_ID", "program_ID")) # df and demographics

pqa <- mutate(pqa, 
              active = active_part_1 + active_part_2,
              ho_thinking = ho_thinking_1 + ho_thinking_2 + ho_thinking_3,
              belonging = belonging_1 + belonging_2,
              agency = agency_1 + agency_2 + agency_3 + agency_4,
              youth_development_overall = active_part_1 + active_part_2 + ho_thinking_1 + ho_thinking_2 + ho_thinking_3 + belonging_1 + belonging_2 + agency_1 + agency_2 + agency_3 + agency_4,
              making_observations = stem_sb_8,
              data_modeling = stem_sb_2 + stem_sb_3 + stem_sb_9,
              interpreting_communicating = stem_sb_6,
              generating_data = stem_sb_4,
              asking_questions = stem_sb_1,
              stem_sb = stem_sb_1 + stem_sb_2 + stem_sb_3 + stem_sb_4 + stem_sb_5 + stem_sb_6 + stem_sb_7 + stem_sb_8 + stem_sb_9)

# pqa <- rename(pqa, sixth_math_sociedad = sixth_math)
# pqa <- rename(pqa, seventh_math_sociedad = seventh_math)
# pqa <- rename(pqa, eighth_math_sociedad = eighth_math)
# pqa <- rename(pqa, dance_sociedad = dance)
# pqa <- rename(pqa, robotics_sociedad = robotics)

pqa$sociedad_class <- ifelse(pqa$eighth_math == 1, "8th Math",
                             ifelse(pqa$seventh_math == 1, "7th Math",
                                    ifelse(pqa$sixth_math == 1, "6th Math",
                                           ifelse(pqa$robotics == 1, "Robotics",
                                                  ifelse(pqa$dance == 1, "Dance", NA)))))

pqa <- rename(pqa, 
              program_ID = SiteIDNumeric,
              response_date = resp_date,
              signal_number = signal)

pqa$program_ID <- as.character(pqa$program_ID)

df <- left_join(df, pqa, by = c("response_date", "program_ID", "signal_number", "sociedad_class"))

## Warning: Column `program_ID` joining factor and character vector, coercing
## into character vector

df <- df %>% 
    mutate(youth_activity_three = case_when(
        youth_activity_rc == "Creating Product" ~ "Creating Product",
        youth_activity_rc == "Basic Skills Activity" ~ "Basic Skills Activity",
        TRUE ~ "Other"
    ))

df$youth_activity_three <- fct_relevel(df$youth_activity_three, 
                                       "Other")

Correlations

df %>% 
    select(challenge, relevance, learning, positive_affect, ho_thinking, stem_sb, agency, active, belonging) %>% 
    correlate() %>% 
    shave() %>% 
    fashion() %>% 
    knitr::kable()

rowname	challenge	relevance	learning	positive_affect	ho_thinking	stem_sb	agency	active	belonging
challenge
relevance	.39
learning	.30	.65
positive_affect	.27	.52	.48
ho_thinking	-.04	.02	.02	.04
stem_sb	.00	.02	.04	-.04	.54
agency	.06	.02	.04	.04	.38	.39
active	.01	.02	.03	.00	.44	.56	.16
belonging	.03	-.02	.03	-.01	.36	.42	.54	.22

df <- df %>% 
    mutate(ho_thinking_dummy = ifelse(ho_thinking > 0, 1, 0),
           agency_dummy = ifelse(agency > 0, 1, 0),
           active_dummy = ifelse(active > 0, 1, 0),
           belonging_dummy = ifelse(belonging > 0, 1, 0),
           stem_sb_dummy = ifelse(stem_sb > 0, 1, 0))

1. Challenge

1A: Just youth activity

m <- lmer(challenge ~ 
              youth_activity_rc + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: 
## challenge ~ youth_activity_rc + (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 7536.7
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.9280 -0.6391 -0.0127  0.5861  3.4603 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.47746  0.6910  
##  program_ID     (Intercept) 0.04963  0.2228  
##  Residual                   0.71677  0.8466  
## Number of obs: 2818, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                                          Estimate Std. Error         df
## (Intercept)                               2.18261    0.09445    9.30000
## youth_activity_rcBasic Skills Activity    0.11745    0.04694 2661.70000
## youth_activity_rcCreating Product         0.37841    0.04794 2667.70000
## youth_activity_rcField Trip Speaker      -0.05545    0.08628 2630.80000
## youth_activity_rcLab Activity             0.20071    0.08506 2639.40000
## youth_activity_rcProgram Staff Led       -0.08018    0.05331 2642.80000
##                                        t value Pr(>|t|)    
## (Intercept)                             23.109 1.51e-09 ***
## youth_activity_rcBasic Skills Activity   2.502   0.0124 *  
## youth_activity_rcCreating Product        7.894 4.22e-15 ***
## youth_activity_rcField Trip Speaker     -0.643   0.5205    
## youth_activity_rcLab Activity            2.360   0.0184 *  
## youth_activity_rcProgram Staff Led      -1.504   0.1327    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) y__BSA yt__CP y__FTS yt__LA
## yth_ctv_BSA -0.199                            
## yth_ctvt_CP -0.188  0.372                     
## yth_ctv_FTS -0.113  0.247  0.219              
## yth_ctvt_LA -0.105  0.205  0.200  0.147       
## yth_ctv_PSL -0.172  0.413  0.305  0.192  0.188

1B: Just STEM-SB as counts

m <- lmer(challenge ~ 
              active +
              ho_thinking +
              belonging +
              agency +
              stem_sb + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: 
## challenge ~ active + ho_thinking + belonging + agency + stem_sb +  
##     (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 7523.9
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.9695 -0.6407 -0.0455  0.5642  3.3854 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.46915  0.6849  
##  program_ID     (Intercept) 0.05772  0.2402  
##  Residual                   0.72415  0.8510  
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##               Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)    2.14895    0.10921   13.40000  19.677 2.70e-11 ***
## active         0.03689    0.03890 2628.20000   0.948   0.3430    
## ho_thinking   -0.08812    0.02020 2645.10000  -4.362 1.34e-05 ***
## belonging      0.05473    0.02771 2625.80000   1.975   0.0484 *  
## agency         0.06509    0.01507 2637.70000   4.318 1.63e-05 ***
## stem_sb        0.01803    0.00951 2637.00000   1.896   0.0581 .  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) active h_thnk blngng agency
## active      -0.353                            
## ho_thinking -0.089 -0.203                     
## belonging   -0.068 -0.011 -0.104              
## agency      -0.117  0.129 -0.178 -0.427       
## stem_sb      0.135 -0.446 -0.278 -0.164 -0.164

1C: Just STEM-SB as dummy codes (if count >= 1, then dummy code = 1)

m <- lmer(challenge ~ 
              active_dummy +
              ho_thinking_dummy +
              belonging_dummy +
              agency_dummy +
              stem_sb_dummy + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: challenge ~ active_dummy + ho_thinking_dummy + belonging_dummy +  
##     agency_dummy + stem_sb_dummy + (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 7536.7
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.9701 -0.6340 -0.0514  0.5998  3.3489 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.4656   0.6823  
##  program_ID     (Intercept) 0.0561   0.2369  
##  Residual                   0.7313   0.8552  
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                     Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)          1.99178    0.16293   68.70000  12.225  < 2e-16 ***
## active_dummy         0.10255    0.13582 2611.90000   0.755  0.45028    
## ho_thinking_dummy   -0.13688    0.05551 2636.00000  -2.466  0.01374 *  
## belonging_dummy      0.12393    0.04475 2628.70000   2.769  0.00566 ** 
## agency_dummy         0.11704    0.04976 2632.80000   2.352  0.01874 *  
## stem_sb_dummy        0.13825    0.04976 2649.20000   2.778  0.00551 ** 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) actv_d h_thn_ blngn_ agncy_
## active_dmmy -0.738                            
## h_thnkng_dm -0.105 -0.019                     
## blngng_dmmy -0.021 -0.112 -0.073              
## agency_dmmy -0.078 -0.021 -0.231 -0.395       
## stm_sb_dmmy  0.001 -0.198 -0.361  0.086 -0.025

1D: Only select activities and STEM-SB dummy codes

m <- lmer(challenge ~ 
              youth_activity_three +
              agency_dummy +
              stem_sb_dummy + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: 
## challenge ~ youth_activity_three + agency_dummy + stem_sb_dummy +  
##     (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 7492
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.8344 -0.6605 -0.0093  0.5873  3.4063 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.46893  0.6848  
##  program_ID     (Intercept) 0.05392  0.2322  
##  Residual                   0.71885  0.8479  
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                                            Estimate Std. Error        df
## (Intercept)                               2.089e+00  1.055e-01 1.310e+01
## youth_activity_threeBasic Skills Activity 1.290e-01  4.258e-02 2.638e+03
## youth_activity_threeCreating Product      3.500e-01  4.664e-02 2.644e+03
## agency_dummy                              7.530e-02  4.437e-02 2.638e+03
## stem_sb_dummy                             4.182e-02  4.569e-02 2.647e+03
##                                           t value Pr(>|t|)    
## (Intercept)                                19.805 3.65e-11 ***
## youth_activity_threeBasic Skills Activity   3.030  0.00247 ** 
## youth_activity_threeCreating Product        7.506 8.30e-14 ***
## agency_dummy                                1.697  0.08975 .  
## stem_sb_dummy                               0.915  0.36014    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) y__BSA yt__CP agncy_
## yth_ctv_BSA -0.041                     
## yth_ctvt_CP  0.012  0.262              
## agency_dmmy -0.290 -0.035 -0.215       
## stm_sb_dmmy -0.300 -0.159 -0.140 -0.090

2. Relevance

2A: Just youth activity

m <- lmer(relevance ~ 
              youth_activity_rc + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: 
## relevance ~ youth_activity_rc + (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 6157.6
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.9336 -0.5211  0.0259  0.5822  4.1256 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.48174  0.6941  
##  program_ID     (Intercept) 0.01492  0.1221  
##  Residual                   0.42481  0.6518  
## Number of obs: 2818, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                                         Estimate Std. Error        df
## (Intercept)                            2.461e+00  6.821e-02 9.100e+00
## youth_activity_rcBasic Skills Activity 1.543e-01  3.619e-02 2.648e+03
## youth_activity_rcCreating Product      2.304e-01  3.697e-02 2.654e+03
## youth_activity_rcField Trip Speaker    2.930e-01  6.645e-02 2.611e+03
## youth_activity_rcLab Activity          1.121e-01  6.552e-02 2.632e+03
## youth_activity_rcProgram Staff Led     1.593e-01  4.107e-02 2.637e+03
##                                        t value Pr(>|t|)    
## (Intercept)                             36.074 3.89e-11 ***
## youth_activity_rcBasic Skills Activity   4.263 2.08e-05 ***
## youth_activity_rcCreating Product        6.231 5.38e-10 ***
## youth_activity_rcField Trip Speaker      4.409 1.08e-05 ***
## youth_activity_rcLab Activity            1.711 0.087174 .  
## youth_activity_rcProgram Staff Led       3.878 0.000108 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) y__BSA yt__CP y__FTS yt__LA
## yth_ctv_BSA -0.213                            
## yth_ctvt_CP -0.200  0.371                     
## yth_ctv_FTS -0.123  0.247  0.218              
## yth_ctvt_LA -0.114  0.205  0.200  0.147       
## yth_ctv_PSL -0.184  0.413  0.305  0.193  0.188

2B: Just STEM-SB as counts

m <- lmer(relevance ~ 
              active +
              ho_thinking +
              belonging +
              agency +
              stem_sb + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: 
## relevance ~ active + ho_thinking + belonging + agency + stem_sb +  
##     (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 6166.8
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.9173 -0.5208  0.0480  0.5775  3.7515 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.47435  0.6887  
##  program_ID     (Intercept) 0.01325  0.1151  
##  Residual                   0.43140  0.6568  
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##               Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)  2.498e+00  7.558e-02  1.500e+01  33.047 1.78e-15 ***
## active       2.253e-02  3.005e-02  2.621e+03   0.750   0.4536    
## ho_thinking  1.671e-02  1.561e-02  2.633e+03   1.070   0.2847    
## belonging   -4.222e-02  2.141e-02  2.618e+03  -1.972   0.0487 *  
## agency       5.451e-03  1.165e-02  2.628e+03   0.468   0.6398    
## stem_sb      1.623e-02  7.348e-03  2.626e+03   2.209   0.0273 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) active h_thnk blngng agency
## active      -0.394                            
## ho_thinking -0.105 -0.203                     
## belonging   -0.075 -0.011 -0.103              
## agency      -0.128  0.129 -0.178 -0.427       
## stem_sb      0.152 -0.446 -0.278 -0.164 -0.164

2C: Just STEM-SB as dummy codes (if count >= 1, then dummy code = 1)

m <- lmer(relevance ~ 
              active_dummy +
              ho_thinking_dummy +
              belonging_dummy +
              agency_dummy +
              stem_sb_dummy + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: relevance ~ active_dummy + ho_thinking_dummy + belonging_dummy +  
##     agency_dummy + stem_sb_dummy + (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 6152.4
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.9406 -0.5201  0.0589  0.5742  3.7108 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.477246 0.69083 
##  program_ID     (Intercept) 0.007793 0.08828 
##  Residual                   0.430964 0.65648 
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                     Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)          2.32765    0.11756  119.10000  19.800  < 2e-16 ***
## active_dummy         0.20301    0.10430 2609.70000   1.946 0.051703 .  
## ho_thinking_dummy   -0.07348    0.04264 2626.00000  -1.723 0.084995 .  
## belonging_dummy      0.01109    0.03439 2621.40000   0.322 0.747104    
## agency_dummy        -0.01050    0.03825 2624.70000  -0.275 0.783602    
## stem_sb_dummy        0.13831    0.03826 2632.40000   3.615 0.000305 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) actv_d h_thn_ blngn_ agncy_
## active_dmmy -0.785                            
## h_thnkng_dm -0.116 -0.019                     
## blngng_dmmy -0.021 -0.112 -0.072              
## agency_dmmy -0.082 -0.021 -0.231 -0.395       
## stm_sb_dmmy  0.003 -0.198 -0.360  0.084 -0.025

2D: Only select activities and STEM-SB dummy codes

m <- lmer(relevance ~ 
              youth_activity_three +
              agency_dummy +
              stem_sb_dummy + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: 
## relevance ~ youth_activity_three + agency_dummy + stem_sb_dummy +  
##     (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 6138.9
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -4.0038 -0.5183  0.0496  0.5838  3.8396 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.47631  0.69015 
##  program_ID     (Intercept) 0.00697  0.08349 
##  Residual                   0.42903  0.65501 
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                                             Estimate Std. Error         df
## (Intercept)                                  2.48924    0.06904   15.20000
## youth_activity_threeBasic Skills Activity    0.05678    0.03292 2627.20000
## youth_activity_threeCreating Product         0.15189    0.03606 2629.50000
## agency_dummy                                -0.04773    0.03432 2628.70000
## stem_sb_dummy                                0.10530    0.03535 2632.70000
##                                           t value Pr(>|t|)    
## (Intercept)                                36.057 4.44e-16 ***
## youth_activity_threeBasic Skills Activity   1.724  0.08475 .  
## youth_activity_threeCreating Product        4.212 2.62e-05 ***
## agency_dummy                               -1.391  0.16440    
## stem_sb_dummy                               2.979  0.00292 ** 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) y__BSA yt__CP agncy_
## yth_ctv_BSA -0.049                     
## yth_ctvt_CP  0.015  0.261              
## agency_dmmy -0.342 -0.035 -0.214       
## stm_sb_dmmy -0.354 -0.158 -0.139 -0.092

3. Learning

3A: Just youth activity

m <- lmer(learning ~ 
              youth_activity_rc + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: 
## learning ~ youth_activity_rc + (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 7505
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.2278 -0.5584  0.1291  0.5673  2.7111 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.400255 0.63266 
##  program_ID     (Intercept) 0.003176 0.05635 
##  Residual                   0.719333 0.84813 
## Number of obs: 2817, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                                         Estimate Std. Error        df
## (Intercept)                            2.682e+00  5.706e-02 1.090e+01
## youth_activity_rcBasic Skills Activity 2.225e-01  4.682e-02 2.626e+03
## youth_activity_rcCreating Product      1.349e-01  4.787e-02 2.668e+03
## youth_activity_rcField Trip Speaker    1.010e-01  8.578e-02 2.331e+03
## youth_activity_rcLab Activity          1.531e-01  8.482e-02 2.568e+03
## youth_activity_rcProgram Staff Led     7.762e-02  5.318e-02 2.609e+03
##                                        t value Pr(>|t|)    
## (Intercept)                             46.998 6.37e-14 ***
## youth_activity_rcBasic Skills Activity   4.753 2.12e-06 ***
## youth_activity_rcCreating Product        2.818  0.00487 ** 
## youth_activity_rcField Trip Speaker      1.178  0.23899    
## youth_activity_rcLab Activity            1.805  0.07125 .  
## youth_activity_rcProgram Staff Led       1.460  0.14448    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) y__BSA yt__CP y__FTS yt__LA
## yth_ctv_BSA -0.330                            
## yth_ctvt_CP -0.310  0.372                     
## yth_ctv_FTS -0.192  0.245  0.218              
## yth_ctvt_LA -0.180  0.206  0.201  0.145       
## yth_ctv_PSL -0.286  0.410  0.308  0.192  0.188

3B: Just STEM-SB as counts

m <- lmer(learning ~ 
              active +
              ho_thinking +
              belonging +
              agency +
              stem_sb + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: learning ~ active + ho_thinking + belonging + agency + stem_sb +  
##     (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 7473.7
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.3078 -0.5717  0.1097  0.5989  2.7246 
## 
## Random effects:
##  Groups         Name        Variance  Std.Dev.
##  participant_ID (Intercept) 0.3971368 0.63019 
##  program_ID     (Intercept) 0.0001029 0.01014 
##  Residual                   0.7216654 0.84951 
## Number of obs: 2798, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##               Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)  2.626e+00  7.073e-02  3.610e+01  37.133   <2e-16 ***
## active       8.049e-02  3.876e-02  2.640e+03   2.077   0.0379 *  
## ho_thinking -2.430e-02  2.005e-02  2.617e+03  -1.212   0.2256    
## belonging    3.292e-02  2.762e-02  2.640e+03   1.192   0.2334    
## agency       1.130e-02  1.500e-02  2.647e+03   0.753   0.4515    
## stem_sb      7.061e-03  9.456e-03  2.623e+03   0.747   0.4553    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) active h_thnk blngng agency
## active      -0.543                            
## ho_thinking -0.147 -0.204                     
## belonging   -0.105 -0.009 -0.103              
## agency      -0.172  0.127 -0.177 -0.427       
## stem_sb      0.211 -0.445 -0.279 -0.166 -0.165

3C: Just STEM-SB as dummy codes (if count >= 1, then dummy code = 1)

m <- lmer(learning ~ 
              active_dummy +
              ho_thinking_dummy +
              belonging_dummy +
              agency_dummy +
              stem_sb_dummy + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: learning ~ active_dummy + ho_thinking_dummy + belonging_dummy +  
##     agency_dummy + stem_sb_dummy + (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 7454.1
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.2559 -0.5710  0.1084  0.6080  2.6291 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.3977   0.6306  
##  program_ID     (Intercept) 0.0000   0.0000  
##  Residual                   0.7194   0.8482  
## Number of obs: 2798, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                     Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)          2.33939    0.13963 2676.00000  16.755  < 2e-16 ***
## active_dummy         0.32925    0.13447 2634.90000   2.448  0.01441 *  
## ho_thinking_dummy   -0.08456    0.05469 2714.10000  -1.546  0.12218    
## belonging_dummy      0.08307    0.04428 2655.80000   1.876  0.06077 .  
## agency_dummy         0.01726    0.04926 2655.80000   0.350  0.72608    
## stem_sb_dummy        0.13065    0.04915 2688.10000   2.658  0.00791 ** 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) actv_d h_thn_ blngn_ agncy_
## active_dmmy -0.853                            
## h_thnkng_dm -0.126 -0.021                     
## blngng_dmmy -0.023 -0.112 -0.070              
## agency_dmmy -0.088 -0.021 -0.230 -0.397       
## stm_sb_dmmy  0.003 -0.196 -0.358  0.083 -0.025

3D: Only select activities and STEM-SB dummy codes

m <- lmer(learning ~ 
              youth_activity_three +
              agency_dummy +
              stem_sb_dummy + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: learning ~ youth_activity_three + agency_dummy + stem_sb_dummy +  
##     (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 7451.4
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.3015 -0.5708  0.1299  0.5844  2.7062 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.39773  0.63066 
##  program_ID     (Intercept) 0.00119  0.03449 
##  Residual                   0.71875  0.84779 
## Number of obs: 2798, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                                            Estimate Std. Error        df
## (Intercept)                               2.623e+00  6.895e-02 2.820e+01
## youth_activity_threeBasic Skills Activity 1.551e-01  4.244e-02 2.631e+03
## youth_activity_threeCreating Product      6.126e-02  4.643e-02 2.610e+03
## agency_dummy                              3.645e-02  4.426e-02 2.649e+03
## stem_sb_dummy                             9.638e-02  4.553e-02 2.634e+03
##                                           t value Pr(>|t|)    
## (Intercept)                                38.040  < 2e-16 ***
## youth_activity_threeBasic Skills Activity   3.655 0.000262 ***
## youth_activity_threeCreating Product        1.319 0.187153    
## agency_dummy                                0.824 0.410259    
## stem_sb_dummy                               2.117 0.034377 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) y__BSA yt__CP agncy_
## yth_ctv_BSA -0.065                     
## yth_ctvt_CP  0.020  0.262              
## agency_dmmy -0.442 -0.034 -0.215       
## stm_sb_dmmy -0.458 -0.157 -0.138 -0.091

4. Affect

4A: Just youth activity

m <- lmer(positive_affect ~ 
              youth_activity_rc + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: positive_affect ~ youth_activity_rc + (1 | program_ID) + (1 |  
##     participant_ID)
##    Data: df
## 
## REML criterion at convergence: 6956.7
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.5513 -0.4396  0.0712  0.5666  3.5332 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.4917   0.7012  
##  program_ID     (Intercept) 0.1106   0.3326  
##  Residual                   0.5729   0.7569  
## Number of obs: 2818, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                                          Estimate Std. Error         df
## (Intercept)                             2.704e+00  1.250e-01  8.000e+00
## youth_activity_rcBasic Skills Activity  2.750e-02  4.204e-02  2.653e+03
## youth_activity_rcCreating Product      -4.754e-03  4.294e-02  2.657e+03
## youth_activity_rcField Trip Speaker    -2.771e-03  7.732e-02  2.639e+03
## youth_activity_rcLab Activity           6.033e-02  7.616e-02  2.635e+03
## youth_activity_rcProgram Staff Led     -4.359e-02  4.774e-02  2.638e+03
##                                        t value Pr(>|t|)    
## (Intercept)                             21.624 2.12e-08 ***
## youth_activity_rcBasic Skills Activity   0.654    0.513    
## youth_activity_rcCreating Product       -0.111    0.912    
## youth_activity_rcField Trip Speaker     -0.036    0.971    
## youth_activity_rcLab Activity            0.792    0.428    
## youth_activity_rcProgram Staff Led      -0.913    0.361    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) y__BSA yt__CP y__FTS yt__LA
## yth_ctv_BSA -0.134                            
## yth_ctvt_CP -0.127  0.371                     
## yth_ctv_FTS -0.076  0.248  0.219              
## yth_ctvt_LA -0.070  0.205  0.200  0.148       
## yth_ctv_PSL -0.116  0.414  0.305  0.193  0.188

4B: Just STEM-SB as counts

m <- lmer(positive_affect ~ 
              active +
              ho_thinking +
              belonging +
              agency +
              stem_sb + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: positive_affect ~ active + ho_thinking + belonging + agency +  
##     stem_sb + (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 6887.8
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.6013 -0.4594  0.0596  0.5555  3.6046 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.4913   0.7009  
##  program_ID     (Intercept) 0.1082   0.3289  
##  Residual                   0.5652   0.7518  
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##               Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)  2.644e+00  1.305e-01  9.900e+00  20.271 2.23e-09 ***
## active       4.583e-02  3.440e-02  2.621e+03   1.332   0.1829    
## ho_thinking -3.913e-03  1.789e-02  2.636e+03  -0.219   0.8269    
## belonging    3.259e-02  2.450e-02  2.619e+03   1.330   0.1836    
## agency       1.466e-02  1.334e-02  2.628e+03   1.100   0.2716    
## stem_sb     -1.851e-02  8.414e-03  2.628e+03  -2.200   0.0279 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) active h_thnk blngng agency
## active      -0.261                            
## ho_thinking -0.065 -0.203                     
## belonging   -0.050 -0.011 -0.104              
## agency      -0.087  0.129 -0.178 -0.427       
## stem_sb      0.100 -0.446 -0.278 -0.164 -0.164

4C: Just STEM-SB as dummy codes (if count >= 1, then dummy code = 1)

m <- lmer(positive_affect ~ 
              active_dummy +
              ho_thinking +
              belonging_dummy +
              agency_dummy +
              stem_sb_dummy +
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: positive_affect ~ active_dummy + ho_thinking + belonging_dummy +  
##     agency_dummy + stem_sb_dummy + (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 6873.3
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.6249 -0.4630  0.0470  0.5445  3.5490 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.4895   0.6997  
##  program_ID     (Intercept) 0.1111   0.3333  
##  Residual                   0.5642   0.7511  
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                   Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)        2.62703    0.16892   26.60000  15.552 7.11e-15 ***
## active_dummy       0.01279    0.11945 2607.40000   0.107  0.91477    
## ho_thinking       -0.01351    0.01717 2647.40000  -0.787  0.43148    
## belonging_dummy    0.04254    0.04046 2619.00000   1.052  0.29311    
## agency_dummy       0.11316    0.04312 2620.90000   2.624  0.00873 ** 
## stem_sb_dummy     -0.03801    0.04507 2642.00000  -0.843  0.39909    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) actv_d h_thnk blngn_ agncy_
## active_dmmy -0.627                            
## ho_thinking  0.007 -0.040                     
## blngng_dmmy -0.025 -0.101 -0.243              
## agency_dmmy -0.089 -0.019 -0.158 -0.367       
## stm_sb_dmmy -0.033 -0.182 -0.423  0.159 -0.040

4D: Only select activities and STEM-SB dummy codes

m <- lmer(positive_affect ~ 
              youth_activity_three +
              agency_dummy +
              stem_sb_dummy + 
              (1|program_ID) + (1|participant_ID),
          data = df)

summary(m)

## Linear mixed model fit by REML t-tests use Satterthwaite approximations
##   to degrees of freedom [lmerMod]
## Formula: 
## positive_affect ~ youth_activity_three + agency_dummy + stem_sb_dummy +  
##     (1 | program_ID) + (1 | participant_ID)
##    Data: df
## 
## REML criterion at convergence: 6868.4
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.6357 -0.4582  0.0463  0.5428  3.5313 
## 
## Random effects:
##  Groups         Name        Variance Std.Dev.
##  participant_ID (Intercept) 0.4902   0.7001  
##  program_ID     (Intercept) 0.1132   0.3365  
##  Residual                   0.5637   0.7508  
## Number of obs: 2799, groups:  participant_ID, 203; program_ID, 9
## 
## Fixed effects:
##                                             Estimate Std. Error         df
## (Intercept)                                  2.64465    0.13150    9.50000
## youth_activity_threeBasic Skills Activity    0.03046    0.03776 2629.20000
## youth_activity_threeCreating Product        -0.04329    0.04138 2634.60000
## agency_dummy                                 0.13410    0.03934 2629.20000
## stem_sb_dummy                               -0.05356    0.04053 2636.50000
##                                           t value Pr(>|t|)    
## (Intercept)                                20.111 3.99e-09 ***
## youth_activity_threeBasic Skills Activity   0.807 0.419894    
## youth_activity_threeCreating Product       -1.046 0.295570    
## agency_dummy                                3.409 0.000662 ***
## stem_sb_dummy                              -1.322 0.186425    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) y__BSA yt__CP agncy_
## yth_ctv_BSA -0.029                     
## yth_ctvt_CP  0.008  0.262              
## agency_dmmy -0.206 -0.035 -0.214       
## stm_sb_dmmy -0.214 -0.159 -0.141 -0.091