Varsity Tutors Analyses

Author

Alexis Davila

Import

Packages to conduct analyses.

library(tidyverse)
── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.2.1     ✔ readr     2.2.0
✔ forcats   1.0.1     ✔ stringr   1.6.0
✔ ggplot2   4.0.3     ✔ tibble    3.3.1
✔ lubridate 1.9.5     ✔ tidyr     1.3.2
✔ purrr     1.2.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(sandwich)
library(lmtest)
Loading required package: zoo

Attaching package: 'zoo'

The following objects are masked from 'package:base':

    as.Date, as.Date.numeric
library(psych)

Attaching package: 'psych'

The following objects are masked from 'package:ggplot2':

    %+%, alpha

Import Kent data.

kent = read_csv("vt_kent_all_students.csv")
Rows: 5634 Columns: 45
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr  (4): school, school_grade, race, race_recoded
dbl (41): student_id, school_num, grade, grade6, grade7, grade8, male, sped,...

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
colnames(kent)
 [1] "student_id"                     "school_num"                    
 [3] "school"                         "grade"                         
 [5] "school_grade"                   "grade6"                        
 [7] "grade7"                         "grade8"                        
 [9] "male"                           "sped"                          
[11] "mll"                            "low_inc"                       
[13] "low_ap"                         "race"                          
[15] "race_recoded"                   "race_A"                        
[17] "race_B"                         "race_H"                        
[19] "race_O"                         "race_W"                        
[21] "moy_math_comp"                  "moy_math_z"                    
[23] "moy_ela_comp"                   "moy_ela_z"                     
[25] "eoy_math_comp"                  "eoy_math_z"                    
[27] "eoy_ela_comp"                   "eoy_ela_z"                     
[29] "vt_acct_id"                     "vt_present"                    
[31] "num_days_active"                "num_days_active_recoded"       
[33] "num_weeks"                      "num_sessions_week"             
[35] "total_sessions"                 "num_sessions_AI"               
[37] "num_sessions_human"             "num_classesviewed"             
[39] "num_essaysreviewed"             "num_AI_practiceproblemsessions"
[41] "both_math"                      "both_ela"                      
[43] "all_tests"                      "math_moy_missing_eoy_present"  
[45] "ela_moy_missing_eoy_present"

Import Penn data.

penn = read_csv("vt_penn_all_students.csv")
Rows: 1279 Columns: 46
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr  (4): school, school_grade, race, race_recoded
dbl (42): student_id, school_num, grade, grade3, grade4, grade5, grade6, mal...

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
colnames(penn)
 [1] "student_id"                     "school_num"                    
 [3] "school"                         "grade"                         
 [5] "school_grade"                   "grade3"                        
 [7] "grade4"                         "grade5"                        
 [9] "grade6"                         "male"                          
[11] "iep"                            "ell"                           
[13] "frpl"                           "race"                          
[15] "race_recoded"                   "race_A"                        
[17] "race_B"                         "race_H"                        
[19] "race_O"                         "race_W"                        
[21] "moy_math_comp"                  "moy_math_z"                    
[23] "moy_ela_comp"                   "moy_ela_z"                     
[25] "eoy_math_comp"                  "eoy_math_z"                    
[27] "eoy_ela_comp"                   "eoy_ela_z"                     
[29] "treat"                          "vt_acct_id"                    
[31] "vt_present"                     "num_days_active"               
[33] "num_days_active_recoded"        "num_weeks"                     
[35] "num_sessions_week"              "total_sessions"                
[37] "num_sessions_AI"                "num_sessions_human"            
[39] "num_classesviewed"              "num_essaysreviewed"            
[41] "num_AI_practiceproblemsessions" "both_math"                     
[43] "both_ela"                       "all_tests"                     
[45] "math_moy_missing_eoy_present"   "ela_moy_missing_eoy_present"

Import Cumberland data.

cumber = read_csv("vt_cumber_all_students.csv")
Rows: 8639 Columns: 46
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr  (4): school, school_grade, race, race_recoded
dbl (41): student_id, school_num, grade, grade6, grade7, grade8, male, iep, ...
lgl  (1): frpl

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
colnames(cumber)
 [1] "student_id"                     "school_num"                    
 [3] "school"                         "grade"                         
 [5] "school_grade"                   "grade6"                        
 [7] "grade7"                         "grade8"                        
 [9] "male"                           "iep"                           
[11] "ell"                            "frpl"                          
[13] "at_risk"                        "race"                          
[15] "race_recoded"                   "race_B"                        
[17] "race_H"                         "race_Other"                    
[19] "race_Multi"                     "race_W"                        
[21] "moy_math_comp"                  "moy_math_z"                    
[23] "moy_ela_comp"                   "moy_ela_z"                     
[25] "eoy_math_comp"                  "eoy_math_z"                    
[27] "eoy_ela_comp"                   "eoy_ela_z"                     
[29] "treat"                          "vt_acct_id"                    
[31] "vt_present"                     "num_days_active"               
[33] "num_days_active_recoded"        "num_weeks"                     
[35] "num_sessions_week"              "total_sessions"                
[37] "num_sessions_AI"                "num_sessions_human"            
[39] "num_classesviewed"              "num_essaysreviewed"            
[41] "num_AI_practiceproblemsessions" "both_math"                     
[43] "both_ela"                       "all_tests"                     
[45] "math_moy_missing_eoy_present"   "ela_moy_missing_eoy_present"

Preparing for pooled variables

Create district columns

penn$district <- "Penn"
kent$district <- "Kent"
cumber$district <- "Cumberland"

Clean race

Inspect similar categories for cleaning pooled variables later (after appending).

table(penn$race)

    AMERICAN INDIAN/ALASKAN                       ASIAN 
                         22                          31 
     BLACK/AFRICAN AMERICAN                    HISPANIC 
                       1101                          43 
MULTI RACIAL - NON HISPANIC             NATIVE HAWAIIAN 
                          1                           8 
            WHITE/CAUCASIAN 
                         73 
table(kent$race)

         American Indian or Alaska Native 
                                       21 
                                    Asian 
                                     1365 
                Black or African American 
                                      760 
                          Hispanic/Latino 
                                     1312 
Native Hawaiian or Other Pacific Islander 
                                      189 
                        Two or More Races 
                                      512 
                                    White 
                                     1475 
table(cumber$race)

    A     B     H     I Multi     P     W 
  181  4069  1465   100   816    46  1962

Clean IEP

Inspect Kent’s Special Ed (sped) variable.

table(kent$sped)

   0    1 
4803  831

Decision: Treat Kent’s SPED variable as an IEP to align with Cumberland and Penn.

kent$iep = kent$sped
table(kent$iep)

   0    1 
4803  831

Clean ELL

Inspect Kent’s MLL variable.

table(kent$mll)

   0    1 
4023 1611

Decision: Treat Kent’s MLL variable as ELL to align with Cumberland and Penn.

kent$ell= kent$mll
table(kent$ell)

   0    1 
4023 1611

Clean FRPL

Free/ reduced price lunch as proxy for low income status. FRPL for Penn, Low-income for Kent, but FRPL missing for Cumberland.

Inspect frpl for Penn.

table(penn$frpl)

  0   1 
353 926

Decision: Treat Penn’s FRPL as low income variable to align with Kent.

penn$low_inc = penn$frpl
table(penn$low_inc)

  0   1 
353 926

Decision: Treat Cumberland’s FRPL as low income variable to align with Kent. Note, since FRPL data was all missing, Cumberland will not have low-income status available and thus will be excluded from this subgroup analysis.

table(cumber$frpl)
< table of extent 0 >
cumber$low_inc = cumber$frpl
table(cumber$low_inc)
< table of extent 0 >

Clean At-risk

Inspect Kent’s variable for low academic performance

table(kent$low_ap)

   0    1 
4157 1477

Decision: Treat Kent’s variable for low academic performance as at-risk variable to align with Cumberland. Since nothing similar is present fr Penn, create at-risk variable for Penn where all entries at NA. Note, since at-risk data was not present, Penn will not have at-risk status available and thus will be excluded from this subgroup analysis.

kent$at_risk = kent$low_ap
table(kent$at_risk)

   0    1 
4157 1477 
penn$at_risk <- NA
table(penn$at_risk)
< table of extent 0 >

Check Cumberland coding for at-risk.

table(cumber$at_risk)

   0    1 
5973 2666

Pool Data

Select columns for appending

#PENN
penn_to_pool = penn %>%
  dplyr::select(student_id,
                school_num, school, district, 
                grade, school_grade,
                
                male, iep, ell, low_inc, at_risk, race,
                
                moy_math_comp, moy_math_z,
                eoy_math_comp, eoy_math_z,
                
                moy_ela_comp, moy_ela_z,
                eoy_ela_comp, eoy_ela_z, 
                
                vt_acct_id, vt_present,
                num_days_active, num_days_active_recoded, 
                num_weeks, num_sessions_week,total_sessions,
                num_sessions_AI, num_sessions_human,
                num_classesviewed, num_essaysreviewed, num_AI_practiceproblemsessions, 
                
                both_math, both_ela, all_tests, 
                math_moy_missing_eoy_present, ela_moy_missing_eoy_present)

#KENT
kent_to_pool = kent %>%
  dplyr::select(student_id,
                school_num, school, district,  
                grade, school_grade,
                
                male, iep, ell, low_inc, at_risk, race,
                
                moy_math_comp, moy_math_z,
                eoy_math_comp, eoy_math_z,
                
                moy_ela_comp, moy_ela_z,
                eoy_ela_comp, eoy_ela_z, 
                
                vt_acct_id, vt_present,
                num_days_active, num_days_active_recoded, 
                num_weeks, num_sessions_week,total_sessions,
                num_sessions_AI, num_sessions_human,
                num_classesviewed, num_essaysreviewed, num_AI_practiceproblemsessions, 
                
                both_math, both_ela, all_tests, 
                math_moy_missing_eoy_present, ela_moy_missing_eoy_present)

#CUMBERLAND
cumber_to_pool = cumber %>%
  dplyr::select(student_id,
                school_num, school, district, 
                grade, school_grade,
                
                male, iep, ell, low_inc, at_risk, race,
                
                moy_math_comp, moy_math_z,
                eoy_math_comp, eoy_math_z,
                
                moy_ela_comp, moy_ela_z,
                eoy_ela_comp, eoy_ela_z, 
                
                vt_acct_id, vt_present,
                num_days_active, num_days_active_recoded, 
                num_weeks, num_sessions_week,total_sessions,
                num_sessions_AI, num_sessions_human,
                num_classesviewed, num_essaysreviewed, num_AI_practiceproblemsessions, 
                
                both_math, both_ela, all_tests, 
                math_moy_missing_eoy_present, ela_moy_missing_eoy_present)

Append data

Use rbind since columns matched by name and by order.

pooled = rbind(
  penn_to_pool,
  kent_to_pool,
  cumber_to_pool
)

Finish pooled race

table(pooled$race)

                                        A 
                                      181 
         American Indian or Alaska Native 
                                       21 
                  AMERICAN INDIAN/ALASKAN 
                                       22 
                                    Asian 
                                     1365 
                                    ASIAN 
                                       31 
                                        B 
                                     4069 
                Black or African American 
                                      760 
                   BLACK/AFRICAN AMERICAN 
                                     1101 
                                        H 
                                     1465 
                                 HISPANIC 
                                       43 
                          Hispanic/Latino 
                                     1312 
                                        I 
                                      100 
                                    Multi 
                                      816 
              MULTI RACIAL - NON HISPANIC 
                                        1 
                          NATIVE HAWAIIAN 
                                        8 
Native Hawaiian or Other Pacific Islander 
                                      189 
                                        P 
                                       46 
                        Two or More Races 
                                      512 
                                        W 
                                     1962 
                                    White 
                                     1475 
                          WHITE/CAUCASIAN 
                                       73

Decision: Combine categories where they appears across all districts: White, Black, Hispanic, Asian, American Indian/Native, Multi-racial, Pacific Islander.

pooled = pooled %>%
mutate(
race_pooled = case_when(
race %in% c("WHITE/CAUCASIAN", "White", "W") ~ "White",
race %in% c("BLACK/AFRICAN AMERICAN","Black or African American", "B") ~ "Black",
race %in% c("ASIAN", "Asian", "A") ~ "Asian",
race %in% c("HISPANIC", "Hispanic/Latino","H") ~ "Hispanic",
race %in% c("MULTI RACIAL - NON HISPANIC","Two or More Races", "Multi") ~ "Multi-racial",
race %in% c("AMERICAN INDIAN/ALASKAN", "American Indian or Alaska Native", "I") ~ "Native American",#for American Indian
race %in% c("NATIVE HAWAIIAN", "Native Hawaiian or Other Pacific Islander", "P") ~ "Pacific Islander",
TRUE ~ NA_character_ 

)
)

table(pooled$race_pooled)

           Asian            Black         Hispanic     Multi-racial 
            1577             5930             2820             1329 
 Native American Pacific Islander            White 
             143              243             3510

Subset students offered Live + AI only

Create pooled data frame containing only students with VT Account IDs. This will also include treatment students with no usage or usage falling below the threshold.

pooled_vt = pooled %>%
  filter(vt_present == 1)
print(pooled_vt)
# A tibble: 1,194 × 38
   student_id school_num school    district grade school_grade  male   iep   ell
        <dbl>      <dbl> <chr>     <chr>    <dbl> <chr>        <dbl> <dbl> <dbl>
 1     203802          4 Colwyn E… Penn         6 Colwyn 6         0     1     0
 2     203862          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
 3     204106          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
 4     204166          4 Colwyn E… Penn         6 Colwyn 6         0     0     0
 5     204211          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
 6     204755          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
 7     204759          4 Colwyn E… Penn         5 Colwyn 5         1     0     0
 8     204769          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
 9     204787          4 Colwyn E… Penn         5 Colwyn 5         1     1     0
10     204843          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
# ℹ 1,184 more rows
# ℹ 29 more variables: low_inc <dbl>, at_risk <dbl>, race <chr>,
#   moy_math_comp <dbl>, moy_math_z <dbl>, eoy_math_comp <dbl>,
#   eoy_math_z <dbl>, moy_ela_comp <dbl>, moy_ela_z <dbl>, eoy_ela_comp <dbl>,
#   eoy_ela_z <dbl>, vt_acct_id <dbl>, vt_present <dbl>, num_days_active <dbl>,
#   num_days_active_recoded <dbl>, num_weeks <dbl>, num_sessions_week <dbl>,
#   total_sessions <dbl>, num_sessions_AI <dbl>, num_sessions_human <dbl>, …

Research Question 1

What are the demographic and academic characteristics of students and schools in LIVE+ AI schools and non-LIVE+ AI schools? We will summarize the demographic and academic characteristics of students and schools in the treatment and comparison groups by reporting descriptive statistics (mean, median, SDs) across both groups. In accordance with What Works Clearinghouse v5.0 standards, we will also report standardized mean differences between the treatment and comparison schools.

Prep work

Function for data summary

#create function to streamline calculations
calc_pct = function(x) {
  prop.table(table(x)) * 100
}

Reconstruct treat using number of sessions per week

  1. Filter out “treatment” students who do not meet usage threshold. They are not part of the final sample group and should not be used as comparison student under RQ1. Among students where vt_present = 1, exclude those where num_sessions_week is less than once per week.
pooled_treat_comp = pooled %>%
# keep those who don't appear in usage data (comp group) OR students that meet usage threshold (treat).
   filter(vt_present != 1 |  num_sessions_week >= 1) 
print(pooled_treat_comp)
# A tibble: 14,894 × 38
   student_id school_num school    district grade school_grade  male   iep   ell
        <dbl>      <dbl> <chr>     <chr>    <dbl> <chr>        <dbl> <dbl> <dbl>
 1     203802          4 Colwyn E… Penn         6 Colwyn 6         0     1     0
 2     203862          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
 3     204106          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
 4     204769          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
 5     204787          4 Colwyn E… Penn         5 Colwyn 5         1     1     0
 6     204896          4 Colwyn E… Penn         5 Colwyn 5         1     0     0
 7     204671          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
 8     205051          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
 9     205898          4 Colwyn E… Penn         6 Colwyn 6         0     0     0
10     206039          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
# ℹ 14,884 more rows
# ℹ 29 more variables: low_inc <dbl>, at_risk <dbl>, race <chr>,
#   moy_math_comp <dbl>, moy_math_z <dbl>, eoy_math_comp <dbl>,
#   eoy_math_z <dbl>, moy_ela_comp <dbl>, moy_ela_z <dbl>, eoy_ela_comp <dbl>,
#   eoy_ela_z <dbl>, vt_acct_id <dbl>, vt_present <dbl>, num_days_active <dbl>,
#   num_days_active_recoded <dbl>, num_weeks <dbl>, num_sessions_week <dbl>,
#   total_sessions <dbl>, num_sessions_AI <dbl>, num_sessions_human <dbl>, …
  1. Create treat: Equals 1 if students had more than 2 sessions. All else equals 0. Kent = 87, Cumberland = 350, Kent = 99, Total = 536.
pooled_treat_comp = pooled_treat_comp %>%
  mutate(treat = ifelse(!is.na(num_sessions_week) &  num_sessions_week >= 1, 1, 0))
table(pooled_treat_comp$treat)

    0     1 
14358   536

School level characteristics

Characteristics at the school level include MOY math and ELA scores, enrollment, and the percentage of students who are Black, Hispanic, White, FRPL-eligible, have IEPs, and are female. Note, each school is either a treatment or comparison school; this data frame does not contain students in comparison schools with unexpected usage (clean groupings by school).

#school-level means and counts for TEST data
pooled_treat_comp = pooled_treat_comp %>%
  mutate(across(c(moy_math_z, #transform all scores to numeric value
                  moy_math_comp,
                  moy_ela_z, 
                  moy_ela_comp,
                  
                  eoy_math_z, 
                  eoy_math_comp,
                  eoy_ela_z,
                  eoy_ela_comp), as.numeric))

school_means = pooled_treat_comp %>%
  group_by(school)%>%
  summarise(
    n_students = n(),
    mean_moy_math_z = mean(moy_math_z, na.rm = TRUE),
     mean_moy_math_raw = mean(moy_math_comp, na.rm = TRUE),
    mean_moy_ela_z  = mean(moy_ela_z, na.rm = TRUE),
      mean_moy_ela_raw  = mean(moy_ela_comp, na.rm = TRUE),
    
    mean_eoy_math_z = mean(eoy_math_z, na.rm = TRUE),
     mean_eoy_math_raw = mean(eoy_math_comp, na.rm = TRUE),
    mean_eoy_ela_z  = mean(eoy_ela_z, na.rm = TRUE),
      mean_eoy_ela_raw  = mean(eoy_ela_comp, na.rm = TRUE),
  )

# DEMOGRAPHICS

# race/ethnicity
race_pct = pooled_treat_comp %>%
  group_by(school, race_pooled) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(school) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = race_pooled,
    values_from = pct,
    names_prefix = "race_"
  )

#male
male_pct = pooled_treat_comp %>%
  group_by(school, male) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(school) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = male,
    values_from = pct,
    names_prefix = "male_"
  )

#low income status
low_inc_pct = pooled_treat_comp %>%
  group_by(school, low_inc) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(school) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = low_inc,
    values_from = pct,
    names_prefix = "low_inc_"
  )

#ell
ell_pct = pooled_treat_comp %>%
  group_by(school, ell) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(school) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = ell,
    values_from = pct,
    names_prefix = "ell_"
  )

#at-risk 
at_risk_pct = pooled_treat_comp %>%
  group_by(school, at_risk) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(school) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = at_risk,
    values_from = pct,
    names_prefix = "at_risk_"
  )

#iep students
iep_pct = pooled_treat_comp %>%
  group_by(school, iep) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(school) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = iep,
    values_from = pct,
    names_prefix = "iep_"
  )

# COMBINE summaries
school_summary = school_means %>%
  left_join(race_pct,   by = "school") %>%
  left_join(male_pct, by = "school") %>%
  left_join(low_inc_pct,   by = "school") %>%
  left_join(ell_pct,    by = "school") %>%
  left_join(at_risk_pct, by = "school")%>%
  left_join(iep_pct, by = "school")

school_summary = school_summary %>%
  mutate(across(where(is.numeric), ~ round(.x, 2)))

print(school_summary)
# A tibble: 32 × 30
   school            n_students mean_moy_math_z mean_moy_math_raw mean_moy_ela_z
   <chr>                  <dbl>           <dbl>             <dbl>          <dbl>
 1 Aldan Elementary…         53           -0.19              201.          -0.08
 2 Anne Chesnutt Mi…        492           -0.07              220.           0.01
 3 Ardmore Avenue E…        311           -0.3               202.          -0.49
 4 Bell Avenue Elem…         55           -0.35              201.          -0.6 
 5 Canyon Ridge Mid…        834           -0.51              471.          -0.54
 6 Cedar Heights Mi…        844           -0.05              489.          -0.12
 7 Colwyn Elementar…         29           -0.31              201.          -0.32
 8 Cumberland Virtu…        126            0.9               239.           0.68
 9 Douglas Byrd Mid…        642           -0.32              214.          -0.23
10 East Lansdowne E…        155           -0.08              206.          -0.03
# ℹ 22 more rows
# ℹ 25 more variables: mean_moy_ela_raw <dbl>, mean_eoy_math_z <dbl>,
#   mean_eoy_math_raw <dbl>, mean_eoy_ela_z <dbl>, mean_eoy_ela_raw <dbl>,
#   race_Asian <dbl>, race_Black <dbl>, `race_Native American` <dbl>,
#   race_White <dbl>, race_Hispanic <dbl>, `race_Multi-racial` <dbl>,
#   `race_Pacific Islander` <dbl>, male_0 <dbl>, male_1 <dbl>, male_NA <dbl>,
#   low_inc_0 <dbl>, low_inc_1 <dbl>, low_inc_NA <dbl>, ell_0 <dbl>, …
write.csv(school_summary, "vt_pooled_RQ1_summary_by_school.csv", row.names = FALSE)

Student level characteristics

Characteristics at the student level include MOY math and ELA scores and the percentage of students who are Black, Hispanic, White, FRPL-eligible, have IEPs and are female.

#school-level means and counts for test data
treat_means = pooled_treat_comp %>%
  group_by(treat)%>%
  summarise(
    n_students = n(),
    mean_moy_math_z = mean(moy_math_z, na.rm = TRUE),
     mean_moy_math_raw = mean(moy_math_comp, na.rm = TRUE),
    mean_moy_ela_z  = mean(moy_ela_z, na.rm = TRUE),
      mean_moy_ela_raw  = mean(moy_ela_comp, na.rm = TRUE),
    
    mean_eoy_math_z = mean(eoy_math_z, na.rm = TRUE),
     mean_eoy_math_raw = mean(eoy_math_comp, na.rm = TRUE),
    mean_eoy_ela_z  = mean(eoy_ela_z, na.rm = TRUE),
      mean_eoy_ela_raw  = mean(eoy_ela_comp, na.rm = TRUE), 
  )

# DEMOGRAPHICS

# race/ethnicity
race_pct = pooled_treat_comp %>%
  group_by(treat, race_pooled) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(treat) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = race_pooled,
    values_from = pct,
    names_prefix = "race_"
  )

#male
male_pct = pooled_treat_comp %>%
  group_by(treat, male) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(treat) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = male,
    values_from = pct,
    names_prefix = "male_"
  )

#low income status
low_inc_pct = pooled_treat_comp %>%
  group_by(treat, low_inc) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(treat) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = low_inc,
    values_from = pct,
    names_prefix = "low_inc_"
  )

#ell
ell_pct = pooled_treat_comp %>%
  group_by(treat, ell) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(treat) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = ell,
    values_from = pct,
    names_prefix = "ell_"
  )

#at-risk 
at_risk_pct = pooled_treat_comp %>%
  group_by(treat, at_risk) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(treat) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = at_risk,
    values_from = pct,
    names_prefix = "at_risk_"
  )

#iep students
iep_pct = pooled_treat_comp %>%
  group_by(treat, iep) %>%
  summarise(n = n(), .groups = "drop") %>%
  group_by(treat) %>%
  mutate(pct = (n / sum(n)) * 100) %>%
  select(-n) %>%
  pivot_wider(
    names_from = iep,
    values_from = pct,
    names_prefix = "iep_"
  )

# COMBINE summaries
student_summary = treat_means %>%
  left_join(race_pct,   by = "treat") %>%
  left_join(male_pct, by = "treat") %>%
  left_join(low_inc_pct,   by = "treat") %>%
  left_join(ell_pct,    by = "treat") %>%
  left_join(at_risk_pct, by = "treat")%>%
  left_join(iep_pct, by = "treat")

student_summary = student_summary %>%
  mutate(across(where(is.numeric), ~ round(.x, 2)))

print(student_summary)
# A tibble: 2 × 30
  treat n_students mean_moy_math_z mean_moy_math_raw mean_moy_ela_z
  <dbl>      <dbl>           <dbl>             <dbl>          <dbl>
1     0      14358           -0.11              317.          -0.15
2     1        536           -0.17              258.          -0.14
# ℹ 25 more variables: mean_moy_ela_raw <dbl>, mean_eoy_math_z <dbl>,
#   mean_eoy_math_raw <dbl>, mean_eoy_ela_z <dbl>, mean_eoy_ela_raw <dbl>,
#   race_Asian <dbl>, race_Black <dbl>, race_Hispanic <dbl>,
#   `race_Multi-racial` <dbl>, `race_Native American` <dbl>,
#   `race_Pacific Islander` <dbl>, race_White <dbl>, male_0 <dbl>,
#   male_1 <dbl>, male_NA <dbl>, low_inc_0 <dbl>, low_inc_1 <dbl>,
#   low_inc_NA <dbl>, ell_0 <dbl>, ell_1 <dbl>, at_risk_0 <dbl>, …
write.csv(student_summary, "vt_pooled_RQ1_summary_by_treat.csv", row.names = FALSE)

Research Question 2

What is the average usage and engagement for students using Live+ AI? Does usage vary meaningfully by student characteristics or school?

Descriptive Statistics

*Pulled together statistics that mimic we want did for Table 2 on the mid year report.

By school.

RQ2_school_summary = pooled_vt %>%
  group_by(school) %>%
  summarize(
    n_students = n(),
    n_students_active = sum(total_sessions > 0, na.rm = TRUE),
    avg_sessions_per_week = mean(num_sessions_week, na.rm = TRUE),
    sd_sessions_per_week = sd(num_sessions_week, na.rm = TRUE),
    n_students_meet_threshold = sum(num_sessions_week >= 1, na.rm = TRUE),
    
    p90_sessions_per_week = quantile(
      num_sessions_week,
      probs = 0.90,
      na.rm = TRUE
    ),
    
      p50_sessions_per_week = quantile(
      num_sessions_week,
      probs = 0.50,
      na.rm = TRUE
    ),
    
    
    p25_sessions_per_week = quantile(
      num_sessions_week,
      probs = 0.25,
      na.rm = TRUE
    ),
    
    
    .groups = "drop"
  )

RQ2_school_summary
# A tibble: 7 × 9
  school n_students n_students_active avg_sessions_per_week sd_sessions_per_week
  <chr>       <int>             <int>                 <dbl>                <dbl>
1 Aldan…         85                76                 0.729                0.957
2 Bell …        115               102                 0.989                1.48 
3 Colwy…         71                69                 1.14                 1.16 
4 Luthe…        392               376                 0.902                1.04 
5 Merid…         88                88                 4.14                 2.49 
6 Sprin…        361               358                 2.31                 2.74 
7 Walnu…         82                81                 0.412                0.494
# ℹ 4 more variables: n_students_meet_threshold <int>,
#   p90_sessions_per_week <dbl>, p50_sessions_per_week <dbl>,
#   p25_sessions_per_week <dbl>
write.csv(RQ2_school_summary, file = "vt_pooled_RQ2_school_summary.csv", row.names = FALSE)

By grade.

RQ2_grade_summary = pooled_vt %>%
  group_by(grade) %>%
  summarize(
    n_students = n(),
    n_students_active = sum(total_sessions > 0, na.rm = TRUE),
    avg_sessions_per_week = mean(num_sessions_week, na.rm = TRUE),
    sd_sessions_per_week = sd(num_sessions_week, na.rm = TRUE),
    n_students_meet_threshold = sum(num_sessions_week >= 1, na.rm = TRUE),
    
    p90_sessions_per_week = quantile(
      num_sessions_week,
      probs = 0.90,
      na.rm = TRUE
    ),
    
      p50_sessions_per_week = quantile(
      num_sessions_week,
      probs = 0.50,
      na.rm = TRUE
    ),

    p25_sessions_per_week = quantile(
      num_sessions_week,
      probs = 0.25,
      na.rm = TRUE
    ),
    .groups = "drop"
  )

RQ2_grade_summary
# A tibble: 6 × 9
  grade n_students n_students_active avg_sessions_per_week sd_sessions_per_week
  <dbl>      <int>             <int>                 <dbl>                <dbl>
1     3         84                73                 0.855                1.14 
2     4         99                98                 0.821                0.728
3     5         67                64                 1.17                 1.81 
4     6        439               417                 1.45                 1.81 
5     7        256               252                 1.75                 2.32 
6     8        249               246                 2.10                 2.76 
# ℹ 4 more variables: n_students_meet_threshold <int>,
#   p90_sessions_per_week <dbl>, p50_sessions_per_week <dbl>,
#   p25_sessions_per_week <dbl>
write.csv(RQ2_grade_summary, file = "vt_pooled_RQ2_grade_summary.csv", row.names = FALSE)

Data frame for reference For regressions, only work with those students who had VT Account IDs (those who were offered Live+AI). Pull from data frame where all students were included, regardless of meeting usage threshold.

print(pooled_vt)
# A tibble: 1,194 × 38
   student_id school_num school    district grade school_grade  male   iep   ell
        <dbl>      <dbl> <chr>     <chr>    <dbl> <chr>        <dbl> <dbl> <dbl>
 1     203802          4 Colwyn E… Penn         6 Colwyn 6         0     1     0
 2     203862          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
 3     204106          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
 4     204166          4 Colwyn E… Penn         6 Colwyn 6         0     0     0
 5     204211          4 Colwyn E… Penn         6 Colwyn 6         1     0     0
 6     204755          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
 7     204759          4 Colwyn E… Penn         5 Colwyn 5         1     0     0
 8     204769          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
 9     204787          4 Colwyn E… Penn         5 Colwyn 5         1     1     0
10     204843          4 Colwyn E… Penn         5 Colwyn 5         0     0     0
# ℹ 1,184 more rows
# ℹ 29 more variables: low_inc <dbl>, at_risk <dbl>, race <chr>,
#   moy_math_comp <dbl>, moy_math_z <dbl>, eoy_math_comp <dbl>,
#   eoy_math_z <dbl>, moy_ela_comp <dbl>, moy_ela_z <dbl>, eoy_ela_comp <dbl>,
#   eoy_ela_z <dbl>, vt_acct_id <dbl>, vt_present <dbl>, num_days_active <dbl>,
#   num_days_active_recoded <dbl>, num_weeks <dbl>, num_sessions_week <dbl>,
#   total_sessions <dbl>, num_sessions_AI <dbl>, num_sessions_human <dbl>, …

Pooled - Regress sessions per week ~ School

# Full model
q2_model_pooled_by_school = lm( #before clustering by school
  num_sessions_week ~
    factor(school), 
  data = pooled_vt
)
summary(q2_model_pooled_by_school)

Call:
lm(formula = num_sessions_week ~ factor(school), data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-3.6435 -0.8018 -0.3831  0.2982 17.3861 

Coefficients:
                                              Estimate Std. Error t value
(Intercept)                                     0.7288     0.2016   3.614
factor(school)Bell Avenue Elementary School     0.2607     0.2659   0.980
factor(school)Colwyn Elementary School          0.4091     0.2989   1.369
factor(school)Luther Nick Jeralds Middle        0.1730     0.2224   0.778
factor(school)Meridian Middle                   3.4147     0.2827  12.078
factor(school)Spring Lake Middle                1.5774     0.2241   7.038
factor(school)Walnut Street Elementary School  -0.3167     0.2878  -1.101
                                              Pr(>|t|)    
(Intercept)                                   0.000314 ***
factor(school)Bell Avenue Elementary School   0.327118    
factor(school)Colwyn Elementary School        0.171344    
factor(school)Luther Nick Jeralds Middle      0.436865    
factor(school)Meridian Middle                  < 2e-16 ***
factor(school)Spring Lake Middle              3.29e-12 ***
factor(school)Walnut Street Elementary School 0.271254    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 1.859 on 1187 degrees of freedom
Multiple R-squared:  0.2226,    Adjusted R-squared:  0.2187 
F-statistic: 56.65 on 6 and 1187 DF,  p-value: < 2.2e-16

Reference group is Aldan Elementary School. Results: Aldan had an average a 0.73 sessions/week (p<0.001). Bell, Colwyn, and Luther observed slightly higher usage than Aldan but it was not significant. Meridian observed 3.41 sessions/week more than Aldan (p<0.001). Spring Lake observed 1.60 sessions/week more than Aldan (p<0.001). Walnut Street students logged 0.31 sessions/week less than Aldan but it was not signifcantly different.

Self QA - Passed

q2_qa_usage_by_school = pooled_vt %>%
  group_by(school) %>%
  summarise(
    mean_num_sessions_week = mean(num_sessions_week, na.rm = TRUE),
    .groups = "drop"
  )
print(q2_qa_usage_by_school)
# A tibble: 7 × 2
  school                          mean_num_sessions_week
  <chr>                                            <dbl>
1 Aldan Elementary School                          0.729
2 Bell Avenue Elementary School                    0.989
3 Colwyn Elementary School                         1.14 
4 Luther Nick Jeralds Middle                       0.902
5 Meridian Middle                                  4.14 
6 Spring Lake Middle                               2.31 
7 Walnut Street Elementary School                  0.412

Pooled - Regress sessions per week ~ District

# Full model
q2_model_pooled_by_district = lm( #before clustering by school
  num_sessions_week ~
    factor(district), 
  data = pooled_vt
)
summary(q2_model_pooled_by_district)

Call:
lm(formula = num_sessions_week ~ factor(district), data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-3.6435 -1.0366 -0.5593  0.3355 18.1172 

Coefficients:
                     Estimate Std. Error t value Pr(>|t|)    
(Intercept)           1.57508    0.07082  22.242  < 2e-16 ***
factor(district)Kent  2.56838    0.21892  11.732  < 2e-16 ***
factor(district)Penn -0.75266    0.12535  -6.004 2.55e-09 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 1.943 on 1191 degrees of freedom
Multiple R-squared:  0.1477,    Adjusted R-squared:  0.1462 
F-statistic: 103.2 on 2 and 1191 DF,  p-value: < 2.2e-16
###################################
##Cluster-robust standard errors by school
q2_pooled_district_clustered = vcovCL(
  q2_model_pooled_by_district,
  cluster = ~ school_num
)

coeftest(
  q2_model_pooled_by_district,
  vcov = q2_pooled_district_clustered #after clustering by school
)

t test of coefficients:

                     Estimate Std. Error t value Pr(>|t|)    
(Intercept)           1.57508    0.53586  2.9394 0.003352 ** 
factor(district)Kent  2.56838    0.53586  4.7930 1.85e-06 ***
factor(district)Penn -0.75266    0.55365 -1.3595 0.174259    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Cumberland as reference group. Results, after school clustering: Average student at Cumberland has 1.57 sessions/week logged (p<0.01), Penn had 0.75 sessions/week less than Cumberland (not significant) and Kent had 2.56 session/week more than Cumberland (p<0.001).

Self QA - Passed

q2_qa_usage_by_district = pooled_vt %>%
  group_by(district) %>%
  summarise(
    mean_num_sessions_week = mean(num_sessions_week, na.rm = TRUE),
    .groups = "drop"
  )
print(q2_qa_usage_by_district)
# A tibble: 3 × 2
  district   mean_num_sessions_week
  <chr>                       <dbl>
1 Cumberland                  1.58 
2 Kent                        4.14 
3 Penn                        0.822

Pooled - Regress sessions per week ~ Male

# Full model
q2_model_pooled_by_male = lm(
  num_sessions_week ~
    male, 
  data = pooled_vt
)
summary(q2_model_pooled_by_male) #before clustering by school

Call:
lm(formula = num_sessions_week ~ male, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-1.5675 -1.2180 -0.7812  0.4233 18.1743 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  1.51804    0.08456  17.952   <2e-16 ***
male         0.04945    0.12185   0.406    0.685    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 2.104 on 1192 degrees of freedom
Multiple R-squared:  0.0001381, Adjusted R-squared:  -0.0007007 
F-statistic: 0.1647 on 1 and 1192 DF,  p-value: 0.6849
###################################
##Cluster-robust standard errors by school
q2_pooled_male_clustered = vcovCL(
  q2_model_pooled_by_male,
  cluster = ~ school_num
)

coeftest(
  q2_model_pooled_by_male,
  vcov = q2_pooled_male_clustered #after clustering
)

t test of coefficients:

            Estimate Std. Error t value  Pr(>|t|)    
(Intercept) 1.518044   0.404230  3.7554 0.0001814 ***
male        0.049451   0.041079  1.2038 0.2289055    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Female as reference group. Results, after school clustering: Average female has 1.51 sessions/week logged (p<0.001); males had 0.05 sessions/week (not significant).

Self QA - Passed

q2_qa_usage_by_male = pooled_vt %>%
  group_by(male) %>%
  summarise(
    mean_num_sessions_week = mean(num_sessions_week, na.rm = TRUE),
    .groups = "drop"
  )
print(q2_qa_usage_by_male)
# A tibble: 2 × 2
   male mean_num_sessions_week
  <dbl>                  <dbl>
1     0                   1.52
2     1                   1.57

Pooled - Regress sessions per week ~ Race

# Set White as the reference group
pooled_vt$race_pooled = relevel(
  factor(pooled_vt$race_pooled),
  ref = "White"
)

# Full model
q2_model_pooled_by_race = lm(
  num_sessions_week ~
    race_pooled, 
  data = pooled_vt
)
summary(q2_model_pooled_by_race) #before clustering by school

Call:
lm(formula = num_sessions_week ~ race_pooled, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-3.9683 -1.0950 -0.6027  0.4050 18.3973 

Coefficients:
                            Estimate Std. Error t value Pr(>|t|)    
(Intercept)                  1.69329    0.21449   7.895 6.58e-15 ***
race_pooledAsian             2.38024    0.44839   5.308 1.32e-07 ***
race_pooledBlack            -0.39831    0.22609  -1.762   0.0784 .  
race_pooledHispanic          0.48890    0.26659   1.834   0.0669 .  
race_pooledMulti-racial     -0.02315    0.32937  -0.070   0.9440    
race_pooledNative American  -0.53233    0.58739  -0.906   0.3650    
race_pooledPacific Islander  0.94411    0.71495   1.321   0.1869    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 2.046 on 1187 degrees of freedom
Multiple R-squared:  0.05827,   Adjusted R-squared:  0.05351 
F-statistic: 12.24 on 6 and 1187 DF,  p-value: 2.129e-13
###################################
##Cluster-robust standard errors by school
q2_pooled_race_clustered = vcovCL(
  q2_model_pooled_by_race,
  cluster = ~ school_num
)

coeftest(
  q2_model_pooled_by_race,
  vcov = q2_pooled_race_clustered #after clustering
)

t test of coefficients:

                             Estimate Std. Error t value  Pr(>|t|)    
(Intercept)                  1.693294   0.456780  3.7070 0.0002194 ***
race_pooledAsian             2.380240   0.621075  3.8324 0.0001335 ***
race_pooledBlack            -0.398315   0.190278 -2.0933 0.0365315 *  
race_pooledHispanic          0.488896   0.295679  1.6535 0.0984997 .  
race_pooledMulti-racial     -0.023145   0.294932 -0.0785 0.9374630    
race_pooledNative American  -0.532334   0.393565 -1.3526 0.1764424    
race_pooledPacific Islander  0.944110   1.127869  0.8371 0.4027192    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is White. Results after school clustering: Average White student had 1.69 sessions/week (p<0.001). Asian students used 2.38 sessions more per week (p<0.001) compared to White students. Black students used 0.39 sessions less per week (p<0.05) compared to White students. Hispanic students used 0.49 sessions more per week than White students (p<0.1). All other groups’ differences in usage was not significantly difference than Whites.

QA - Passed

q2_qa_usage_by_race = pooled_vt %>%
  group_by(race_pooled) %>%
  summarise(
    mean_num_sessions_week = mean(num_sessions_week, na.rm = TRUE),
    .groups = "drop"
  )
print(q2_qa_usage_by_race)
# A tibble: 7 × 2
  race_pooled      mean_num_sessions_week
  <fct>                             <dbl>
1 White                              1.69
2 Asian                              4.07
3 Black                              1.29
4 Hispanic                           2.18
5 Multi-racial                       1.67
6 Native American                    1.16
7 Pacific Islander                   2.64

Pooled - Regress sessions per week ~ IEP

# Full model
q2_model_pooled_by_iep = lm(
  num_sessions_week ~
    iep, 
  data = pooled_vt
)
summary(q2_model_pooled_by_iep) #before clustering by school

Call:
lm(formula = num_sessions_week ~ iep, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-1.5812 -1.1966 -0.7790  0.4188 18.4418 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  1.58119    0.06478   24.41   <2e-16 ***
iep         -0.33069    0.18786   -1.76   0.0786 .  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 2.101 on 1192 degrees of freedom
Multiple R-squared:  0.002593,  Adjusted R-squared:  0.001756 
F-statistic: 3.099 on 1 and 1192 DF,  p-value: 0.07861
###################################
##Cluster-robust standard errors by school
q2_pooled_iep_clustered = vcovCL(
  q2_model_pooled_by_iep,
  cluster = ~ school_num
)

coeftest(
  q2_model_pooled_by_iep,
  vcov = q2_pooled_iep_clustered #after clustering
)

t test of coefficients:

            Estimate Std. Error t value  Pr(>|t|)    
(Intercept)  1.58119    0.42502  3.7203 0.0002083 ***
iep         -0.33069    0.22998 -1.4379 0.1507233    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is students without IEP. Results after school clustering, Students without IEP averaged 1.58 sessions/week (p<0.001). Compared to students without IEPs, students with IEPs had 0.33 sessions less per week (not significant).

Self QA - Passed

q2_qa_usage_by_iep = pooled_vt %>%
  group_by(iep) %>%
  summarise(
    mean_num_sessions_week = mean(num_sessions_week, na.rm = TRUE),
    .groups = "drop"
  )
print(q2_qa_usage_by_iep)
# A tibble: 2 × 2
    iep mean_num_sessions_week
  <dbl>                  <dbl>
1     0                   1.58
2     1                   1.25

Pooled - Regress sessions per week ~ ELL

# Full model
q2_model_pooled_by_ell = lm(
  num_sessions_week ~
    ell, 
  data = pooled_vt
)
summary(q2_model_pooled_by_ell) #before clustering by school

Call:
lm(formula = num_sessions_week ~ ell, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-3.0966 -1.1372 -0.7372  0.4235 18.2551 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  1.43724    0.06154  23.354  < 2e-16 ***
ell          1.75939    0.25238   6.971 5.19e-12 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 2.062 on 1192 degrees of freedom
Multiple R-squared:  0.03917,   Adjusted R-squared:  0.03837 
F-statistic:  48.6 on 1 and 1192 DF,  p-value: 5.188e-12
###################################
##Cluster-robust standard errors by school
q2_pooled_ell_clustered = vcovCL(
  q2_model_pooled_by_ell,
  cluster = ~ school_num
)

coeftest(
  q2_model_pooled_by_ell,
  vcov = q2_pooled_ell_clustered #after clustering
)

t test of coefficients:

            Estimate Std. Error t value  Pr(>|t|)    
(Intercept)  1.43724    0.37065  3.8776 0.0001112 ***
ell          1.75939    0.47191  3.7283 0.0002018 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group are student who are not ELLs. Results after clustering: Student who are not ELL have 1.44 session per week (p<0.001). ELLs logged 1.76 sessions more per week than non ELLs (p<0.001).

Self QA - Passed

q2_qa_usage_by_ell = pooled_vt %>%
  group_by(ell) %>%
  summarise(
    mean_num_sessions_week = mean(num_sessions_week, na.rm = TRUE),
    .groups = "drop"
  )
print(q2_qa_usage_by_ell)
# A tibble: 2 × 2
    ell mean_num_sessions_week
  <dbl>                  <dbl>
1     0                   1.44
2     1                   3.20

Pooled - Regress sessions per week ~ Low-Income

# Full model
q2_model_pooled_by_low_inc = lm(
  num_sessions_week ~
    low_inc, 
  data = pooled_vt
)
summary(q2_model_pooled_by_low_inc) #before clustering by school

Call:
lm(formula = num_sessions_week ~ low_inc, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-2.0000 -1.1551 -0.7341  0.6344 14.2500 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)   2.0000     0.1716  11.656  < 2e-16 ***
low_inc      -0.7397     0.2057  -3.597 0.000359 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 1.986 on 439 degrees of freedom
  (753 observations deleted due to missingness)
Multiple R-squared:  0.02862,   Adjusted R-squared:  0.02641 
F-statistic: 12.94 on 1 and 439 DF,  p-value: 0.0003591
###################################
##Cluster-robust standard errors by school
q2_pooled_low_inc_clustered = vcovCL(
  q2_model_pooled_by_low_inc,
  cluster = ~ school_num
)

coeftest(
  q2_model_pooled_by_low_inc,
  vcov = q2_pooled_low_inc_clustered #after clustering
)

t test of coefficients:

            Estimate Std. Error t value Pr(>|t|)  
(Intercept)  2.00005    0.88805  2.2522  0.02480 *
low_inc     -0.73968    0.38053 -1.9438  0.05255 .
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is students who are not low income. Results after clustering: Students who are not low income logged 2 sessions per week (p>0.05). Students who are low income logged 0.74 sessions less per week compared to students who are not low income (p<0.1).

Self QA - Passed

NAs are from Cumberland not having FRPL data

q2_qa_usage_by_low_inc = pooled_vt %>%
  group_by(low_inc) %>%
  summarise(
    mean_num_sessions_week = mean(num_sessions_week, na.rm = TRUE),
    .groups = "drop"
  )
print(q2_qa_usage_by_low_inc)
# A tibble: 3 × 2
  low_inc mean_num_sessions_week
    <dbl>                  <dbl>
1       0                   2.00
2       1                   1.26
3      NA                   1.58

Pooled - Regress sessions per week ~ At-Risk

# Full model
q2_model_pooled_by_at_risk = lm(
  num_sessions_week ~
    at_risk, 
  data = pooled_vt
)
summary(q2_model_pooled_by_at_risk) #before clustering by school

Call:
lm(formula = num_sessions_week ~ at_risk, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-1.9539 -1.4539 -0.7728  0.5272 17.9195 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)   1.7728     0.1030  17.217   <2e-16 ***
at_risk       0.1811     0.1644   1.102    0.271    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 2.328 on 839 degrees of freedom
  (353 observations deleted due to missingness)
Multiple R-squared:  0.001444,  Adjusted R-squared:  0.000254 
F-statistic: 1.213 on 1 and 839 DF,  p-value: 0.271
###################################
##Cluster-robust standard errors by school
q2_pooled_at_risk_clustered = vcovCL(
  q2_model_pooled_by_at_risk,
  cluster = ~ school_num
)

coeftest(
  q2_model_pooled_by_at_risk,
  vcov = q2_pooled_at_risk_clustered #after clustering
)

t test of coefficients:

            Estimate Std. Error t value Pr(>|t|)  
(Intercept)  1.77278    0.71607  2.4757  0.01349 *
at_risk      0.18107    0.97133  0.1864  0.85217  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group are student who are not academically at-risk. Results after clustering: students who are not at-risk logged 1.77 sessions/week (p<0.05); compared to at-risk students who logged 0.18 sessions more per week (not significant).

Self QA - Passed Note: Penn had no at-risk status data and was excluded from analysis.

q2_qa_usage_by_at_risk = pooled_vt %>%
  group_by(at_risk) %>%
  summarise(
    mean_num_sessions_week = mean(num_sessions_week, na.rm = TRUE),
    .groups = "drop"
  )
print(q2_qa_usage_by_at_risk)
# A tibble: 3 × 2
  at_risk mean_num_sessions_week
    <dbl>                  <dbl>
1       0                  1.77 
2       1                  1.95 
3      NA                  0.822

Pooled - F Test Usage ~ School

One-way ANOVA F-test.

q2_anova_school = aov(num_sessions_week ~ school, data = pooled_vt)
summary(q2_anova_school)
              Df Sum Sq Mean Sq F value Pr(>F)    
school         6   1175  195.78   56.65 <2e-16 ***
Residuals   1187   4102    3.46                   
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Difference between schools’ usage is significant.

#Signficant results, Tukey 
TukeyHSD(q2_anova_school)
  Tukey multiple comparisons of means
    95% family-wise confidence level

Fit: aov(formula = num_sessions_week ~ school, data = pooled_vt)

$school
                                                                     diff
Bell Avenue Elementary School-Aldan Elementary School          0.26068111
Colwyn Elementary School-Aldan Elementary School               0.40908734
Luther Nick Jeralds Middle-Aldan Elementary School             0.17299314
Meridian Middle-Aldan Elementary School                        3.41467334
Spring Lake Middle-Aldan Elementary School                     1.57740815
Walnut Street Elementary School-Aldan Elementary School       -0.31672582
Colwyn Elementary School-Bell Avenue Elementary School         0.14840623
Luther Nick Jeralds Middle-Bell Avenue Elementary School      -0.08768797
Meridian Middle-Bell Avenue Elementary School                  3.15399222
Spring Lake Middle-Bell Avenue Elementary School               1.31672704
Walnut Street Elementary School-Bell Avenue Elementary School -0.57740693
Luther Nick Jeralds Middle-Colwyn Elementary School           -0.23609420
Meridian Middle-Colwyn Elementary School                       3.00558600
Spring Lake Middle-Colwyn Elementary School                    1.16832081
Walnut Street Elementary School-Colwyn Elementary School      -0.72581316
Meridian Middle-Luther Nick Jeralds Middle                     3.24168019
Spring Lake Middle-Luther Nick Jeralds Middle                  1.40441501
Walnut Street Elementary School-Luther Nick Jeralds Middle    -0.48971896
Spring Lake Middle-Meridian Middle                            -1.83726518
Walnut Street Elementary School-Meridian Middle               -3.73139916
Walnut Street Elementary School-Spring Lake Middle            -1.89413397
                                                                     lwr
Bell Avenue Elementary School-Aldan Elementary School         -0.5246530
Colwyn Elementary School-Aldan Elementary School              -0.4736297
Luther Nick Jeralds Middle-Aldan Elementary School            -0.4839148
Meridian Middle-Aldan Elementary School                        2.5797048
Spring Lake Middle-Aldan Elementary School                     0.9154932
Walnut Street Elementary School-Aldan Elementary School       -1.1665708
Colwyn Elementary School-Bell Avenue Elementary School        -0.6802535
Luther Nick Jeralds Middle-Bell Avenue Elementary School      -0.6699385
Meridian Middle-Bell Avenue Elementary School                  2.3763934
Spring Lake Middle-Bell Avenue Elementary School               0.7288333
Walnut Street Elementary School-Bell Avenue Elementary School -1.3709584
Luther Nick Jeralds Middle-Colwyn Elementary School           -0.9442293
Meridian Middle-Colwyn Elementary School                       2.1297437
Spring Lake Middle-Colwyn Elementary School                    0.4555384
Walnut Street Elementary School-Colwyn Elementary School      -1.6158489
Meridian Middle-Luther Nick Jeralds Middle                     2.5940395
Spring Lake Middle-Luther Nick Jeralds Middle                  1.0039185
Walnut Street Elementary School-Luther Nick Jeralds Middle    -1.1564291
Spring Lake Middle-Meridian Middle                            -2.4899839
Walnut Street Elementary School-Meridian Middle               -4.5741012
Walnut Street Elementary School-Spring Lake Middle            -2.5657780
                                                                     upr
Bell Avenue Elementary School-Aldan Elementary School          1.0460152
Colwyn Elementary School-Aldan Elementary School               1.2918044
Luther Nick Jeralds Middle-Aldan Elementary School             0.8299011
Meridian Middle-Aldan Elementary School                        4.2496419
Spring Lake Middle-Aldan Elementary School                     2.2393231
Walnut Street Elementary School-Aldan Elementary School        0.5331191
Colwyn Elementary School-Bell Avenue Elementary School         0.9770659
Luther Nick Jeralds Middle-Bell Avenue Elementary School       0.4945625
Meridian Middle-Bell Avenue Elementary School                  3.9315911
Spring Lake Middle-Bell Avenue Elementary School               1.9046207
Walnut Street Elementary School-Bell Avenue Elementary School  0.2161446
Luther Nick Jeralds Middle-Colwyn Elementary School            0.4720409
Meridian Middle-Colwyn Elementary School                       3.8814283
Spring Lake Middle-Colwyn Elementary School                    1.8811032
Walnut Street Elementary School-Colwyn Elementary School       0.1642226
Meridian Middle-Luther Nick Jeralds Middle                     3.8893208
Spring Lake Middle-Luther Nick Jeralds Middle                  1.8049115
Walnut Street Elementary School-Luther Nick Jeralds Middle     0.1769912
Spring Lake Middle-Meridian Middle                            -1.1845464
Walnut Street Elementary School-Meridian Middle               -2.8886971
Walnut Street Elementary School-Spring Lake Middle            -1.2224899
                                                                  p adj
Bell Avenue Elementary School-Aldan Elementary School         0.9582422
Colwyn Elementary School-Aldan Elementary School              0.8184563
Luther Nick Jeralds Middle-Aldan Elementary School            0.9870536
Meridian Middle-Aldan Elementary School                       0.0000000
Spring Lake Middle-Aldan Elementary School                    0.0000000
Walnut Street Elementary School-Aldan Elementary School       0.9279905
Colwyn Elementary School-Bell Avenue Elementary School        0.9984299
Luther Nick Jeralds Middle-Bell Avenue Elementary School      0.9994151
Meridian Middle-Bell Avenue Elementary School                 0.0000000
Spring Lake Middle-Bell Avenue Elementary School              0.0000000
Walnut Street Elementary School-Bell Avenue Elementary School 0.3246248
Luther Nick Jeralds Middle-Colwyn Elementary School           0.9573429
Meridian Middle-Colwyn Elementary School                      0.0000000
Spring Lake Middle-Colwyn Elementary School                   0.0000301
Walnut Street Elementary School-Colwyn Elementary School      0.1957775
Meridian Middle-Luther Nick Jeralds Middle                    0.0000000
Spring Lake Middle-Luther Nick Jeralds Middle                 0.0000000
Walnut Street Elementary School-Luther Nick Jeralds Middle    0.3130775
Spring Lake Middle-Meridian Middle                            0.0000000
Walnut Street Elementary School-Meridian Middle               0.0000000
Walnut Street Elementary School-Spring Lake Middle            0.0000000

Aligns with results shown in q2_model_pooled_by_school.

Pooled - F Test Usage ~ District

One-way ANOVA F-test.

q2_anova_district = aov(num_sessions_week ~ district, data = pooled_vt)
summary(q2_anova_district)
              Df Sum Sq Mean Sq F value Pr(>F)    
district       2    779   389.6   103.2 <2e-16 ***
Residuals   1191   4498     3.8                   
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Difference between districts’ usage is significant.

#Signficant results, Tukey 
TukeyHSD(q2_anova_district)
  Tukey multiple comparisons of means
    95% family-wise confidence level

Fit: aov(formula = num_sessions_week ~ district, data = pooled_vt)

$district
                      diff       lwr        upr p adj
Kent-Cumberland  2.5683816  2.054648  3.0821157     0
Penn-Cumberland -0.7526599 -1.046812 -0.4585079     0
Penn-Kent       -3.3210416 -3.864379 -2.7777039     0

Aligns with results shown in q2_model_pooled_by_district.

Pooled - T Test Usage ~ Male

Welch’s T-test.

t.test(num_sessions_week ~ male, data = pooled_vt)

    Welch Two Sample t-test

data:  num_sessions_week by male
t = -0.4061, df = 1188.4, p-value = 0.6847
alternative hypothesis: true difference in means between group 0 and group 1 is not equal to 0
95 percent confidence interval:
 -0.2883592  0.1894579
sample estimates:
mean in group 0 mean in group 1 
       1.518044        1.567495

Difference between the genders’ usage is not significant.

Pooled - F Test Usage ~ Race

One-way ANOVA F-test.

q2_anova_race = aov(num_sessions_week ~ race_pooled, data = pooled_vt)
summary(q2_anova_race)
              Df Sum Sq Mean Sq F value   Pr(>F)    
race_pooled    6    307   51.24   12.24 2.13e-13 ***
Residuals   1187   4969    4.19                     
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Difference between race subgroups’ usage is significant.

#Signficant results, Tukey 
TukeyHSD(q2_anova_race)
  Tukey multiple comparisons of means
    95% family-wise confidence level

Fit: aov(formula = num_sessions_week ~ race_pooled, data = pooled_vt)

$race_pooled
                                       diff        lwr        upr     p adj
Asian-White                       2.3802400  1.0559579  3.7045221 0.0000027
Black-White                      -0.3983148 -1.0660437  0.2694140 0.5743225
Hispanic-White                    0.4888962 -0.2984631  1.2762556 0.5252589
Multi-racial-White               -0.0231450 -0.9959203  0.9496303 1.0000000
Native American-White            -0.5323342 -2.2671446  1.2024762 0.9716318
Pacific Islander-White            0.9441102 -1.1674338  3.0556541 0.8424277
Black-Asian                      -2.7785548 -3.9605166 -1.5965931 0.0000000
Hispanic-Asian                   -1.8913438 -3.1447817 -0.6379059 0.0001844
Multi-racial-Asian               -2.4033850 -3.7808695 -1.0259005 0.0000062
Native American-Asian            -2.9125742 -4.9027347 -0.9224136 0.0003355
Pacific Islander-Asian           -1.4361299 -3.7620253  0.8897656 0.5322386
Hispanic-Black                    0.8872110  0.3741366  1.4002854 0.0000079
Multi-racial-Black                0.3751698 -0.3926858  1.1430255 0.7782246
Native American-Black            -0.1340193 -1.7627850  1.4947463 0.9999830
Pacific Islander-Black            1.3424250 -0.6828968  3.3677468 0.4425606
Multi-racial-Hispanic            -0.5120412 -1.3859264  0.3618439 0.5955683
Native American-Hispanic         -1.0212304 -2.7025842  0.6601234 0.5524715
Pacific Islander-Hispanic         0.4552139 -1.6126355  2.5230634 0.9950511
Native American-Multi-racial     -0.5091892 -2.2849447  1.2665663 0.9798676
Pacific Islander-Multi-racial     0.9672552 -1.1780556  3.1125660 0.8370758
Pacific Islander-Native American  1.4764443 -1.1053446  4.0582332 0.6236294

Align with results shown in q2_model_pooled_by_race.

Pooled - T Test Usage ~ IEP

Welch’s T-test.

t.test(num_sessions_week ~ iep, data = pooled_vt)

    Welch Two Sample t-test

data:  num_sessions_week by iep
t = 1.695, df = 177.22, p-value = 0.09183
alternative hypothesis: true difference in means between group 0 and group 1 is not equal to 0
95 percent confidence interval:
 -0.05432281  0.71569669
sample estimates:
mean in group 0 mean in group 1 
       1.581187        1.250500

Difference between the groups is not significant.

Pooled - T Test Usage ~ ELL

Welch’s T-test.

t.test(num_sessions_week ~ ell, data = pooled_vt)

    Welch Two Sample t-test

data:  num_sessions_week by ell
t = -4.9749, df = 74.139, p-value = 4.103e-06
alternative hypothesis: true difference in means between group 0 and group 1 is not equal to 0
95 percent confidence interval:
 -2.464042 -1.054747
sample estimates:
mean in group 0 mean in group 1 
       1.437238        3.196633

Difference between the groups is significant.

Pooled - T Test Usage ~ Low-income

Welch’s T-test.

t.test(num_sessions_week ~ low_inc, data = pooled_vt)

    Welch Two Sample t-test

data:  num_sessions_week by low_inc
t = 3.0915, df = 186.92, p-value = 0.002296
alternative hypothesis: true difference in means between group 0 and group 1 is not equal to 0
95 percent confidence interval:
 0.2676821 1.2116721
sample estimates:
mean in group 0 mean in group 1 
       2.000049        1.260372

Difference between the groups is significant.

Pooled - T Test Usage ~ At-risk

Welch’s T-test.

t.test(num_sessions_week ~ at_risk, data = pooled_vt)

    Welch Two Sample t-test

data:  num_sessions_week by at_risk
t = -1.1113, df = 722.66, p-value = 0.2668
alternative hypothesis: true difference in means between group 0 and group 1 is not equal to 0
95 percent confidence interval:
 -0.5009584  0.1388212
sample estimates:
mean in group 0 mean in group 1 
       1.772783        1.953852

Difference between the groups is not significant.

Research Question 3

How is the usability of Live + AI perceived by teachers? How do teachers perceive its impact on student confidence and learning?

Analyze teacher survey outside of R.

Research Question 4

What is the effect of offering LIVE+ AI on student learning compared to students who are not offered the tool? Do improvements in student learning vary meaningfully by student characteristics?

Conduct matched comparisons in e2i Coach.

Research Question 5

Among students who are offered LIVE+ AI, do students who use the tool more show greater gains in learning? PAP: Examine the relationships between student usage and learning outcomes using correlational analyses…Regression models adjusting for school clustering will explore associations, with subgroup analyses conducted as appropriate.

*Repeat for each subject.

POOLED - Regress delta MATH ~ Number of sessions

**Regress change in scores on total sessions (district as factor, but exclude Penn here), cluster by school. Look at “Direct growth” in draft code, was on the right track.

Change in math achievement

Calculate the change in Math scores between MOY and EOY.

pooled_vt$delta_math_z = pooled_vt$eoy_math_z - pooled_vt$moy_math_z

Descriptive statistics

Number of sessions (total). Among students who were offered tutoring, recoded NA to 0.

summary(pooled_vt$total_sessions)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
   0.00    4.00   10.50   19.09   24.00  256.00

MOY Math Z score.

summary(pooled_vt$moy_math_z)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
-2.9800 -0.8400 -0.2300 -0.2049  0.3800  4.7800      67

EOY Math Z Score.

summary(pooled_vt$eoy_math_z)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
-3.2100 -0.7800 -0.1400 -0.1227  0.5500  4.3700      50

Delta Math Z Score.

summary(pooled_vt$delta_math_z)
    Min.  1st Qu.   Median     Mean  3rd Qu.     Max.     NA's 
-4.99000 -0.25000  0.10000  0.09393  0.45000  3.74000      104

—- MAIN —-

Regression model - Overall

# Full model
q5_model_pooled_overall = lm(
  delta_math_z ~
    total_sessions, 
  data = pooled_vt
)
summary(q5_model_pooled_overall) #before clustering by school

Call:
lm(formula = delta_math_z ~ total_sessions, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-5.0848 -0.3395  0.0114  0.3642  3.6410 

Coefficients:
                 Estimate Std. Error t value Pr(>|t|)    
(Intercept)     0.1003572  0.0271974    3.69 0.000235 ***
total_sessions -0.0003294  0.0008244   -0.40 0.689585    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.7238 on 1088 degrees of freedom
  (104 observations deleted due to missingness)
Multiple R-squared:  0.0001467, Adjusted R-squared:  -0.0007723 
F-statistic: 0.1596 on 1 and 1088 DF,  p-value: 0.6896
###################################
##Cluster-robust standard errors by school
q5_pooled_overall_clustered = vcovCL(
  q5_model_pooled_overall,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_overall,
  vcov = q5_pooled_overall_clustered #after clustering
)

t test of coefficients:

                  Estimate  Std. Error t value  Pr(>|t|)    
(Intercept)     0.10035715  0.02990995  3.3553 0.0008201 ***
total_sessions -0.00032938  0.00043148 -0.7634 0.4454003    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Results after clustering are not significant. Sessions logged between MOY and EOY did not seem to change math z scores significantly.

Regression model - District

# Full model
q5_model_pooled_by_district = lm(
  delta_math_z ~
    total_sessions * district, 
  data = pooled_vt
)
summary(q5_model_pooled_by_district) #before clustering by school

Call:
lm(formula = delta_math_z ~ total_sessions * district, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-5.0639 -0.3508  0.0080  0.3732  3.6653 

Coefficients:
                              Estimate Std. Error t value Pr(>|t|)  
(Intercept)                  7.495e-02  3.408e-02   2.199   0.0281 *
total_sessions              -6.109e-05  9.766e-04  -0.063   0.9501  
districtKent                 2.743e-01  1.605e-01   1.709   0.0877 .
districtPenn                 4.919e-02  5.870e-02   0.838   0.4022  
total_sessions:districtKent -2.628e-03  4.090e-03  -0.642   0.5207  
total_sessions:districtPenn -1.903e-03  2.020e-03  -0.942   0.3463  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.723 on 1084 degrees of freedom
  (104 observations deleted due to missingness)
Multiple R-squared:  0.005932,  Adjusted R-squared:  0.001347 
F-statistic: 1.294 on 5 and 1084 DF,  p-value: 0.2641
###################################
##Cluster-robust standard errors by school
q5_pooled_by_district_clustered = vcovCL(
  q5_model_pooled_by_district,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_district,
  vcov = q5_pooled_by_district_clustered #after clustering
)

t test of coefficients:

                               Estimate  Std. Error  t value  Pr(>|t|)    
(Intercept)                  7.4949e-02  1.8189e-02   4.1206 4.067e-05 ***
total_sessions              -6.1091e-05  5.7001e-05  -1.0717   0.28407    
districtKent                 2.7434e-01  1.8189e-02  15.0827 < 2.2e-16 ***
districtPenn                 4.9191e-02  6.0071e-02   0.8189   0.41303    
total_sessions:districtKent -2.6275e-03  5.7001e-05 -46.0965 < 2.2e-16 ***
total_sessions:districtPenn -1.9034e-03  7.3646e-04  -2.5846   0.00988 ** 
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is Cumberland. Results after clustering: Cumberland did not see significant changes in math score.

For Kent students, for each additional session, their change in math z scores will yield 0.003 points lower compared to Cumberland student logging sessions at the same rate, p<0.001 (2.7434e-01 represents the delta math score for Kent students with no sessions logged, p<0.001)

For Penn students, each additional session, their change in math z scores will yield 0.002 points lower compared to Cumberland students logging sessions at the same rate, p<0.01 (4.9191e-02 represent the delta math score for Penn students with no sessions logged, p>0.4).

—- SUBGROUPS —-

Regression model - Male

# Full model
q5_model_pooled_by_male = lm(
  delta_math_z ~
    total_sessions * male, 
  data = pooled_vt
)
summary(q5_model_pooled_by_male) #before clustering by school

Call:
lm(formula = delta_math_z ~ total_sessions * male, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-5.1163 -0.3384  0.0119  0.3650  3.6621 

Coefficients:
                     Estimate Std. Error t value Pr(>|t|)  
(Intercept)          0.082046   0.037285   2.201    0.028 *
total_sessions      -0.001026   0.001111  -0.924    0.356  
male                 0.035753   0.054516   0.656    0.512  
total_sessions:male  0.001523   0.001656   0.920    0.358  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.7234 on 1086 degrees of freedom
  (104 observations deleted due to missingness)
Multiple R-squared:  0.002974,  Adjusted R-squared:  0.0002201 
F-statistic:  1.08 on 3 and 1086 DF,  p-value: 0.3566
###################################
##Cluster-robust standard errors by school
q5_pooled_by_male_clustered = vcovCL(
  q5_model_pooled_by_male,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_male,
  vcov = q5_pooled_by_male_clustered #after clustering
)

t test of coefficients:

                       Estimate  Std. Error t value  Pr(>|t|)    
(Intercept)          0.08204650  0.02078827  3.9468 8.431e-05 ***
total_sessions      -0.00102622  0.00045808 -2.2403   0.02527 *  
male                 0.03575348  0.03644306  0.9811   0.32677    
total_sessions:male  0.00152331  0.00074228  2.0522   0.04039 *  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is female. Results after clustering:

Females with zero sessions logged observed 0.082 point change in math z score, p<0.001. For female students, each additional session yielded -0.001 point change in math z scores, p<0.05. Males with zero sessions logged observed 0.04 points more in delta math z score when compared to females that no sessions logged (not sig).For male students, each additional session yielded an additional 0.001 point change in math z score, p<0.05.

Regression model - Race

# Full model
q5_model_pooled_by_race = lm(
  delta_math_z ~
    total_sessions * race_pooled, 
  data = pooled_vt
)
summary(q5_model_pooled_by_race) #before clustering by school

Call:
lm(formula = delta_math_z ~ total_sessions * race_pooled, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-5.0716 -0.3338  0.0077  0.3659  3.6458 

Coefficients:
                                             Estimate Std. Error t value
(Intercept)                                 0.1871186  0.1129207   1.657
total_sessions                             -0.0016315  0.0039537  -0.413
race_pooledAsian                           -0.1868774  0.2297965  -0.813
race_pooledBlack                           -0.0890280  0.1173557  -0.759
race_pooledHispanic                        -0.0767686  0.1381853  -0.556
race_pooledMulti-racial                    -0.0953836  0.1646861  -0.579
race_pooledNative American                 -0.1114557  0.2542087  -0.438
race_pooledPacific Islander                -0.1961595  0.4152190  -0.472
total_sessions:race_pooledAsian             0.0078799  0.0055217   1.427
total_sessions:race_pooledBlack             0.0006625  0.0040796   0.162
total_sessions:race_pooledHispanic          0.0016806  0.0045236   0.372
total_sessions:race_pooledMulti-racial      0.0001262  0.0053571   0.024
total_sessions:race_pooledNative American   0.0043845  0.0080060   0.548
total_sessions:race_pooledPacific Islander  0.0040330  0.0108018   0.373
                                           Pr(>|t|)  
(Intercept)                                  0.0978 .
total_sessions                               0.6799  
race_pooledAsian                             0.4163  
race_pooledBlack                             0.4482  
race_pooledHispanic                          0.5786  
race_pooledMulti-racial                      0.5626  
race_pooledNative American                   0.6612  
race_pooledPacific Islander                  0.6367  
total_sessions:race_pooledAsian              0.1538  
total_sessions:race_pooledBlack              0.8710  
total_sessions:race_pooledHispanic           0.7103  
total_sessions:race_pooledMulti-racial       0.9812  
total_sessions:race_pooledNative American    0.5840  
total_sessions:race_pooledPacific Islander   0.7090  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.7258 on 1076 degrees of freedom
  (104 observations deleted due to missingness)
Multiple R-squared:  0.005692,  Adjusted R-squared:  -0.006321 
F-statistic: 0.4738 on 13 and 1076 DF,  p-value: 0.9396
###################################
##Cluster-robust standard errors by school
q5_pooled_by_race_clustered = vcovCL(
  q5_model_pooled_by_race,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_race,
  vcov = q5_pooled_by_race_clustered #after clustering
)

t test of coefficients:

                                              Estimate  Std. Error t value
(Intercept)                                 0.18711859  0.07608720  2.4593
total_sessions                             -0.00163145  0.00199841 -0.8164
race_pooledAsian                           -0.18687737  0.33981343 -0.5499
race_pooledBlack                           -0.08902804  0.07162963 -1.2429
race_pooledHispanic                        -0.07676862  0.06723759 -1.1418
race_pooledMulti-racial                    -0.09538362  0.22390762 -0.4260
race_pooledNative American                 -0.11145567  0.11336454 -0.9832
race_pooledPacific Islander                -0.19615951  0.12489011 -1.5707
total_sessions:race_pooledAsian             0.00787995  0.00511880  1.5394
total_sessions:race_pooledBlack             0.00066253  0.00192736  0.3438
total_sessions:race_pooledHispanic          0.00168058  0.00155660  1.0797
total_sessions:race_pooledMulti-racial      0.00012617  0.00409456  0.0308
total_sessions:race_pooledNative American   0.00438445  0.00305398  1.4357
total_sessions:race_pooledPacific Islander  0.00403299  0.00201750  1.9990
                                           Pr(>|t|)  
(Intercept)                                 0.01408 *
total_sessions                              0.41447  
race_pooledAsian                            0.58247  
race_pooledBlack                            0.21418  
race_pooledHispanic                         0.25381  
race_pooledMulti-racial                     0.67020  
race_pooledNative American                  0.32575  
race_pooledPacific Islander                 0.11656  
total_sessions:race_pooledAsian             0.12400  
total_sessions:race_pooledBlack             0.73110  
total_sessions:race_pooledHispanic          0.28054  
total_sessions:race_pooledMulti-racial      0.97542  
total_sessions:race_pooledNative American   0.15139  
total_sessions:race_pooledPacific Islander  0.04586 *
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

White is the reference group. Results after clustering:

Main race groups did not observe significant changes to math z scores from MOY to EOY.

Regarding Pacific Islanders (PIs): The average white students with no sessions logged saw 0.19 point increase in math z score, p<0.05. For white students, each additional session yielded -0.002 point decrease in math z score (not sig).

For PI studnets with no sessions logged, change in math z scores were 0.2 points less than whites with no sessions logged. decrease in math z scores (not sig). For PI students, each additional session yielded an additional 0.004 point change in math z score when compared to white students logging the same number of sessions, p<0.05.

Regression model - IEP

# Full model
q5_model_pooled_by_iep = lm(
  delta_math_z ~
    total_sessions * iep, 
  data = pooled_vt
)
summary(q5_model_pooled_by_iep) #before clustering by school

Call:
lm(formula = delta_math_z ~ total_sessions * iep, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-5.0872 -0.3434  0.0064  0.3615  3.6370 

Coefficients:
                     Estimate Std. Error t value Pr(>|t|)    
(Intercept)         0.1048386  0.0294227   3.563 0.000382 ***
total_sessions     -0.0004479  0.0009024  -0.496 0.619730    
iep                -0.0302648  0.0783942  -0.386 0.699528    
total_sessions:iep  0.0006983  0.0022312   0.313 0.754369    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.7244 on 1086 degrees of freedom
  (104 observations deleted due to missingness)
Multiple R-squared:  0.0002995, Adjusted R-squared:  -0.002462 
F-statistic: 0.1085 on 3 and 1086 DF,  p-value: 0.9552
###################################
##Cluster-robust standard errors by school
q5_pooled_by_iep_clustered = vcovCL(
  q5_model_pooled_by_iep,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_iep,
  vcov = q5_pooled_by_iep_clustered #after clustering
)

t test of coefficients:

                      Estimate  Std. Error t value Pr(>|t|)    
(Intercept)         0.10483863  0.02629755  3.9866 7.15e-05 ***
total_sessions     -0.00044793  0.00034447 -1.3004   0.1938    
iep                -0.03026483  0.07185257 -0.4212   0.6737    
total_sessions:iep  0.00069829  0.00129949  0.5374   0.5911    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is no-IEP. Results after clustering:

No observed significant changes to math z scores from MOY to EOY for IEP or nonIEP students using Live+AI.

Regression model - ELL

# Full model
q5_model_pooled_by_ell = lm(
  delta_math_z ~
    total_sessions * ell, 
  data = pooled_vt
)
summary(q5_model_pooled_by_ell) #before clustering by school

Call:
lm(formula = delta_math_z ~ total_sessions * ell, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-5.0864 -0.3421  0.0111  0.3664  3.6357 

Coefficients:
                     Estimate Std. Error t value Pr(>|t|)    
(Intercept)         0.1067335  0.0278783   3.829 0.000136 ***
total_sessions     -0.0006051  0.0008745  -0.692 0.489104    
ell                -0.1231560  0.1298468  -0.948 0.343100    
total_sessions:ell  0.0031921  0.0028485   1.121 0.262697    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.724 on 1086 degrees of freedom
  (104 observations deleted due to missingness)
Multiple R-squared:  0.001348,  Adjusted R-squared:  -0.001411 
F-statistic: 0.4885 on 3 and 1086 DF,  p-value: 0.6903
###################################
##Cluster-robust standard errors by school
q5_pooled_by_ell_clustered = vcovCL(
  q5_model_pooled_by_ell,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_ell,
  vcov = q5_pooled_by_ell_clustered #after clustering
)

t test of coefficients:

                      Estimate  Std. Error t value  Pr(>|t|)    
(Intercept)         0.10673348  0.02538405  4.2047 2.829e-05 ***
total_sessions     -0.00060515  0.00041772 -1.4487    0.1477    
ell                -0.12315598  0.21497960 -0.5729    0.5668    
total_sessions:ell  0.00319210  0.00274888  1.1612    0.2458    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is no-ELL. Results after clustering:

No observed significant changes to math z scores from MOY to EOY for ELL or nonELL students using Live+AI.

Regression model - Low-Income

# Full model
q5_model_pooled_by_low_inc = lm(
  delta_math_z ~
    total_sessions * low_inc, 
  data = pooled_vt
)
summary(q5_model_pooled_by_low_inc) #before clustering by school

Call:
lm(formula = delta_math_z ~ total_sessions * low_inc, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-1.4652 -0.3397 -0.0321  0.2896  3.1671 

Coefficients:
                         Estimate Std. Error t value Pr(>|t|)    
(Intercept)             0.1963326  0.0580308   3.383 0.000782 ***
total_sessions         -0.0013695  0.0014904  -0.919 0.358666    
low_inc                -0.0728116  0.0710396  -1.025 0.305970    
total_sessions:low_inc  0.0005118  0.0022121   0.231 0.817142    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.515 on 427 degrees of freedom
  (763 observations deleted due to missingness)
Multiple R-squared:  0.004977,  Adjusted R-squared:  -0.002014 
F-statistic: 0.7119 on 3 and 427 DF,  p-value: 0.5453
###################################
##Cluster-robust standard errors by school
q5_pooled_by_low_inc_clustered = vcovCL(
  q5_model_pooled_by_low_inc,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_low_inc,
  vcov = q5_pooled_by_low_inc_clustered #after clustering
)

t test of coefficients:

                          Estimate  Std. Error t value  Pr(>|t|)    
(Intercept)             0.19633255  0.06084849  3.2266  0.001349 ** 
total_sessions         -0.00136950  0.00069307 -1.9760  0.048799 *  
low_inc                -0.07281162  0.01510821 -4.8193 2.005e-06 ***
total_sessions:low_inc  0.00051180  0.00171584  0.2983  0.765635    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group are students who are not low-income.

For non LC students with zero sessions, they observed a 0.2 point change in math z score, p<0.01. For non LC students, each additional session logged yielded a -0.001 point decrease in math z scores , p<0.05. For LC students with zero sessions, their change in math z scores was 0.07 less than non LC students with zero sessions, p<0.001. No significant changes in math z scores for low-income students using Live+AI.

Regression model - At-Risk

# Full model
q5_model_pooled_by_at_risk = lm(
  delta_math_z ~
    total_sessions * at_risk, 
  data = pooled_vt
)
summary(q5_model_pooled_by_at_risk) #before clustering by school

Call:
lm(formula = delta_math_z ~ total_sessions * at_risk, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-5.0249 -0.3360  0.0261  0.3792  3.6058 

Coefficients:
                         Estimate Std. Error t value Pr(>|t|)   
(Intercept)             0.1357022  0.0462215   2.936  0.00343 **
total_sessions         -0.0003874  0.0012116  -0.320  0.74928   
at_risk                -0.1276652  0.0801370  -1.593  0.11157   
total_sessions:at_risk  0.0019666  0.0024655   0.798  0.42534   
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.8148 on 738 degrees of freedom
  (452 observations deleted due to missingness)
Multiple R-squared:  0.003554,  Adjusted R-squared:  -0.0004962 
F-statistic: 0.8775 on 3 and 738 DF,  p-value: 0.4523
###################################
##Cluster-robust standard errors by school
q5_pooled_by_at_risk_clustered = vcovCL(
  q5_model_pooled_by_at_risk,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_at_risk,
  vcov = q5_pooled_by_at_risk_clustered #after clustering
)

t test of coefficients:

                          Estimate  Std. Error  t value  Pr(>|t|)    
(Intercept)             1.3570e-01  1.8517e-02   7.3285 6.122e-13 ***
total_sessions         -3.8736e-04  9.5267e-06 -40.6607 < 2.2e-16 ***
at_risk                -1.2767e-01  7.8747e-02  -1.6212    0.1054    
total_sessions:at_risk  1.9666e-03  1.9838e-03   0.9913    0.3219    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is not at-risk. Results after clustering:

No observed significant changes to math z scores from MOY to EOY for at-risk students using Live+AI.

For non at-risk students, each additional session yielded -0.0003 point change in delta math z score, p<0.001.

POOLED - Regress delta ELA ~ Sessions per week

Change in ELA achievement

Calculate the change in ELA scores between MOY and EOY.

pooled_vt$delta_ela_z = pooled_vt$eoy_ela_z - pooled_vt$moy_ela_z

Descriptive statistics

Number of sessions (total). Among students who were offered tutoring, recoded NA to 0.

summary(pooled_vt$total_sessions)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
   0.00    4.00   10.50   19.09   24.00  256.00

MOY ELA Z score.

summary(pooled_vt$moy_ela_z)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
-2.8100 -0.6400 -0.1000 -0.1518  0.4700  2.2900     105

EOY ELA Z Score.

summary(pooled_vt$eoy_ela_z)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
-3.6700 -0.7525 -0.1100 -0.1661  0.4800  2.2700      82

Delta ELA Z Score.

summary(pooled_vt$delta_ela_z)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
-4.3900 -0.3500 -0.0150 -0.0189  0.3000  2.9500     164

—- MAIN —-

Regression model - Overall

# Full model
q5_model_pooled_overall = lm(
  delta_ela_z ~
    total_sessions, 
  data = pooled_vt
)
summary(q5_model_pooled_overall) #before clustering by school

Call:
lm(formula = delta_ela_z ~ total_sessions, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-4.3292 -0.3307  0.0034  0.3290  2.9063 

Coefficients:
                 Estimate Std. Error t value Pr(>|t|)   
(Intercept)    -0.0660193  0.0256010  -2.579   0.0101 * 
total_sessions  0.0026135  0.0008817   2.964   0.0031 **
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.6441 on 1028 degrees of freedom
  (164 observations deleted due to missingness)
Multiple R-squared:  0.008475,  Adjusted R-squared:  0.007511 
F-statistic: 8.787 on 1 and 1028 DF,  p-value: 0.003104
###################################
##Cluster-robust standard errors by school
q5_pooled_overall_clustered = vcovCL(
  q5_model_pooled_overall,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_overall,
  vcov = q5_pooled_overall_clustered #after clustering
)

t test of coefficients:

                 Estimate Std. Error t value Pr(>|t|)  
(Intercept)    -0.0660193  0.0286532 -2.3041  0.02142 *
total_sessions  0.0026135  0.0014400  1.8149  0.06983 .
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

For students with no sessions logged, they saw a -0.06 point change in delta ELA z scores, p<0.05.For each additional session logged, students on average observed a 0.002 point change increase in delta ELA z scores, p<0.1.

Regression model - District

# Full model
q5_model_pooled_by_district = lm(
  delta_ela_z ~
    total_sessions * district, 
  data = pooled_vt
)
summary(q5_model_pooled_by_district) #before clustering by school

Call:
lm(formula = delta_ela_z ~ total_sessions * district, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-4.3431 -0.3252  0.0065  0.3337  2.6313 

Coefficients:
                              Estimate Std. Error t value Pr(>|t|)  
(Intercept)                 -4.960e-02  3.240e-02  -1.531   0.1260  
total_sessions               1.348e-03  1.138e-03   1.185   0.2362  
districtKent                 3.118e-01  1.367e-01   2.281   0.0227 *
districtPenn                -8.037e-02  5.301e-02  -1.516   0.1298  
total_sessions:districtKent  2.679e-03  3.605e-03   0.743   0.4576  
total_sessions:districtPenn -7.821e-05  1.923e-03  -0.041   0.9676  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.6334 on 1024 degrees of freedom
  (164 observations deleted due to missingness)
Multiple R-squared:  0.04489,   Adjusted R-squared:  0.04023 
F-statistic: 9.626 on 5 and 1024 DF,  p-value: 5.411e-09
###################################
##Cluster-robust standard errors by school
q5_pooled_by_district_clustered = vcovCL(
  q5_model_pooled_by_district,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_district,
  vcov = q5_pooled_by_district_clustered #after clustering
)

t test of coefficients:

                               Estimate  Std. Error t value  Pr(>|t|)    
(Intercept)                 -4.9604e-02  2.1692e-02 -2.2867   0.02241 *  
total_sessions               1.3484e-03  3.3203e-04  4.0611 5.256e-05 ***
districtKent                 3.1182e-01  2.1692e-02 14.3751 < 2.2e-16 ***
districtPenn                -8.0371e-02  3.9587e-02 -2.0302   0.04259 *  
total_sessions:districtKent  2.6787e-03  3.3203e-04  8.0676 1.994e-15 ***
total_sessions:districtPenn -7.8214e-05  9.4977e-04 -0.0824   0.93438    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is Cumberland. Results after clustering:

For Cumberland students, for each additional session, delta ELA z scores would increase by an additional 0.001 points, p<0.001.

For Kent students, for each additional session, delta ELA z scores increased by an additional 0.002 points more when compared to Cumberland students logging sessions at the same rate, p<0.001 (3.1182e-01 represents the delta ELA score for Kent students with no sessions logged, p<0.001).

For Penn students, each additional session, delta ELA z scores decreased by an additional 0.00007 points less when compared to Cumberland students, p>0.9 (-8.0371e-02 represent the delta ELA score for Penn students with no sessions logged, p<0.05).

—- SUBGROUPS —-

Regression model - Male

# Full model
q5_model_pooled_by_male = lm(
  delta_ela_z ~
    total_sessions * male, 
  data = pooled_vt
)
summary(q5_model_pooled_by_male) #before clustering by school

Call:
lm(formula = delta_ela_z ~ total_sessions * male, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-4.3108 -0.3341  0.0018  0.3232  2.9199 

Coefficients:
                      Estimate Std. Error t value Pr(>|t|)  
(Intercept)         -0.0494346  0.0348837  -1.417   0.1567  
total_sessions       0.0025215  0.0011777   2.141   0.0325 *
male                -0.0352742  0.0514335  -0.686   0.4930  
total_sessions:male  0.0002123  0.0017779   0.119   0.9050  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.6445 on 1026 degrees of freedom
  (164 observations deleted due to missingness)
Multiple R-squared:  0.009079,  Adjusted R-squared:  0.006182 
F-statistic: 3.134 on 3 and 1026 DF,  p-value: 0.02484
###################################
##Cluster-robust standard errors by school
q5_pooled_by_male_clustered = vcovCL(
  q5_model_pooled_by_male,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_male,
  vcov = q5_pooled_by_male_clustered #after clustering
)

t test of coefficients:

                       Estimate  Std. Error t value Pr(>|t|)  
(Intercept)         -0.04943458  0.02304449 -2.1452  0.03217 *
total_sessions       0.00252148  0.00110103  2.2901  0.02222 *
male                -0.03527425  0.03344898 -1.0546  0.29187  
total_sessions:male  0.00021226  0.00232704  0.0912  0.92734  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is female. Results after clustering:

For female students, each additional session yielded -0.0025 point change in math z scores, p<0.05. For treatment female students with zero sessions, delta ELA z score equaled -0.04, p<0.05.

For male students, no significant changes to ELA z scores were observed with or without sessions logged, when compared to females.

Regression model - Race

# Full model
q5_model_pooled_by_race = lm(
  delta_ela_z ~
    total_sessions * race_pooled, 
  data = pooled_vt
)
summary(q5_model_pooled_by_race) #before clustering by school

Call:
lm(formula = delta_ela_z ~ total_sessions * race_pooled, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-4.3010 -0.3319  0.0056  0.3236  2.9358 

Coefficients:
                                             Estimate Std. Error t value
(Intercept)                                -6.516e-02  9.761e-02  -0.668
total_sessions                              2.654e-03  2.982e-03   0.890
race_pooledAsian                            2.221e-01  2.053e-01   1.082
race_pooledBlack                           -2.899e-02  1.022e-01  -0.284
race_pooledHispanic                         1.788e-01  1.231e-01   1.453
race_pooledMulti-racial                     6.995e-03  1.506e-01   0.046
race_pooledNative American                  1.451e-02  2.319e-01   0.063
race_pooledPacific Islander                -8.698e-02  3.075e-01  -0.283
total_sessions:race_pooledAsian             1.350e-04  4.542e-03   0.030
total_sessions:race_pooledBlack            -7.509e-05  3.195e-03  -0.024
total_sessions:race_pooledHispanic         -2.413e-03  3.721e-03  -0.649
total_sessions:race_pooledMulti-racial     -5.426e-03  5.418e-03  -1.001
total_sessions:race_pooledNative American  -1.464e-03  6.884e-03  -0.213
total_sessions:race_pooledPacific Islander  1.142e-02  9.141e-03   1.249
                                           Pr(>|t|)
(Intercept)                                   0.505
total_sessions                                0.374
race_pooledAsian                              0.280
race_pooledBlack                              0.777
race_pooledHispanic                           0.147
race_pooledMulti-racial                       0.963
race_pooledNative American                    0.950
race_pooledPacific Islander                   0.777
total_sessions:race_pooledAsian               0.976
total_sessions:race_pooledBlack               0.981
total_sessions:race_pooledHispanic            0.517
total_sessions:race_pooledMulti-racial        0.317
total_sessions:race_pooledNative American     0.832
total_sessions:race_pooledPacific Islander    0.212

Residual standard error: 0.6429 on 1016 degrees of freedom
  (164 observations deleted due to missingness)
Multiple R-squared:  0.02374,   Adjusted R-squared:  0.01124 
F-statistic:   1.9 on 13 and 1016 DF,  p-value: 0.02653
###################################
##Cluster-robust standard errors by school
q5_pooled_by_race_clustered = vcovCL(
  q5_model_pooled_by_race,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_race,
  vcov = q5_pooled_by_race_clustered #after clustering
)

t test of coefficients:

                                              Estimate  Std. Error t value
(Intercept)                                -6.5164e-02  7.7268e-02 -0.8434
total_sessions                              2.6541e-03  2.9430e-03  0.9018
race_pooledAsian                            2.2213e-01  1.2461e-01  1.7825
race_pooledBlack                           -2.8986e-02  6.4484e-02 -0.4495
race_pooledHispanic                         1.7875e-01  6.3013e-02  2.8368
race_pooledMulti-racial                     6.9948e-03  7.8656e-02  0.0889
race_pooledNative American                  1.4507e-02  1.4509e-01  0.1000
race_pooledPacific Islander                -8.6982e-02  1.7328e-01 -0.5020
total_sessions:race_pooledAsian             1.3499e-04  2.6167e-03  0.0516
total_sessions:race_pooledBlack            -7.5089e-05  3.2113e-03 -0.0234
total_sessions:race_pooledHispanic         -2.4133e-03  2.2035e-03 -1.0952
total_sessions:race_pooledMulti-racial     -5.4260e-03  1.8926e-03 -2.8670
total_sessions:race_pooledNative American  -1.4638e-03  3.4913e-03 -0.4193
total_sessions:race_pooledPacific Islander  1.1420e-02  4.5574e-03  2.5058
                                           Pr(>|t|)   
(Intercept)                                0.399230   
total_sessions                             0.367351   
race_pooledAsian                           0.074959 . 
race_pooledBlack                           0.653163   
race_pooledHispanic                        0.004648 **
race_pooledMulti-racial                    0.929156   
race_pooledNative American                 0.920376   
race_pooledPacific Islander                0.615802   
total_sessions:race_pooledAsian            0.958868   
total_sessions:race_pooledBlack            0.981350   
total_sessions:race_pooledHispanic         0.273691   
total_sessions:race_pooledMulti-racial     0.004230 **
total_sessions:race_pooledNative American  0.675113   
total_sessions:race_pooledPacific Islander 0.012372 * 
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

White is the reference group. Significant results after clustering:

Black students with no sessions logged observed a 0.03 point change less than White students with no sessions logged, p<0.1. Hispanic students with no sessions logged observed a 0.2 point change more than White student with no sessions logged, p<0.01. For Multi-racial students, for each additional session logged, ELA z scores saw -0.005 point change, p<0.01. For PI students, for each additional session logged, ELA z scores saw 0.01 point increase more than White students using at the same rate, p<0.05.

Regression model - IEP

# Full model
q5_model_pooled_by_iep = lm(
  delta_ela_z ~
    total_sessions * iep, 
  data = pooled_vt
)
summary(q5_model_pooled_by_iep) #before clustering by school

Call:
lm(formula = delta_ela_z ~ total_sessions * iep, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-4.3282 -0.3306  0.0044  0.3295  2.9052 

Coefficients:
                     Estimate Std. Error t value Pr(>|t|)   
(Intercept)        -0.0670923  0.0275730  -2.433  0.01513 * 
total_sessions      0.0026630  0.0009461   2.815  0.00498 **
iep                 0.0076885  0.0747180   0.103  0.91806   
total_sessions:iep -0.0003789  0.0026302  -0.144  0.88548   
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.6447 on 1026 degrees of freedom
  (164 observations deleted due to missingness)
Multiple R-squared:  0.008496,  Adjusted R-squared:  0.005596 
F-statistic:  2.93 on 3 and 1026 DF,  p-value: 0.03269
###################################
##Cluster-robust standard errors by school
q5_pooled_by_iep_clustered = vcovCL(
  q5_model_pooled_by_iep,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_iep,
  vcov = q5_pooled_by_iep_clustered #after clustering
)

t test of coefficients:

                      Estimate  Std. Error t value Pr(>|t|)  
(Intercept)        -0.06709227  0.02976551 -2.2540   0.0244 *
total_sessions      0.00266301  0.00162511  1.6387   0.1016  
iep                 0.00768854  0.03291664  0.2336   0.8154  
total_sessions:iep -0.00037892  0.00171630 -0.2208   0.8253  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is no-IEP. Results after clustering:

No observed significant changes to ELA z scores from MOY to EOY for IEP or nonIEP students using Live+AI.

Regression model - ELL

# Full model
q5_model_pooled_by_ell = lm(
  delta_ela_z ~
    total_sessions * ell, 
  data = pooled_vt
)
summary(q5_model_pooled_by_ell) #before clustering by school

Call:
lm(formula = delta_ela_z ~ total_sessions * ell, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-4.3228 -0.3275  0.0039  0.3338  2.9063 

Coefficients:
                     Estimate Std. Error t value Pr(>|t|)   
(Intercept)        -0.0727908  0.0263345  -2.764  0.00581 **
total_sessions      0.0027743  0.0009447   2.937  0.00339 **
ell                 0.1354500  0.1184224   1.144  0.25298   
total_sessions:ell -0.0023535  0.0028404  -0.829  0.40753   
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.6443 on 1026 degrees of freedom
  (164 observations deleted due to missingness)
Multiple R-squared:  0.009739,  Adjusted R-squared:  0.006844 
F-statistic: 3.364 on 3 and 1026 DF,  p-value: 0.01818
###################################
##Cluster-robust standard errors by school
q5_pooled_by_ell_clustered = vcovCL(
  q5_model_pooled_by_ell,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_ell,
  vcov = q5_pooled_by_ell_clustered #after clustering
)

t test of coefficients:

                      Estimate  Std. Error t value Pr(>|t|)   
(Intercept)        -0.07279076  0.02519819 -2.8887 0.003949 **
total_sessions      0.00277429  0.00127184  2.1813 0.029386 * 
ell                 0.13544998  0.11680400  1.1596 0.246467   
total_sessions:ell -0.00235354  0.00077728 -3.0279 0.002524 **
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group are non ELLs.

For non ELLs with no sessions logged, they observed a -0.07 point change in ELA z score, p<0.01. For non ELLs, for each additional session, they observed 0.003 point change in ELA z scores, p<0.05. For ELLs with no sessions logged, ELA z scores did not change significantly compared to no ELLs with no sessions logged, p>0.2. For ELLs, for each additional session, they observed a 0.002 point change less compared to non ELLs using at the same rate, p<0.01.

Regression model - Low-Income

# Full model
q5_model_pooled_by_low_inc = lm(
  delta_ela_z ~
    total_sessions * low_inc, 
  data = pooled_vt
)
summary(q5_model_pooled_by_low_inc) #before clustering by school

Call:
lm(formula = delta_ela_z ~ total_sessions * low_inc, data = pooled_vt)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.67264 -0.28749  0.00877  0.29235  2.79314 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)  
(Intercept)            -0.020920   0.061328  -0.341   0.7332  
total_sessions          0.001840   0.001576   1.168   0.2436  
low_inc                -0.126439   0.074977  -1.686   0.0924 .
total_sessions:low_inc  0.005403   0.002336   2.313   0.0212 *
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.5435 on 430 degrees of freedom
  (760 observations deleted due to missingness)
Multiple R-squared:  0.04367,   Adjusted R-squared:  0.037 
F-statistic: 6.545 on 3 and 430 DF,  p-value: 0.0002459
###################################
##Cluster-robust standard errors by school
q5_pooled_by_low_inc_clustered = vcovCL(
  q5_model_pooled_by_low_inc,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_low_inc,
  vcov = q5_pooled_by_low_inc_clustered #after clustering
)

t test of coefficients:

                         Estimate Std. Error t value Pr(>|t|)   
(Intercept)            -0.0209202  0.1047096 -0.1998 0.841737   
total_sessions          0.0018404  0.0021022  0.8754 0.381829   
low_inc                -0.1264393  0.0980183 -1.2900 0.197759   
total_sessions:low_inc  0.0054029  0.0020389  2.6499 0.008348 **
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group are students who are not low-income.

Non LC students’ ELA z scores did not change significantly, regardless of sessions logged. For LC students, each additional session saw a 0.002 point change less compared non LC students using at the same rate, p<0.01.

Regression model - At-Risk

# Full model
q5_model_pooled_by_at_risk = lm(
  delta_ela_z ~
    total_sessions * at_risk, 
  data = pooled_vt
)
summary(q5_model_pooled_by_at_risk) #before clustering by school

Call:
lm(formula = delta_ela_z ~ total_sessions * at_risk, data = pooled_vt)

Residuals:
    Min      1Q  Median      3Q     Max 
-4.3154 -0.3362  0.0134  0.3538  2.7239 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)  
(Intercept)            -0.079278   0.043465  -1.824   0.0686 .
total_sessions          0.002335   0.001382   1.689   0.0916 .
at_risk                 0.116320   0.073798   1.576   0.1154  
total_sessions:at_risk  0.002167   0.002567   0.844   0.3989  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.7029 on 679 degrees of freedom
  (511 observations deleted due to missingness)
Multiple R-squared:  0.02216,   Adjusted R-squared:  0.01784 
F-statistic:  5.13 on 3 and 679 DF,  p-value: 0.00163
###################################
##Cluster-robust standard errors by school
q5_pooled_by_at_risk_clustered = vcovCL(
  q5_model_pooled_by_at_risk,
  cluster = ~ school_num
)

coeftest(
  q5_model_pooled_by_at_risk,
  vcov = q5_pooled_by_at_risk_clustered #after clustering
)

t test of coefficients:

                          Estimate  Std. Error t value  Pr(>|t|)    
(Intercept)            -0.07927836  0.01962216 -4.0402 5.950e-05 ***
total_sessions          0.00233470  0.00050267  4.6446 4.093e-06 ***
at_risk                 0.11631951  0.05171012  2.2495    0.0248 *  
total_sessions:at_risk  0.00216666  0.00403549  0.5369    0.5915    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Reference group is not at-risk. Results after clustering:

For non at-risk students with no sessions, ELA z score changed by -0.08 points, p<0.001. For non at-risk students, each additional session yielded -0.002 point change in math z score, p<0.001. For at-risk student with no sessions, ELA z score changed by 0.11 points more than non at-risk students with no sessions logged, p<0.05. At-risk students did not see significant changes to ELA z scores when using Live+AI.