Supplementary Material
Reliability Generalization of the TTCT: Parameter-Based MASEM - Sensitivity Analyses
03 October 2023
1 Data Preparation
1.1 Installing and
loading relevant R packages
library(pacman)
pacman::p_load(psych,
reshape2,
metaSEM,
metafor,
robumeta,
clubSandwich,
lavaan,
dplyr,
devtools,
semPower)
# devtools::install_github("MathiasHarrer/dmetar")
library(dmetar)1.2 Data input
## Data
## Note: If variables are not included, then code them completely as NA,
## including the diagonals (i.e., not 1 but NA in the diagonal for
## completely missing variables)
Auth_mat <-
'
1,
0.54, 1,
0.11, 0.39, 1,
0.08, 0.18, 0.32, 1,
0.36, 0.25, 0.15, -0.07, 1
'
Barkul_mat_a <-
'
1,
0.95, 1,
0.47, 0.56, 1,
0.86, 0.84, 0.56, 1,
0.47, 0.49, 0.38, 0.6, 1
'
Barkul_mat_b <- '
1,
0.89, 1,
0.31, 0.47, 1,
0.64, 0.62, 0.38, 1,
0.28, 0.24, NA, 0.43, 1'
Berman_mat <-
'
1,
0.39, 1,
0.14, -0.23, 1,
-0.38, 0.14, 0.05, 1,
0.004, -0.29, 0.80, 0.25, 1
'
# Adjusted
Boynton_mat <-
'1
0.576, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA
'
Callans_mat <-
'1
0.78, 1,
0.09, -0.01, 1,
0.26, 0.26, 0.27, 1,
0.51, 0.43, 0.42, 0.38, 1
'
#Carter Control Group Pre-Test
Carter_mat_a <-
'1
0.86, 1,
0.31, 0.28, 1,
0.05, 0.05, 0.54, 1,
0.81, 0.72, 0.49, 0.21, 1
'
#Carter Experimental Group Pre-Test
Carter_mat_b <-
'1,
0.82, 1,
0.62, 0.39, 1,
0.51, 0.40, 0.28, 1,
0.68, 0.60, 0.40, 0.33, 1
'
Chi_mat <-'
1,
0.433, 1,
-0.02, 0.42, 1,
0.18, 0.37, 0.42, 1,
-0.19, 0.4, 0.54, 0.45, 1,
'
Cho_mat_F <- '
1,
0.61, 1,
0.42, 0.43, 1,
0.32, 0.21, 0.58, 1,
0.69, 0.33, 0.63, 0.54, 1'
#Changed to Male correlation matrices from total
Cho_mat_M <- '
1,
0.59, 1,
0.26, 0.02, 1,
0.14, 0.31, 0.36, 1,
0.82, 0.69, 0.25, 0.24, 1,
'
Cockcroft_mat <- '
1,
0.29, 1,
0.42, 0.12, 1,
0.24, 0.65, 0.31, 1,
0.18, 0.31, 0.31, 0.46, 1 '
Conway_mat <-
'1,
0.82, 1,
0.45, 0.47, 1,
0.56, 0.56, 0.68, 1,
0.50, 0.43, 0.49, 0.65, 1
'
Crawford_mat <-
'1
0.94, 1,
0.38, 0.44, 1,
0.37, 0.48, 0.38, 1,
0.67, 0.72, 0.65, 0.40, 1
'
Digranes_mat <-
'1,
0.76, 1,
0.17, 0.45, 1,
0.80, 0.71, 0.09, 1,
0.81, 0.79, 0.37, 0.74, 1
'
## Adjusted
Dufner_mat <-
'1,
0.45, 1,
0.11, 0.09, 1,
-0.07, 0.42, 0.16, 1,
NA, NA, NA, NA, NA
'
Fishkin_mat <-
'1
0.60, 1,
0, -0.03, 1,
0.13, 0.11, 0.11, 1,
0.19, 0.26, 0.01, 0.09, 1
'
Forsyth_mat <-
'1,
0.91, 1,
0.56, 0.49, 1,
0.35, 0.40, 0.50, 1,
0.82, 0.77, 0.62, 0.33, 1'
#8th grade
Garcia_mat <-
'1,
0.626, 1,
0.392, 0.45, 1,
0.435, 0.387, 0.495, 1,
0.416, 0.525, 0.32, 0.567, 1'
Gollmar_mat <-
'1,
0.672, 1,
-0.26, 0.019, 1,
-0.13, 0.241, 0.377, 1,
0.188, 0.217, 0.22, 0.296, 1'
## Adjusted
Hamlen_mat <-'
1,
0.816, 1,
NA, NA, NA,
NA, NA, NA, NA,
0.319, 0.247, NA, NA, 1 '
Hokanson_mat <-
'1,
0.79, 1,
0.14, 0.28, 1,
0.32, 0.44, 0.48, 1,
0.48, 0.45, 0.36, 0.36, 1'
Houtz_mat <- '
1,
0.59, 1,
0.27, 0.16, 1,
0.49, 0.27, 0.22, 1,
0.5, 0.34, 0.46, 0.37, 1 '
Humble_mat <-
'1,
0.83, 1,
0.22, 0.28 1
0.44 0.38 0.06 1
0.34 0.25 0.07 0.37 1'
Ibrahim_mat_1 <- '
1,
0.611, 1,
0.524, 0.12, 1,
0.23, 0.246, 0.267, 1,
0.46, 0.485, 0.155, 0.484, 1 '
Kiehn_mat <-
'1,
0.8, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA
'
#Kim (2006b)
Kim_mat_1 <- '
1,
0.844, 1,
0.351, 0.332, 1,
0.196, 0.209, 0.428, 1,
0.666, 0.563, 0.212, 0.443, 1
'
#Kim et al.(2006)
Kim_mat_2a <- '
1,
0.84, 1,
0.45, 0.49, 1,
0.39, 0.38, 0.49, 1,
0.65, 0.65, 0.34, 0.51, 1 '
Kim_mat_2b <- '
1,
0.79, 1,
0.08, 0.15, 1,
0.14, 0.15, 0.3, 1,
0.47, 0.48, 0.17, 0.25, 1 '
Kim_mat_2c <- '
1,
0.86, 1,
0.31, 0.32, 1,
0.22, 0.25, 0.41, 1,
0.66, 0.57, 0.2, 0.4, 1
'
#Kim et al. 2016
Kim_mat_3_a <- '
1,
0.46, 1,
-0.28, 0.24, 1,
0.1, 0.4, 0.51, 1,
-0.28, 0.12, 0.6, 0.66, 1'
Kim_mat_3_b <- '
1,
0.4, 1,
-0.27, 0.23, 1,
0.08, 0.34, 0.5, 1,
-0.22, 0.17, 0.64, 0.72, 1
'
Lew_mat <- '
1,
0.46, 1,
-0.15, 0.04, 1,
0.14, 0.36, 0.24, 1,
NA, NA, NA, NA, NA '
## Adjusted
Miranda_mat <-
'1,
0.49, 1,
0.02, 0.11, 1,
-0.25, -0.30, -0.31, 1,
0.41, 0.20, 0.14, -0.29, 1'
## Adjusted
#1st grade
Morrison_mat_a <-
'1,
0.36, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA'
## Adjusted
#3rd grade
Morrison_mat_b <-
'1,
0.28, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA'
## Adjusted
#5th grade
Morrison_mat_c <-
'1,
0.42, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA'
Nguyen_mat <-
'1,
0.75, 1,
0.11, 0.09, 1,
0.17, 0.10, 0.15, 1,
0.38, 0.34, 0.13, 0.03, 1'
Zbarskaya_mat <-
'1,
0.614, 1,
0.008, 0.135, 1,
0.168, 0.327, 0.504, 1,
0.189, 0.153, 0.225, 0.442, 1'
#Children's house
Rose_mat_a <-
'1,
0.81, 1,
0.07, 0.27, 1,
0.66, 0.62, 0.57, 1,
-0.05, -0.06, -0.19, -0.12, 1'
#Montessori
Rose_mat_b <-
'1,
0.67, 1,
0.05, 0.13, 1,
0.56, 0.16, -0.30, 1,
0.34, 0.12, 0.43, 0.19, 1'
Roskos_mat_Y <-'
1,
0.26, 1,
0.15, 0.08, 1,
0.37, 0.47, 0.2, 1,
0.53, 0.31, 0.47, 0.37, 1 '
Roskos_mat_O <- '
1,
0.16, 1,
-0.06, -0.02, 1,
0.06, 0.25, 0.04, 1,
0.14, 0.29, 0.36, 0.47, 1
'
Samuels_mat <-
'1,
0.75, 1,
-0.11, 0.02, 1,
0.10, 0.24, 0.48, 1,
-0.16, -0.06, 0.07, 0.24, 1'
Shore_mat <-
'1,
0.26, 1,
-0.01, 0.33, 1,
-0.12, 0.70, 0.30, 1,
-0.29, 0.12, 0.54, 0.34, 1'
Stephens_mat <-
'1,
0.86, 1,
0.17, 0.16, 1,
0.33, 0.30, 0.12, 1,
0.65, 0.60, 0.30, 0.31, 1'
#Control Pre-Test
Storer_mat_a <-
'1,
0.77, 1,
0.11, 0.29, 1,
0.28, 0.19, 0.35, 1,
0.33, 0.49, -0.02, -0.14, 1'
#Treatment Pre-Test
Storer_mat_b <-
'1,
0.73, 1,
0.23, 0.20, 1,
0.43, 0.55, 0.29, 1,
0.57, 0.48, 0.40, 0.25, 1'
Tannehill_mat <-
'1,
0.84, 1,
0.35, 0.41, 1,
0.28, 0.36, 0.38, 1,
0.35, 0.33, 0.39, 0.41, 1'
Tisone_mat <-
'1,
0.68, 1,
-0.03, -0.12, 1,
0.15, 0.26, 0.30, 1,
0.13, 0.35, 0.11, 0.53, 1'
Trigani_mat <-
'1,
0.72, 1,
0.28, 0.24, 1,
0.17, 0.26, 0.37, 1,
0.69, 0.47, 0.52, 0.16, 1'
Voss_mat <-
'1,
0.71, 1,
0.43, 0.51, 1,
0.38, 0.44, 0.53, 1,
0.65, 0.79, 0.59, 0.42, 1'
Warne_mat <-
'1,
0.74, 1,
0.40, 0.37, 1,
0.35, 0.37, 0.43, 1,
0.63, 0.50, 0.50, 0.26, 1'
Watson_mat <-
'1,
0.64, 1,
NA, NA, NA,
0.41, 0.40, NA, 1,
NA, NA, NA, NA, NA'
Yoon_mat <-
'1,
0.73, 1,
-0.16, 0.08, 1,
0.08, 0.29, 0.47, 1,
0.09, 0.28, 0.53, 0.66, 1'
Zhang_mat <-
'1,
0.78, 1,
0.09, 0.15, 1,
0.25, 0.24, 0.33, 1,
0.83, 0.70, 0.06, 0.21, 1'
## New matrices after the update in 2023
Acar_mat <-
'1,
0.65, 1,
0.188, 0.274, 1,
0.338, 0.409, 0.363, 1,
0.529, 0.477, 0.413, 0.305, 1'
Gao_mat <-
'1,
0.31, 1,
0.28, 0.34, 1,
0.39, 0.41, 0.25, 1,
0.28, 0.17, 0.26, 0.31, 1'
Liu_mat <-
'1,
0.94, 1,
0.32, 0.32, 1,
0.49, 0.48, 0.66, 1,
0.41, 0.39, 0.62, .67, 1'
Wan_mat <-
'1,
0.27, 1,
0.24, 0.35, 1,
0.17, 0.19, 0.27, 1,
0.19, 0.14, 0.12, 0.35, 1'
Balcia_mat <-
'1,
0.292, 1,
0, 0.221, 1,
0.176, 0.148, 0.304, 1,
-0.117, 0.154, 0.035, 0.047, 1'
Balcib_mat <-
'1,
0.43, 1,
0.116, 0.166, 1,
0.267, 0.225, 0.363, 1,
-0.19, 0.016, -0.128, -0.107, 1'
Acaretal_mat <-
'1,
0.763, 1,
0.089, 0.21, 1,
0.294, 0.367, 0.469, 1,
0.567, 0.54, 0.277, 0.318, 1'
#This names columns and rows for correlation matrices
Auth_mat1 <-char2num(Auth_mat)
Auth <- getCov(Auth_mat1, diag=T)
colnames(Auth) <- c("Fl","Or", "El","Ab","Res")
rownames(Auth) <- c("Fl","Or", "El","Ab","Res")
#Barkul
Barkul_mat1 <-char2num(Barkul_mat_a)
Barkul <- getCov(Barkul_mat1, diag=T)
colnames(Barkul) <- c("Fl","Or", "El","Ab","Res")
rownames(Barkul) <- c("Fl","Or", "El","Ab","Res")
Barkul_mat2 <-char2num(Barkul_mat_b)
Barkul2 <- getCov(Barkul_mat2, diag=T)
colnames(Barkul2) <- c("Fl","Or", "El","Ab","Res")
rownames(Barkul2) <- c("Fl","Or", "El","Ab","Res")
Berman_mat1 <-char2num(Berman_mat)
Berman <- getCov(Berman_mat1, diag=T)
colnames(Berman) <- c("Fl","Or", "El","Ab","Res")
rownames(Berman) <- c("Fl","Or", "El","Ab","Res")
Boynton_mat1 <-char2num(Boynton_mat)
Boynton <- getCov(Boynton_mat1, diag=T)
colnames(Boynton) <- c("Fl","Or", "El","Ab","Res")
rownames(Boynton) <- c("Fl","Or", "El","Ab","Res")
Callans_mat1 <-char2num(Callans_mat)
Callans <- getCov(Callans_mat1, diag=T)
colnames(Callans) <- c("Fl","Or", "El","Ab","Res")
rownames(Callans) <- c("Fl","Or", "El","Ab","Res")
Carter_mat1 <-char2num(Carter_mat_a)
Carter_a <- getCov(Carter_mat1, diag=T)
colnames(Carter_a) <- c("Fl","Or", "El","Ab","Res")
rownames(Carter_a) <- c("Fl","Or", "El","Ab","Res")
Carter_mat2 <-char2num(Carter_mat_b)
Carter_b <- getCov(Carter_mat2, diag=T)
colnames(Carter_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Carter_b) <- c("Fl","Or", "El","Ab","Res")
Chi_mat1 <-char2num(Chi_mat)
Chi <- getCov(Chi_mat1, diag=T)
colnames(Chi) <- c("Fl","Or", "Ab","El","Res")
rownames(Chi) <- c("Fl","Or", "Ab","El","Res")
Cho_mat1_F <-char2num(Cho_mat_F)
Cho_F <- getCov(Cho_mat1_F, diag=T)
colnames(Cho_F) <- c("Fl","Or", "Ab","El","Res")
rownames(Cho_F) <- c("Fl","Or", "Ab","El","Res")
Cho_mat1_M <-char2num(Cho_mat_M)
Cho_M <- getCov(Cho_mat1_M, diag=T)
colnames(Cho_M) <- c("Fl","Or", "Ab","El","Res")
rownames(Cho_M) <- c("Fl","Or", "Ab","El","Res")
Cockcroft_mat1 <-char2num(Cockcroft_mat)
Cockcroft <- getCov(Cockcroft_mat1, diag=T)
colnames(Cockcroft) <- c("Fl","Or", "Ab","El","Res")
rownames(Cockcroft) <- c("Fl","Or", "Ab","El","Res")
Conway_mat1 <-char2num(Conway_mat)
Conway <- getCov(Conway_mat1, diag=T)
colnames(Conway) <- c("Fl","Or", "El","Ab","Res")
rownames(Conway) <- c("Fl","Or", "El","Ab","Res")
Crawford_mat1 <-char2num(Crawford_mat)
Crawford <- getCov(Crawford_mat1, diag=T)
colnames(Crawford) <- c("Fl","Or", "El","Ab","Res")
rownames(Crawford) <- c("Fl","Or", "El","Ab","Res")
Digranes_mat1 <-char2num(Digranes_mat)
Digranes <- getCov(Digranes_mat1, diag=T)
colnames(Digranes) <- c("Fl","Or", "El","Ab","Res")
rownames(Digranes) <- c("Fl","Or", "El","Ab","Res")
Dufner_mat1 <-char2num(Dufner_mat)
Dufner <- getCov(Dufner_mat1, diag=T)
colnames(Dufner) <- c("Fl","Or", "El","Ab","Res")
rownames(Dufner) <- c("Fl","Or", "El","Ab","Res")
Fishkin_mat1 <-char2num(Fishkin_mat)
Fishkin <- getCov(Fishkin_mat1, diag=T)
colnames(Fishkin) <- c("Fl","Or", "El","Ab","Res")
rownames(Fishkin) <- c("Fl","Or", "El","Ab","Res")
Forsyth_mat1 <-char2num(Forsyth_mat)
Forsyth <- getCov(Forsyth_mat1, diag=T)
colnames(Forsyth) <- c("Fl","Or", "El","Ab","Res")
rownames(Forsyth) <- c("Fl","Or", "El","Ab","Res")
Garcia_mat1 <-char2num(Garcia_mat)
Garcia <- getCov(Garcia_mat1, diag=T)
colnames(Garcia) <- c("Fl","Or", "El","Ab","Res")
rownames(Garcia) <- c("Fl","Or", "El","Ab","Res")
Gollmar_mat1 <-char2num(Gollmar_mat)
Gollmar <- getCov(Gollmar_mat1, diag=T)
colnames(Gollmar) <- c("Fl","Or", "El","Ab","Res")
rownames(Gollmar) <- c("Fl","Or", "El","Ab","Res")
Hamlen_mat1 <-char2num(Hamlen_mat)
Hamlen<- getCov(Hamlen_mat1, diag=T)
colnames(Hamlen) <- c("Fl","Or", "Ab","El","Res")
rownames(Hamlen) <- c("Fl","Or", "Ab","El","Res")
Hokanson_mat1 <-char2num(Hokanson_mat)
Hokanson <- getCov(Hokanson_mat1, diag=T)
colnames(Hokanson) <- c("Fl","Or", "El","Ab","Res")
rownames(Hokanson) <- c("Fl","Or", "El","Ab","Res")
Houtz_mat1 <-char2num(Houtz_mat)
Houtz<- getCov(Houtz_mat1, diag=T)
colnames(Houtz) <- c("Fl","Or", "Ab","El","Res")
rownames(Houtz) <- c("Fl","Or", "Ab","El","Res")
Humble_mat1 <-char2num(Humble_mat)
Humble <- getCov(Humble_mat1, diag=T)
colnames(Humble) <- c("Fl","Or", "El","Ab","Res")
rownames(Humble) <- c("Fl","Or", "El","Ab","Res")
Ibrahim_mat1 <-char2num(Ibrahim_mat_1)
Ibrahim<- getCov(Ibrahim_mat1, diag=T)
colnames(Ibrahim) <- c("Fl","Or", "Ab","El","Res")
rownames(Ibrahim) <- c("Fl","Or", "Ab","El","Res")
Kiehn_mat1 <-char2num(Kiehn_mat)
Kiehn <- getCov(Kiehn_mat1, diag=T)
colnames(Kiehn) <- c("Fl","Or", "El","Ab","Res")
rownames(Kiehn) <- c("Fl","Or", "El","Ab","Res")
Kim_1_mat1 <-char2num(Kim_mat_1)
Kim_1<- getCov(Kim_1_mat1, diag=T)
colnames(Kim_1) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_1) <- c("Fl","Or", "Ab","El","Res")
Kim_2a_mat1 <-char2num(Kim_mat_2a)
Kim_2a<- getCov(Kim_2a_mat1, diag=T)
colnames(Kim_2a) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_2a) <- c("Fl","Or", "Ab","El","Res")
Kim_2b_mat1 <-char2num(Kim_mat_2b)
Kim_2b<- getCov(Kim_2b_mat1, diag=T)
colnames(Kim_2b) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_2b) <- c("Fl","Or", "Ab","El","Res")
Kim_2c_mat1 <-char2num(Kim_mat_2c)
Kim_2c<- getCov(Kim_2c_mat1, diag=T)
colnames(Kim_2c) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_2c) <- c("Fl","Or", "Ab","El","Res")
Kim_3_mat1 <-char2num(Kim_mat_3_a)
Kim_3a <- getCov(Kim_3_mat1, diag=T)
colnames(Kim_3a) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_3a) <- c("Fl","Or", "Ab","El","Res")
Kim_3_mat2 <-char2num(Kim_mat_3_b)
Kim_3b <- getCov(Kim_3_mat2, diag=T)
colnames(Kim_3b) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_3b) <- c("Fl","Or", "Ab","El","Res")
Lew_mat1 <-char2num(Lew_mat)
Lew<- getCov(Lew_mat1, diag=T)
colnames(Lew) <- c("Fl","Or", "Ab","El","Res")
rownames(Lew) <- c("Fl","Or", "Ab","El","Res")
Miranda_mat1 <-char2num(Miranda_mat)
Miranda <- getCov(Miranda_mat1, diag=T)
colnames(Miranda) <- c("Fl","Or", "El","Ab","Res")
rownames(Miranda) <- c("Fl","Or", "El","Ab","Res")
Morrison_mat1 <-char2num(Morrison_mat_a)
Morrison_a <- getCov(Morrison_mat1, diag=T)
colnames(Morrison_a) <- c("Fl","Or", "El","Ab","Res")
rownames(Morrison_a) <- c("Fl","Or", "El","Ab","Res")
Morrison_mat2 <-char2num(Morrison_mat_b)
Morrison_b <- getCov(Morrison_mat2, diag=T)
colnames(Morrison_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Morrison_b) <- c("Fl","Or", "El","Ab","Res")
Morrison_mat3 <-char2num(Morrison_mat_c)
Morrison_c <- getCov(Morrison_mat3, diag=T)
colnames(Morrison_c) <- c("Fl","Or", "El","Ab","Res")
rownames(Morrison_c) <- c("Fl","Or", "El","Ab","Res")
Nguyen_mat1 <-char2num(Nguyen_mat)
Nguyen <- getCov(Nguyen_mat1, diag=T)
colnames(Nguyen) <- c("Fl","Or", "El","Ab","Res")
rownames(Nguyen) <- c("Fl","Or", "El","Ab","Res")
Zbarskaya_mat1 <-char2num(Zbarskaya_mat)
Zbarskaya <- getCov(Zbarskaya_mat1, diag=T)
colnames(Zbarskaya) <- c("Fl","Or", "El","Ab","Res")
rownames(Zbarskaya) <- c("Fl","Or", "El","Ab","Res")
#Children's Place
Rose_mat1 <- char2num(Rose_mat_a)
Rose <- getCov(Rose_mat1, diag=T)
colnames(Rose) <- c("Fl","Or", "El","Ab","Res")
rownames(Rose) <- c("Fl","Or", "El","Ab","Res")
#Montessori
Rose_mat2 <- char2num(Rose_mat_b)
Rose_b <- getCov(Rose_mat2, diag=T)
colnames(Rose_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Rose_b) <- c("Fl","Or", "El","Ab","Res")
Roskos_mat1_Y <-char2num(Roskos_mat_Y)
Roskos_Y<- getCov(Roskos_mat1_Y, diag=T)
colnames(Roskos_Y) <- c("Fl","Or", "Ab","El","Res")
rownames(Roskos_Y) <- c("Fl","Or", "Ab","El","Res")
Roskos_mat1_O <-char2num(Roskos_mat_O)
Roskos_O<- getCov(Roskos_mat1_O, diag=T)
colnames(Roskos_O) <- c("Fl","Or", "Ab","El","Res")
rownames(Roskos_O) <- c("Fl","Or", "Ab","El","Res")
Samuels_mat1 <-char2num(Samuels_mat)
Samuels <- getCov(Samuels_mat1, diag=T)
colnames(Samuels) <- c("Fl","Or", "El","Ab","Res")
rownames(Samuels) <- c("Fl","Or", "El","Ab","Res")
Shore_mat1 <-char2num(Shore_mat)
Shore <- getCov(Shore_mat1, diag=T)
colnames(Shore) <- c("Fl","Or", "El","Ab","Res")
rownames(Shore) <- c("Fl","Or", "El","Ab","Res")
Stephens_mat1 <-char2num(Stephens_mat)
Stephens <- getCov(Stephens_mat1, diag=T)
colnames(Stephens) <- c("Fl","Or", "El","Ab","Res")
rownames(Stephens) <- c("Fl","Or", "El","Ab","Res")
Storer_mat1 <- char2num(Storer_mat_a)
Storer_a <- getCov(Rose_mat1, diag=T)
colnames(Storer_a) <- c("Fl","Or", "El","Ab","Res")
rownames(Storer_a) <- c("Fl","Or", "El","Ab","Res")
Storer_mat2 <- char2num(Storer_mat_b)
Storer_b <- getCov(Storer_mat2, diag=T)
colnames(Storer_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Storer_b) <- c("Fl","Or", "El","Ab","Res")
Tannehill_mat1 <-char2num(Tannehill_mat)
Tannehill <- getCov(Tannehill_mat1, diag=T)
colnames(Tannehill) <- c("Fl","Or", "El","Ab","Res")
rownames(Tannehill) <- c("Fl","Or", "El","Ab","Res")
Tisone_mat1 <-char2num(Tisone_mat)
Tisone <- getCov(Tisone_mat1, diag=T)
colnames(Tisone) <- c("Fl","Or", "El","Ab","Res")
rownames(Tisone) <- c("Fl","Or", "El","Ab","Res")
Trigani_mat1 <-char2num(Trigani_mat)
Trigani <- getCov(Trigani_mat1, diag=T)
colnames(Trigani) <- c("Fl","Or", "El","Ab","Res")
rownames(Trigani) <- c("Fl","Or", "El","Ab","Res")
Voss_mat1 <-char2num(Voss_mat)
Voss <- getCov(Voss_mat1, diag=T)
colnames(Voss) <- c("Fl","Or", "El","Ab","Res")
rownames(Voss) <- c("Fl","Or", "El","Ab","Res")
Warne_mat1 <-char2num(Warne_mat)
Warne <- getCov(Warne_mat1, diag=T)
colnames(Warne) <- c("Fl","Or", "El","Ab","Res")
rownames(Warne) <- c("Fl","Or", "El","Ab","Res")
Watson_mat1 <-char2num(Watson_mat)
Watson <- getCov(Watson_mat1, diag=T)
colnames(Watson) <- c("Fl","Or", "El","Ab","Res")
rownames(Watson) <- c("Fl","Or", "El","Ab","Res")
Yoon_mat1 <-char2num(Yoon_mat)
Yoon <- getCov(Yoon_mat1, diag=T)
colnames(Yoon) <- c("Fl","Or", "El","Ab","Res")
rownames(Yoon) <- c("Fl","Or", "El","Ab","Res")
Zhang_mat1 <-char2num(Zhang_mat)
Zhang <- getCov(Zhang_mat1, diag=T)
colnames(Zhang) <- c("Fl","Or", "El","Ab","Res")
rownames(Zhang) <- c("Fl","Or", "El","Ab","Res")
## New studies after the update in 2023
Acar_mat1 <-char2num(Acar_mat)
Acar <- getCov(Acar_mat1, diag=T)
colnames(Acar) <- c("Fl","Or", "El","Ab","Res")
rownames(Acar) <- c("Fl","Or", "El","Ab","Res")
Acaretal_mat1 <-char2num(Acaretal_mat)
Acaretal <- getCov(Acaretal_mat1, diag=T)
colnames(Acaretal) <- c("Fl","Or", "El","Ab","Res")
rownames(Acaretal) <- c("Fl","Or", "El","Ab","Res")
Balcia_mat1 <-char2num(Balcia_mat)
Balcia <- getCov(Balcia_mat1, diag=T)
colnames(Balcia) <- c("Fl","Or", "El","Ab","Res")
rownames(Balcia) <- c("Fl","Or", "El","Ab","Res")
Balcib_mat1 <-char2num(Balcib_mat)
Balcib <- getCov(Balcib_mat1, diag=T)
colnames(Balcib) <- c("Fl","Or", "El","Ab","Res")
rownames(Balcib) <- c("Fl","Or", "El","Ab","Res")
Gao_mat1 <-char2num(Gao_mat)
Gao <- getCov(Gao_mat1, diag=T)
colnames(Gao) <- c("Fl","Or", "El","Ab","Res")
rownames(Gao) <- c("Fl","Or", "El","Ab","Res")
Liu_mat1 <-char2num(Liu_mat)
Liu <- getCov(Liu_mat1, diag=T)
colnames(Liu) <- c("Fl","Or", "El","Ab","Res")
rownames(Liu) <- c("Fl","Or", "El","Ab","Res")
Wan_mat1 <-char2num(Wan_mat)
Wan <- getCov(Wan_mat1, diag=T)
colnames(Wan) <- c("Fl","Or", "El","Ab","Res")
rownames(Wan) <- c("Fl","Or", "El","Ab","Res")
data <- list(Auth, Barkul, Barkul2, Berman, Boynton, Callans,
Carter_a, Carter_b, Chi, Cho_F, Cho_M, Cockcroft,
Conway, Crawford,
Digranes, Dufner, Fishkin, Forsyth, Garcia, Gollmar,
Hamlen, Hokanson, Houtz, Humble, Ibrahim,
Kiehn, Kim_1, Kim_2a, Kim_2b, Kim_2c, Kim_3a, Kim_3b,
Lew, Miranda, Morrison_a, Morrison_b, Morrison_c,
Nguyen,Zbarskaya, Rose, Rose_b,
Roskos_Y, Roskos_O, Samuels, Shore, Stephens,
Storer_a, Storer_b, Tannehill, Tisone, Trigani,
Voss, Warne, Watson, Yoon, Zhang,
Acar, Acaretal, Balcia, Balcib, Gao, Liu, Wan)
n <- c(30, 599, 147, 13, 62, 60, 24, 24, 203, 24, 35, 36, 25, 21, 17, 98, 116, 45, 95, 128, 118, 1758,
42, 125, 99, 89, 500, 1000, 1000, 1000, 125, 137, 135, 12, 184, 122, 121, 187, 125, 19, 12,
39, 31, 51, 18, 84, 43, 46, 199, 24, 107, 120, 432, 6, 163, 1067,
477, 319, 1047, 95, 264, 105, 375)
names <- c("Auth" , "Barkul" , "Barkul2" , "Berman" , "Boynton" , "Callans" ,
"Carter_a" , "Carter_b" , "Chi" , "Cho_F" , "Cho_M" , "Cockcroft" ,
"Conway" , "Crawford",
"Digranes" , "Dufner" , "Fishkin" , "Forsyth" , "Garcia" , "Gollmar" ,
"Hamlen" , "Hokanson" , "Houtz" , "Humble" , "Ibrahim",
"Kiehn" , "Kim_1" , "Kim_2a" , "Kim_2b" , "Kim_2c" , "Kim_3a" , "Kim_3b" ,
"Lew" , "Miranda" , "Morrison_a" , "Morrison_b" , "Morrison_c" ,
"Nguyen" , "Zbarskaya" , "Rose" , "Rose_b" ,
"Roskos_Y" , "Roskos_O" , "Samuels" , "Shore" , "Stephens" ,
"Storer_a" , "Storer_b" , "Tannehill" , "Tisone" , "Trigani" ,
"Voss" , "Warne" , "Watson" , "Yoon" , "Zhang",
"Acar", "Acaretal", "Balcia", "Balcib", "Gao", "Liu", "Wan")
study.names <- c("Auth" , "Barkul" , "Barkul2" , "Berman" , "Boynton" , "Callans" ,
"Carter" , "Carter" , "Chi" , "Cho" , "Cho" , "Cockcroft" ,
"Conway" , "Crawford",
"Digranes" , "Dufner" , "Fishkin" , "Forsyth" , "Garcia" , "Gollmar" ,
"Hamlen" , "Hokanson" , "Houtz" , "Humble" , "Ibrahim",
"Kiehn" , "Kim_1" , "Kim_2" , "Kim_2" , "Kim_2" , "Kim_3" , "Kim_3" ,
"Lew" , "Miranda" , "Morrison" , "Morrison" , "Morrison" ,
"Nguyen" , "Zbarskaya" , "Rose" , "Rose" ,
"Roskos" , "Roskos" , "Samuels" , "Shore" , "Stephens" ,
"Storer" , "Storer" , "Tannehill" , "Tisone" , "Trigani" ,
"Voss" , "Warne" , "Watson" , "Yoon" , "Zhang",
"Acar", "Acaretal", "Balcia", "Balcib", "Gao", "Liu", "Wan")
names(data) <- study.names
## Possible moderators
## Age group
## Code: 1 = Adults, 0 = Kindergarten up to High/Middle school
adults <- c(1,1,1,1,1,1,0,0,0,1,1,0,1,0,0,0,
0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,
0,1,0,0,0,1,0,0,1,1,1,0,0,0,0,0,
0,1,0,1,1,0,1,0,
1, 0, 0, 0, 1, 0, 1)
## Evidence against discriminant validity
## Coding: 1 = evidence present, 0 = evidence not present/reported
validity <- c(0,0,1,0,1,0,1,1,0,1,1,1,0,0,1,0,1,0,
0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,
0,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,
0,0,
1, 1, 0, 1, 0, 0, 0)
## Test forms
## Subset the data
formsAB <- c(NA,"A","A","A","Both","A","A","A","A",
NA,NA,"B","A","A","A","B","A","Both",
"B","A","A","A","B","A","A","A","A","A",
"A","A","A","A",NA,"A","A","A","A","A","A",
"A","B","B","A","A","A","A","A","A","A","A",
"A","A","B","A","A","A",
"A", "A", "Both", "Both", NA, NA, NA)
forms <- as.vector(as.numeric(which(formsAB %in% c(NA, "Both")))) # which gives the positions of codes 1
## Standardized test scores
## Subset the data
scoring <- c(1,0,0,NA,0,1,0,0,0,0,0,0,1,1,0,0,0,
1,1,1,1,0,0,1,1,1,1,0,0,0,1,1,1,0,NA,
NA,NA,1,0,0,1,1,1,0,1,1,1,1,0,1,1,0,
0,1,1,0,
0, 0, 0, 0, NA, NA, NA)
scores <- as.vector(as.numeric(which(scoring %in% NA))) # which gives the positions of codes 1
head(data, 5)## $Auth
## Fl Or El Ab Res
## Fl 1.00 0.54 0.11 0.08 0.36
## Or 0.54 1.00 0.39 0.18 0.25
## El 0.11 0.39 1.00 0.32 0.15
## Ab 0.08 0.18 0.32 1.00 -0.07
## Res 0.36 0.25 0.15 -0.07 1.00
##
## $Barkul
## Fl Or El Ab Res
## Fl 1.00 0.95 0.47 0.86 0.47
## Or 0.95 1.00 0.56 0.84 0.49
## El 0.47 0.56 1.00 0.56 0.38
## Ab 0.86 0.84 0.56 1.00 0.60
## Res 0.47 0.49 0.38 0.60 1.00
##
## $Barkul2
## Fl Or El Ab Res
## Fl 1.00 0.89 0.31 0.64 0.28
## Or 0.89 1.00 0.47 0.62 0.24
## El 0.31 0.47 1.00 0.38 NA
## Ab 0.64 0.62 0.38 1.00 0.43
## Res 0.28 0.24 NA 0.43 1.00
##
## $Berman
## Fl Or El Ab Res
## Fl 1.000 0.39 0.14 -0.38 0.004
## Or 0.390 1.00 -0.23 0.14 -0.290
## El 0.140 -0.23 1.00 0.05 0.800
## Ab -0.380 0.14 0.05 1.00 0.250
## Res 0.004 -0.29 0.80 0.25 1.000
##
## $Boynton
## Fl Or El Ab Res
## Fl 1.000 0.576 NA NA NA
## Or 0.576 1.000 NA NA NA
## El NA NA NA NA NA
## Ab NA NA NA NA NA
## Res NA NA NA NA NA1.3 Data checking and subsetting
## Check for positive definiteness
is.pd(data)## Auth Barkul Barkul2 Berman Boynton Callans Carter Carter
## TRUE TRUE NA TRUE TRUE TRUE TRUE TRUE
## Chi Cho Cho Cockcroft Conway Crawford Digranes Dufner
## TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
## Fishkin Forsyth Garcia Gollmar Hamlen Hokanson Houtz Humble
## TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
## Ibrahim Kiehn Kim_1 Kim_2 Kim_2 Kim_2 Kim_3 Kim_3
## TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
## Lew Miranda Morrison Morrison Morrison Nguyen Zbarskaya Rose
## TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
## Rose Roskos Roskos Samuels Shore Stephens Storer Storer
## TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
## Tannehill Tisone Trigani Voss Warne Watson Yoon Zhang
## TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
## Acar Acaretal Balcia Balcib Gao Liu Wan
## TRUE TRUE TRUE TRUE TRUE TRUE TRUE## Simplify the data structure
TTCT <- data
TTCT.n <- n
# Overview of missing data
pattern.na(TTCT, show.na = TRUE)## Fl Or El Ab Res
## Fl 0 0 7 6 8
## Or 0 0 7 6 8
## El 7 7 7 7 10
## Ab 6 6 7 6 9
## Res 8 8 10 9 8# Overview of complete data
pattern.na(TTCT, show.na = FALSE)## Fl Or El Ab Res
## Fl 63 63 56 57 55
## Or 63 63 56 57 55
## El 56 56 56 56 53
## Ab 57 57 56 57 54
## Res 55 55 53 54 55# Which correlation matrices have missing values?
# Search in the pattern matrices for TRUE (i.e., NAs present)
# Create the relevant objects
x <- "TRUE"
y <- c(rep(NA, length(TTCT)))
for (i in 1:length(TTCT)) {
## Check missingness
y[i] <- x %in% as.matrix(is.na(TTCT[[i]]))
}
y## [1] FALSE FALSE TRUE FALSE TRUE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
## [13] FALSE FALSE FALSE TRUE FALSE FALSE FALSE FALSE TRUE FALSE FALSE FALSE
## [25] FALSE TRUE FALSE FALSE FALSE FALSE FALSE FALSE TRUE FALSE TRUE TRUE
## [37] TRUE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
## [49] FALSE FALSE FALSE FALSE FALSE TRUE FALSE FALSE FALSE FALSE FALSE FALSE
## [61] FALSE FALSE FALSEtable(y)## y
## FALSE TRUE
## 53 10## Identify the correlation matrices with missing values
missexclude <- as.vector(which(y %in% x))
## Exclude the correlation matrices with missing values
TTCT <- TTCT[-missexclude]
TTCT.n <- TTCT.n[-missexclude]
names.studies <- names[-missexclude]
names.studid <- study.names[-missexclude]
mod.adults <- adults[-missexclude]
mod.validity <- validity[-missexclude]
mod.forms <- formsAB[-missexclude]
mod.scores <- scoring[-missexclude]
## Combine everything in an object
TTCT1 <- list(data = TTCT, n = TTCT.n)
TTCT1## $data
## $data$Auth
## Fl Or El Ab Res
## Fl 1.00 0.54 0.11 0.08 0.36
## Or 0.54 1.00 0.39 0.18 0.25
## El 0.11 0.39 1.00 0.32 0.15
## Ab 0.08 0.18 0.32 1.00 -0.07
## Res 0.36 0.25 0.15 -0.07 1.00
##
## $data$Barkul
## Fl Or El Ab Res
## Fl 1.00 0.95 0.47 0.86 0.47
## Or 0.95 1.00 0.56 0.84 0.49
## El 0.47 0.56 1.00 0.56 0.38
## Ab 0.86 0.84 0.56 1.00 0.60
## Res 0.47 0.49 0.38 0.60 1.00
##
## $data$Berman
## Fl Or El Ab Res
## Fl 1.000 0.39 0.14 -0.38 0.004
## Or 0.390 1.00 -0.23 0.14 -0.290
## El 0.140 -0.23 1.00 0.05 0.800
## Ab -0.380 0.14 0.05 1.00 0.250
## Res 0.004 -0.29 0.80 0.25 1.000
##
## $data$Callans
## Fl Or El Ab Res
## Fl 1.00 0.78 0.09 0.26 0.51
## Or 0.78 1.00 -0.01 0.26 0.43
## El 0.09 -0.01 1.00 0.27 0.42
## Ab 0.26 0.26 0.27 1.00 0.38
## Res 0.51 0.43 0.42 0.38 1.00
##
## $data$Carter
## Fl Or El Ab Res
## Fl 1.00 0.86 0.31 0.05 0.81
## Or 0.86 1.00 0.28 0.05 0.72
## El 0.31 0.28 1.00 0.54 0.49
## Ab 0.05 0.05 0.54 1.00 0.21
## Res 0.81 0.72 0.49 0.21 1.00
##
## $data$Carter
## Fl Or El Ab Res
## Fl 1.00 0.82 0.62 0.51 0.68
## Or 0.82 1.00 0.39 0.40 0.60
## El 0.62 0.39 1.00 0.28 0.40
## Ab 0.51 0.40 0.28 1.00 0.33
## Res 0.68 0.60 0.40 0.33 1.00
##
## $data$Chi
## Fl Or Ab El Res
## Fl 1.000 0.433 -0.02 0.18 -0.19
## Or 0.433 1.000 0.42 0.37 0.40
## Ab -0.020 0.420 1.00 0.42 0.54
## El 0.180 0.370 0.42 1.00 0.45
## Res -0.190 0.400 0.54 0.45 1.00
##
## $data$Cho
## Fl Or Ab El Res
## Fl 1.00 0.61 0.42 0.32 0.69
## Or 0.61 1.00 0.43 0.21 0.33
## Ab 0.42 0.43 1.00 0.58 0.63
## El 0.32 0.21 0.58 1.00 0.54
## Res 0.69 0.33 0.63 0.54 1.00
##
## $data$Cho
## Fl Or Ab El Res
## Fl 1.00 0.59 0.26 0.14 0.82
## Or 0.59 1.00 0.02 0.31 0.69
## Ab 0.26 0.02 1.00 0.36 0.25
## El 0.14 0.31 0.36 1.00 0.24
## Res 0.82 0.69 0.25 0.24 1.00
##
## $data$Cockcroft
## Fl Or Ab El Res
## Fl 1.00 0.29 0.42 0.24 0.18
## Or 0.29 1.00 0.12 0.65 0.31
## Ab 0.42 0.12 1.00 0.31 0.31
## El 0.24 0.65 0.31 1.00 0.46
## Res 0.18 0.31 0.31 0.46 1.00
##
## $data$Conway
## Fl Or El Ab Res
## Fl 1.00 0.82 0.45 0.56 0.50
## Or 0.82 1.00 0.47 0.56 0.43
## El 0.45 0.47 1.00 0.68 0.49
## Ab 0.56 0.56 0.68 1.00 0.65
## Res 0.50 0.43 0.49 0.65 1.00
##
## $data$Crawford
## Fl Or El Ab Res
## Fl 1.00 0.94 0.38 0.37 0.67
## Or 0.94 1.00 0.44 0.48 0.72
## El 0.38 0.44 1.00 0.38 0.65
## Ab 0.37 0.48 0.38 1.00 0.40
## Res 0.67 0.72 0.65 0.40 1.00
##
## $data$Digranes
## Fl Or El Ab Res
## Fl 1.00 0.76 0.17 0.80 0.81
## Or 0.76 1.00 0.45 0.71 0.79
## El 0.17 0.45 1.00 0.09 0.37
## Ab 0.80 0.71 0.09 1.00 0.74
## Res 0.81 0.79 0.37 0.74 1.00
##
## $data$Fishkin
## Fl Or El Ab Res
## Fl 1.00 0.60 0.00 0.13 0.19
## Or 0.60 1.00 -0.03 0.11 0.26
## El 0.00 -0.03 1.00 0.11 0.01
## Ab 0.13 0.11 0.11 1.00 0.09
## Res 0.19 0.26 0.01 0.09 1.00
##
## $data$Forsyth
## Fl Or El Ab Res
## Fl 1.00 0.91 0.56 0.35 0.82
## Or 0.91 1.00 0.49 0.40 0.77
## El 0.56 0.49 1.00 0.50 0.62
## Ab 0.35 0.40 0.50 1.00 0.33
## Res 0.82 0.77 0.62 0.33 1.00
##
## $data$Garcia
## Fl Or El Ab Res
## Fl 1.000 0.626 0.392 0.435 0.416
## Or 0.626 1.000 0.450 0.387 0.525
## El 0.392 0.450 1.000 0.495 0.320
## Ab 0.435 0.387 0.495 1.000 0.567
## Res 0.416 0.525 0.320 0.567 1.000
##
## $data$Gollmar
## Fl Or El Ab Res
## Fl 1.000 0.672 -0.260 -0.130 0.188
## Or 0.672 1.000 0.019 0.241 0.217
## El -0.260 0.019 1.000 0.377 0.220
## Ab -0.130 0.241 0.377 1.000 0.296
## Res 0.188 0.217 0.220 0.296 1.000
##
## $data$Hokanson
## Fl Or El Ab Res
## Fl 1.00 0.79 0.14 0.32 0.48
## Or 0.79 1.00 0.28 0.44 0.45
## El 0.14 0.28 1.00 0.48 0.36
## Ab 0.32 0.44 0.48 1.00 0.36
## Res 0.48 0.45 0.36 0.36 1.00
##
## $data$Houtz
## Fl Or Ab El Res
## Fl 1.00 0.59 0.27 0.49 0.50
## Or 0.59 1.00 0.16 0.27 0.34
## Ab 0.27 0.16 1.00 0.22 0.46
## El 0.49 0.27 0.22 1.00 0.37
## Res 0.50 0.34 0.46 0.37 1.00
##
## $data$Humble
## Fl Or El Ab Res
## Fl 1.00 0.83 0.22 0.44 0.34
## Or 0.83 1.00 0.28 0.38 0.25
## El 0.22 0.28 1.00 0.06 0.07
## Ab 0.44 0.38 0.06 1.00 0.37
## Res 0.34 0.25 0.07 0.37 1.00
##
## $data$Ibrahim
## Fl Or Ab El Res
## Fl 1.000 0.611 0.524 0.230 0.460
## Or 0.611 1.000 0.120 0.246 0.485
## Ab 0.524 0.120 1.000 0.267 0.155
## El 0.230 0.246 0.267 1.000 0.484
## Res 0.460 0.485 0.155 0.484 1.000
##
## $data$Kim_1
## Fl Or Ab El Res
## Fl 1.000 0.844 0.351 0.196 0.666
## Or 0.844 1.000 0.332 0.209 0.563
## Ab 0.351 0.332 1.000 0.428 0.212
## El 0.196 0.209 0.428 1.000 0.443
## Res 0.666 0.563 0.212 0.443 1.000
##
## $data$Kim_2
## Fl Or Ab El Res
## Fl 1.00 0.84 0.45 0.39 0.65
## Or 0.84 1.00 0.49 0.38 0.65
## Ab 0.45 0.49 1.00 0.49 0.34
## El 0.39 0.38 0.49 1.00 0.51
## Res 0.65 0.65 0.34 0.51 1.00
##
## $data$Kim_2
## Fl Or Ab El Res
## Fl 1.00 0.79 0.08 0.14 0.47
## Or 0.79 1.00 0.15 0.15 0.48
## Ab 0.08 0.15 1.00 0.30 0.17
## El 0.14 0.15 0.30 1.00 0.25
## Res 0.47 0.48 0.17 0.25 1.00
##
## $data$Kim_2
## Fl Or Ab El Res
## Fl 1.00 0.86 0.31 0.22 0.66
## Or 0.86 1.00 0.32 0.25 0.57
## Ab 0.31 0.32 1.00 0.41 0.20
## El 0.22 0.25 0.41 1.00 0.40
## Res 0.66 0.57 0.20 0.40 1.00
##
## $data$Kim_3
## Fl Or Ab El Res
## Fl 1.00 0.46 -0.28 0.10 -0.28
## Or 0.46 1.00 0.24 0.40 0.12
## Ab -0.28 0.24 1.00 0.51 0.60
## El 0.10 0.40 0.51 1.00 0.66
## Res -0.28 0.12 0.60 0.66 1.00
##
## $data$Kim_3
## Fl Or Ab El Res
## Fl 1.00 0.40 -0.27 0.08 -0.22
## Or 0.40 1.00 0.23 0.34 0.17
## Ab -0.27 0.23 1.00 0.50 0.64
## El 0.08 0.34 0.50 1.00 0.72
## Res -0.22 0.17 0.64 0.72 1.00
##
## $data$Miranda
## Fl Or El Ab Res
## Fl 1.00 0.49 0.02 -0.25 0.41
## Or 0.49 1.00 0.11 -0.30 0.20
## El 0.02 0.11 1.00 -0.31 0.14
## Ab -0.25 -0.30 -0.31 1.00 -0.29
## Res 0.41 0.20 0.14 -0.29 1.00
##
## $data$Nguyen
## Fl Or El Ab Res
## Fl 1.00 0.75 0.11 0.17 0.38
## Or 0.75 1.00 0.09 0.10 0.34
## El 0.11 0.09 1.00 0.15 0.13
## Ab 0.17 0.10 0.15 1.00 0.03
## Res 0.38 0.34 0.13 0.03 1.00
##
## $data$Zbarskaya
## Fl Or El Ab Res
## Fl 1.000 0.614 0.008 0.168 0.189
## Or 0.614 1.000 0.135 0.327 0.153
## El 0.008 0.135 1.000 0.504 0.225
## Ab 0.168 0.327 0.504 1.000 0.442
## Res 0.189 0.153 0.225 0.442 1.000
##
## $data$Rose
## Fl Or El Ab Res
## Fl 1.00 0.81 0.07 0.66 -0.05
## Or 0.81 1.00 0.27 0.62 -0.06
## El 0.07 0.27 1.00 0.57 -0.19
## Ab 0.66 0.62 0.57 1.00 -0.12
## Res -0.05 -0.06 -0.19 -0.12 1.00
##
## $data$Rose
## Fl Or El Ab Res
## Fl 1.00 0.67 0.05 0.56 0.34
## Or 0.67 1.00 0.13 0.16 0.12
## El 0.05 0.13 1.00 -0.30 0.43
## Ab 0.56 0.16 -0.30 1.00 0.19
## Res 0.34 0.12 0.43 0.19 1.00
##
## $data$Roskos
## Fl Or Ab El Res
## Fl 1.00 0.26 0.15 0.37 0.53
## Or 0.26 1.00 0.08 0.47 0.31
## Ab 0.15 0.08 1.00 0.20 0.47
## El 0.37 0.47 0.20 1.00 0.37
## Res 0.53 0.31 0.47 0.37 1.00
##
## $data$Roskos
## Fl Or Ab El Res
## Fl 1.00 0.16 -0.06 0.06 0.14
## Or 0.16 1.00 -0.02 0.25 0.29
## Ab -0.06 -0.02 1.00 0.04 0.36
## El 0.06 0.25 0.04 1.00 0.47
## Res 0.14 0.29 0.36 0.47 1.00
##
## $data$Samuels
## Fl Or El Ab Res
## Fl 1.00 0.75 -0.11 0.10 -0.16
## Or 0.75 1.00 0.02 0.24 -0.06
## El -0.11 0.02 1.00 0.48 0.07
## Ab 0.10 0.24 0.48 1.00 0.24
## Res -0.16 -0.06 0.07 0.24 1.00
##
## $data$Shore
## Fl Or El Ab Res
## Fl 1.00 0.26 -0.01 -0.12 -0.29
## Or 0.26 1.00 0.33 0.70 0.12
## El -0.01 0.33 1.00 0.30 0.54
## Ab -0.12 0.70 0.30 1.00 0.34
## Res -0.29 0.12 0.54 0.34 1.00
##
## $data$Stephens
## Fl Or El Ab Res
## Fl 1.00 0.86 0.17 0.33 0.65
## Or 0.86 1.00 0.16 0.30 0.60
## El 0.17 0.16 1.00 0.12 0.30
## Ab 0.33 0.30 0.12 1.00 0.31
## Res 0.65 0.60 0.30 0.31 1.00
##
## $data$Storer
## Fl Or El Ab Res
## Fl 1.00 0.81 0.07 0.66 -0.05
## Or 0.81 1.00 0.27 0.62 -0.06
## El 0.07 0.27 1.00 0.57 -0.19
## Ab 0.66 0.62 0.57 1.00 -0.12
## Res -0.05 -0.06 -0.19 -0.12 1.00
##
## $data$Storer
## Fl Or El Ab Res
## Fl 1.00 0.73 0.23 0.43 0.57
## Or 0.73 1.00 0.20 0.55 0.48
## El 0.23 0.20 1.00 0.29 0.40
## Ab 0.43 0.55 0.29 1.00 0.25
## Res 0.57 0.48 0.40 0.25 1.00
##
## $data$Tannehill
## Fl Or El Ab Res
## Fl 1.00 0.84 0.35 0.28 0.35
## Or 0.84 1.00 0.41 0.36 0.33
## El 0.35 0.41 1.00 0.38 0.39
## Ab 0.28 0.36 0.38 1.00 0.41
## Res 0.35 0.33 0.39 0.41 1.00
##
## $data$Tisone
## Fl Or El Ab Res
## Fl 1.00 0.68 -0.03 0.15 0.13
## Or 0.68 1.00 -0.12 0.26 0.35
## El -0.03 -0.12 1.00 0.30 0.11
## Ab 0.15 0.26 0.30 1.00 0.53
## Res 0.13 0.35 0.11 0.53 1.00
##
## $data$Trigani
## Fl Or El Ab Res
## Fl 1.00 0.72 0.28 0.17 0.69
## Or 0.72 1.00 0.24 0.26 0.47
## El 0.28 0.24 1.00 0.37 0.52
## Ab 0.17 0.26 0.37 1.00 0.16
## Res 0.69 0.47 0.52 0.16 1.00
##
## $data$Voss
## Fl Or El Ab Res
## Fl 1.00 0.71 0.43 0.38 0.65
## Or 0.71 1.00 0.51 0.44 0.79
## El 0.43 0.51 1.00 0.53 0.59
## Ab 0.38 0.44 0.53 1.00 0.42
## Res 0.65 0.79 0.59 0.42 1.00
##
## $data$Warne
## Fl Or El Ab Res
## Fl 1.00 0.74 0.40 0.35 0.63
## Or 0.74 1.00 0.37 0.37 0.50
## El 0.40 0.37 1.00 0.43 0.50
## Ab 0.35 0.37 0.43 1.00 0.26
## Res 0.63 0.50 0.50 0.26 1.00
##
## $data$Yoon
## Fl Or El Ab Res
## Fl 1.00 0.73 -0.16 0.08 0.09
## Or 0.73 1.00 0.08 0.29 0.28
## El -0.16 0.08 1.00 0.47 0.53
## Ab 0.08 0.29 0.47 1.00 0.66
## Res 0.09 0.28 0.53 0.66 1.00
##
## $data$Zhang
## Fl Or El Ab Res
## Fl 1.00 0.78 0.09 0.25 0.83
## Or 0.78 1.00 0.15 0.24 0.70
## El 0.09 0.15 1.00 0.33 0.06
## Ab 0.25 0.24 0.33 1.00 0.21
## Res 0.83 0.70 0.06 0.21 1.00
##
## $data$Acar
## Fl Or El Ab Res
## Fl 1.000 0.650 0.188 0.338 0.529
## Or 0.650 1.000 0.274 0.409 0.477
## El 0.188 0.274 1.000 0.363 0.413
## Ab 0.338 0.409 0.363 1.000 0.305
## Res 0.529 0.477 0.413 0.305 1.000
##
## $data$Acaretal
## Fl Or El Ab Res
## Fl 1.000 0.763 0.089 0.294 0.567
## Or 0.763 1.000 0.210 0.367 0.540
## El 0.089 0.210 1.000 0.469 0.277
## Ab 0.294 0.367 0.469 1.000 0.318
## Res 0.567 0.540 0.277 0.318 1.000
##
## $data$Balcia
## Fl Or El Ab Res
## Fl 1.000 0.292 0.000 0.176 -0.117
## Or 0.292 1.000 0.221 0.148 0.154
## El 0.000 0.221 1.000 0.304 0.035
## Ab 0.176 0.148 0.304 1.000 0.047
## Res -0.117 0.154 0.035 0.047 1.000
##
## $data$Balcib
## Fl Or El Ab Res
## Fl 1.000 0.430 0.116 0.267 -0.190
## Or 0.430 1.000 0.166 0.225 0.016
## El 0.116 0.166 1.000 0.363 -0.128
## Ab 0.267 0.225 0.363 1.000 -0.107
## Res -0.190 0.016 -0.128 -0.107 1.000
##
## $data$Gao
## Fl Or El Ab Res
## Fl 1.00 0.31 0.28 0.39 0.28
## Or 0.31 1.00 0.34 0.41 0.17
## El 0.28 0.34 1.00 0.25 0.26
## Ab 0.39 0.41 0.25 1.00 0.31
## Res 0.28 0.17 0.26 0.31 1.00
##
## $data$Liu
## Fl Or El Ab Res
## Fl 1.00 0.94 0.32 0.49 0.41
## Or 0.94 1.00 0.32 0.48 0.39
## El 0.32 0.32 1.00 0.66 0.62
## Ab 0.49 0.48 0.66 1.00 0.67
## Res 0.41 0.39 0.62 0.67 1.00
##
## $data$Wan
## Fl Or El Ab Res
## Fl 1.00 0.27 0.24 0.17 0.19
## Or 0.27 1.00 0.35 0.19 0.14
## El 0.24 0.35 1.00 0.27 0.12
## Ab 0.17 0.19 0.27 1.00 0.35
## Res 0.19 0.14 0.12 0.35 1.00
##
##
## $n
## [1] 30 599 13 60 24 24 203 24 35 36 25 21 17 116 45
## [16] 95 128 1758 42 125 99 500 1000 1000 1000 125 137 12 187 125
## [31] 19 12 39 31 51 18 84 43 46 199 24 107 120 432 163
## [46] 1067 477 319 1047 95 264 105 3752 Parameter-Based Meta-Analytic Structural Equation Modeling (pbMASEM)
Parameter-based MASEM comprises two steps: (1) Model-based generation of the relevant effect sizes and their sampling (co-)variances; and (2) Meta-analysis of the effect sizes.
2.1 Stage-1 Analysis: Model-Based Generation of Effect Sizes
2.1.1 Model Specification
In this step, we specify the analytic models for reliability
estimation. The extant literature indicated that several models may
describe the structure of the TTCT: (a) single-factor model with a
general TTCT factor (gTTCT); (b) two-factor model with a
factor representing innovativeness (INNOV) and a factor
describing adaptability (ADAPT); and (c) the two-factor
model from (b) with a cross-loading of the indicator Res.
In the following section, we specify these models and define the
reliability estimates.
## Model 1
## Single-factor model
SingleFactorModel <- " # Measurement Model
gCT =~ L1*Fl + L2*Or + L3*El + L4*Ab + L5*Res
# Residual variances
Fl ~~ R1*Fl
Or ~~ R2*Or
El ~~ R3*El
Ab ~~ R4*Ab
Res ~~ R5*Res
# Factor variances fixed to 1
gCT ~~ 1*gCT
# Additional constraints
R1 > 0
R2 > 0
R3 > 0
R4 > 0
R5 > 0
"
## Model 4
## Two-factor model with correlated traits ADAPT and INNOV and Res assigned to ADAPT
TwoFactorModel <- " # Measurement Model
Innov =~ L1*Fl + L2*Or
Adapt =~ L3*El + L4*Ab + L5*Res
# Residual variances
Fl ~~ R1*Fl
Or ~~ R2*Or
El ~~ R3*El
Ab ~~ R4*Ab
Res ~~ R5*Res
# Factor variances fixed to 1
Adapt ~~ 1*Adapt
Innov ~~ 1*Innov
# Factor correlation
Adapt ~~ fcorr*Innov
# Reliability coefficients
SREL1 := ((L1+L2)^2)/((L1+L2)^2 +
R1+R2)
SREL2 := ((L3+L4+L5)^2)/((L3+L4+L5)^2 +
R3+R4+R5)
# Composite reliability by Fu et al. (2022)
SRELCR := ((L1+L2)^2+(L3+L4+L5)^2+2*fcorr*(L1+L2)*(L3+L4+L5))/
((L1+L2)^2+(L3+L4+L5)^2+2*fcorr*(L1+L2)*(L3+L4+L5) +
R1+R2+R3+R4+R5)
# Additional constraints
R1 > 0
R2 > 0
R3 > 0
R4 > 0
R5 > 0
"
## Model 2
## Two-factor model with correlated traits ADAPT and INNOV and Res assigned to Innov
TwoFactorModelB <- " # Measurement Model
Innov =~ L1*Fl + L2*Or + L5*Res
Adapt =~ L3*El + L4*Ab
# Residual variances
Fl ~~ R1*Fl
Or ~~ R2*Or
El ~~ R3*El
Ab ~~ R4*Ab
Res ~~ R5*Res
# Factor variances fixed to 1
Adapt ~~ 1*Adapt
Innov ~~ 1*Innov
# Factor correlation
Adapt ~~ fcorr*Innov
# Reliability coefficients
SREL1 := ((L1+L2+L5)^2)/((L1+L2+L5)^2 +
R1+R2+R5)
SREL2 := ((L3+L4)^2)/((L3+L4)^2 +
R3+R4)
# Composite reliability by Fu et al. (2022)
SRELCR := ((L1+L2+L5)^2+(L3+L4)^2+2*fcorr*(L1+L2+L5)*(L3+L4))/
((L1+L2+L5)^2+(L3+L4)^2+2*fcorr*(L1+L2+L5)*(L3+L4) +
R1+R2+R3+R4+R5)
# Additional constraints
R1 > 0
R2 > 0
R3 > 0
R4 > 0
R5 > 0
"
## Model 3
## Two-factor model with correlated traits ADAPT and INNOV and a cross-loading of RES
TwoFactorModelC <- " # Measurement Model
Innov =~ L1*Fl + L2*Or + CL*Res
Adapt =~ L3*El + L4*Ab + L5*Res
# Residual variances
Fl ~~ R1*Fl
Or ~~ R2*Or
El ~~ R3*El
Ab ~~ R4*Ab
Res ~~ R5*Res
# Factor variances fixed to 1
Adapt ~~ 1*Adapt
Innov ~~ 1*Innov
# Factor correlation
Adapt ~~ fcorr*Innov
# Reliability coefficients
SREL1C := ((L1+L2+CL)^2)/((L1+L2+CL)^2 +
R1+R2+R5)
SREL2C := ((L3+L4+L5)^2)/((L3+L4+L5)^2 +
R3+R4+R5)
# Composite reliability by Fu et al. (2022)
SRELCRC := ((L1+L2+CL)^2+(L3+L4+L5)^2 +
2*fcorr*(L1+L2+CL)*(L3+L4+L5))/
((L1+L2+CL)^2+(L3+L4+L5)^2 +
2*fcorr*(L1+L2+CL)*(L3+L4+L5) +
R1+R2+R3+R4+R5)
# Additional constraints
R1 > 0
R2 > 0
R3 > 0
R4 > 0
R5 > 0
"2.1.2 Model Estimation
## Define a function to run the SEM in laavan and extract the
## model parameters
tra_study_lavaan <- function(id, data, para_names=NULL, model, ...) {
cor_i <- data$data[[id]]
n_i <- data$n[[id]]
fit_i <- sem(model=model, std.lv = T, sample.cov=cor_i, sample.nobs=n_i, ...)
results <- list()
coefs <- standardizedSolution(fit_i)$est.std[1:5]
vcoefs <- vcov(fit_i)[1:5,1:5]
if (is.null(para_names)) para_names <- names(coefs)
names(coefs) <- para_names
colnames(vcoefs) <- rownames(vcoefs) <- para_names
results$coefs <- coefs
results$vcoefs <- vcoefs
results$fit <- fit_i
results
}
## Specify some more parameters
k <- length(TTCT1$data)
para_names <- c("L1", "L2", "L3", "L4", "L5")
var_names <- c("Fl", "Or", "El", "Ab", "Res")2.1.2.1 Model 1: Single-factor model
## Model 1
## Single-factor model
## Model estimation
sfm.fit.all <- lapply(1:k, FUN = tra_study_lavaan,
data = TTCT1,
para_names = para_names,
model = SingleFactorModel,
estimator = "ML",
fixed.x = FALSE)
head(sfm.fit.all, 5)## [[1]]
## [[1]]$coefs
## L1 L2 L3 L4 L5
## 0.5976890 0.8960524 0.4174755 0.2081323 0.3135450
##
## [[1]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.042572394 -0.011264727 0.010409972 0.004974506 0.007613374
## L2 -0.011264727 0.054008776 -0.005773406 -0.002480583 -0.003957476
## L3 0.010409972 -0.005773406 0.039023005 0.003161825 0.004834038
## L4 0.004974506 -0.002480583 0.003161825 0.038628760 0.002318153
## L5 0.007613374 -0.003957476 0.004834038 0.002318153 0.038679852
##
## [[1]]$fit
## lavaan 0.6.16 ended normally after 15 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 30
##
## Model Test User Model:
##
## Test statistic 6.068
## Degrees of freedom 5
## P-value (Chi-square) 0.300
##
##
## [[2]]
## [[2]]$coefs
## L1 L2 L3 L4 L5
## 0.9730757 0.9742243 0.5408478 0.8774297 0.5112117
##
## [[2]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0008893069 0.0007806354 0.0004381411 0.0007106858 0.0004141340
## L2 0.0007806354 0.0008874639 0.0004384418 0.0007111127 0.0004144189
## L3 0.0004381411 0.0004384418 0.0014525250 0.0003954058 0.0002303779
## L4 0.0007106858 0.0007111127 0.0003954058 0.0010363019 0.0003737394
## L5 0.0004141340 0.0004144189 0.0002303779 0.0003737394 0.0014797243
##
## [[2]]$fit
## lavaan 0.6.16 ended normally after 31 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 599
##
## Model Test User Model:
##
## Test statistic 241.901
## Degrees of freedom 5
## P-value (Chi-square) 0.000
##
##
## [[3]]
## [[3]]$coefs
## L1 L2 L3 L4 L5
## -0.999999970 -0.390001184 -0.140000116 0.379999174 -0.004000719
##
## [[3]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0355029837 1.384620e-02 4.970452e-03 -1.349108e-02 1.420374e-04
## L2 0.0138462020 6.560591e-02 1.938482e-03 -5.261538e-03 5.539476e-05
## L3 0.0049704525 1.938482e-03 7.031099e-02 -1.888765e-03 1.988538e-05
## L4 -0.0134910830 -5.261538e-03 -1.888765e-03 6.587916e-02 -5.397403e-05
## L5 0.0001420374 5.539476e-05 1.988538e-05 -5.397403e-05 7.100539e-02
##
## [[3]]$fit
## lavaan 0.6.16 ended normally after 35 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 13
##
## Model Test User Model:
##
## Test statistic 19.730
## Degrees of freedom 5
## P-value (Chi-square) 0.001
##
##
## [[4]]
## [[4]]$coefs
## L1 L2 L3 L4 L5
## 0.9292358 0.8337253 0.1106548 0.3106395 0.5530644
##
## [[4]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0132722984 0.0032240784 0.0006336542 0.0017664748 0.0030590838
## L2 0.0032240784 0.0138996953 0.0008978624 0.0025289566 0.0045609420
## L3 0.0006336542 0.0008978624 0.0180692306 0.0003088704 0.0005538184
## L4 0.0017664748 0.0025289566 0.0003088704 0.0173304083 0.0015578240
## L5 0.0030590838 0.0045609420 0.0005538184 0.0015578240 0.0155443239
##
## [[4]]$fit
## lavaan 0.6.16 ended normally after 15 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 60
##
## Model Test User Model:
##
## Test statistic 20.992
## Degrees of freedom 5
## P-value (Chi-square) 0.001
##
##
## [[5]]
## [[5]]$coefs
## L1 L2 L3 L4 L5
## 0.96896788 0.88317014 0.35068734 0.08396762 0.83684140
##
## [[5]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.022635827 0.016067093 0.0065103169 0.0015596295 0.015363744
## L2 0.016067093 0.025577869 0.0063465707 0.0015191196 0.015246930
## L3 0.006510317 0.006346571 0.0391775607 0.0005982731 0.005991323
## L4 0.001559629 0.001519120 0.0005982731 0.0416466192 0.001434152
## L5 0.015363744 0.015246930 0.0059913227 0.0014341521 0.027062538
##
## [[5]]$fit
## lavaan 0.6.16 ended normally after 20 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 24
##
## Model Test User Model:
##
## Test statistic 13.218
## Degrees of freedom 5
## P-value (Chi-square) 0.021## Extract relevant model parameters
## FIT INDICES
## Chi-squares and p values of the parameter-based models
## Select CFI and SRMR
## SRMR performs well in small samples (see Pavlov et al., 2021, EPM)
sfm_model_fit <- t(sapply(sfm.fit.all, function(x) {
fitMeasures(x$fit, c("chisq", "pvalue", "ntotal",
"cfi", "srmr"))}))
row.names(sfm_model_fit) <- names(TTCT1$data)
options(scipen=50)
round(sfm_model_fit,4)## chisq pvalue ntotal cfi srmr
## Auth 6.0684 0.2996 30 0.9263 0.0915
## Barkul 241.9006 0.0000 599 0.9157 0.0556
## Berman 19.7296 0.0014 13 0.0000 0.2533
## Callans 20.9919 0.0008 60 0.8249 0.1265
## Carter 13.2179 0.0214 24 0.8721 0.1466
## Carter 2.1289 0.8310 24 1.0000 0.0348
## Chi 97.7060 0.0000 203 0.6591 0.1254
## Cho 13.9922 0.0157 24 0.8011 0.0947
## Cho 10.2306 0.0690 35 0.9212 0.0961
## Cockcroft 8.8922 0.1134 36 0.8845 0.0962
## Conway 14.1535 0.0147 25 0.8468 0.1038
## Crawford 9.7633 0.0822 21 0.9322 0.1001
## Digranes 7.0518 0.2168 17 0.9618 0.0756
## Fishkin 2.2949 0.8070 116 1.0000 0.0336
## Forsyth 15.3126 0.0091 45 0.9354 0.0807
## Garcia 22.4599 0.0004 95 0.8848 0.0630
## Gollmar 68.0533 0.0000 128 0.5816 0.1725
## Hokanson 601.4395 0.0000 1758 0.8187 0.1122
## Houtz 6.7467 0.2402 42 0.9596 0.0725
## Humble 13.0713 0.0227 125 0.9593 0.0605
## Ibrahim 45.6004 0.0000 99 0.7167 0.1149
## Kim_1 212.7346 0.0000 500 0.8282 0.1158
## Kim_2 262.3095 0.0000 1000 0.9004 0.0802
## Kim_2 134.6176 0.0000 1000 0.9102 0.0849
## Kim_2 305.3774 0.0000 1000 0.8737 0.1028
## Kim_3 181.6475 0.0000 125 0.2080 0.3009
## Kim_3 69.7880 0.0000 137 0.7345 0.1431
## Miranda 1.9213 0.8599 12 1.0000 0.0801
## Nguyen 6.3830 0.2707 187 0.9927 0.0418
## Zbarskaya 60.8402 0.0000 125 0.5753 0.1558
## Rose 14.5966 0.0122 19 0.7449 0.1172
## Rose 7.1499 0.2097 12 0.7917 0.1517
## Roskos 8.9598 0.1107 39 0.8776 0.0875
## Roskos 2.6746 0.7500 31 1.0000 0.0691
## Samuels 21.1041 0.0008 51 0.7149 0.1485
## Shore 12.9287 0.0241 18 0.5875 0.1627
## Stephens 6.2337 0.2841 84 0.9927 0.0498
## Storer 33.0343 0.0000 43 0.7132 0.1172
## Storer 10.4611 0.0632 46 0.9234 0.0746
## Tannehill 50.4142 0.0000 199 0.8790 0.1066
## Tisone 10.3408 0.0661 24 0.7382 0.1507
## Trigani 44.2975 0.0000 107 0.7992 0.1077
## Voss 20.2329 0.0011 120 0.9486 0.0594
## Warne 89.9866 0.0000 432 0.8950 0.0731
## Yoon 161.6311 0.0000 163 0.4854 0.2409
## Zhang 131.0945 0.0000 1067 0.9490 0.0807
## Acar 83.3332 0.0000 477 0.8816 0.0710
## Acaretal 92.5820 0.0000 319 0.8430 0.1082
## Balcia 145.2432 0.0000 1047 0.5870 0.0766
## Balcib 13.0968 0.0225 95 0.7898 0.0752
## Gao 11.6331 0.0402 264 0.9628 0.0368
## Liu 102.8416 0.0000 105 0.7433 0.2090
## Wan 30.5925 0.0000 375 0.8500 0.0581## Write out these indices
write.csv(round(sfm_model_fit,4), "SingleFactorModel-Fit.csv")
write.table(round(sfm_model_fit,4), "SingleFactorModel-Fit.txt", sep='\t')
head(sfm_model_fit, 5)## chisq pvalue ntotal cfi srmr
## Auth 6.068386 0.2996214212 30 0.9263234 0.09146972
## Barkul 241.900625 0.0000000000 599 0.9156861 0.05555471
## Berman 19.729588 0.0014044654 13 0.0000000 0.25325626
## Callans 20.991851 0.0008129362 60 0.8249091 0.12650613
## Carter 13.217865 0.0214204692 24 0.8721300 0.146636832.1.2.2 Model 2:
Two-factor model with Res assigned to
Innov
## Two-factor model with two correlated traits and Res assigned to Innov
## Model estimation
tfm2.fit.all <- lapply(1:k, FUN = tra_study_lavaan,
data = TTCT1,
para_names = para_names,
model = TwoFactorModelB,
estimator = "ML",
fixed.x = FALSE)
head(tfm2.fit.all, 5)## [[1]]
## [[1]]$coefs
## L1 L2 L3 L4 L5
## 0.5895150 0.9156545 0.3050138 0.8405801 0.3806894
##
## [[1]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0453756427 -0.016991967 0.0092965831 0.0018551323 0.0008401689
## L2 -0.0169919669 0.062985668 -0.0068274011 0.0028814532 0.0013049783
## L3 0.0092965831 -0.006827401 0.0385664551 0.0009598415 0.0004347016
## L4 0.0018551323 0.002881453 0.0009598415 0.1800957928 -0.0543381624
## L5 0.0008401689 0.001304978 0.0004347016 -0.0543381624 0.0625523360
##
## [[1]]$fit
## lavaan 0.6.16 ended normally after 24 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 30
##
## Model Test User Model:
##
## Test statistic 3.827
## Degrees of freedom 4
## P-value (Chi-square) 0.430
##
##
## [[2]]
## [[2]]$coefs
## L1 L2 L3 L4 L5
## 0.9788528 0.9694149 0.5060144 0.5773642 0.9699251
##
## [[2]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0008809722 0.0007803088 0.0004113168 0.0003845587 0.0006460277
## L2 0.0007803088 0.0008960755 0.0004092506 0.0003808509 0.0006397988
## L3 0.0004113168 0.0004092506 0.0014832342 0.0001987962 0.0003339616
## L4 0.0003845587 0.0003808509 0.0001987962 0.0014797718 0.0003727326
## L5 0.0006460277 0.0006397988 0.0003339616 0.0003727326 0.0011392359
##
## [[2]]$fit
## lavaan 0.6.16 ended normally after 35 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 599
##
## Model Test User Model:
##
## Test statistic 209.518
## Degrees of freedom 4
## P-value (Chi-square) 0.000
##
##
## [[3]]
## [[3]]$coefs
## L1 L2 L3 L4 L5
## -0.003999884 0.290000077 -0.999999919 0.400005817 0.125002325
##
## [[3]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.07100536071 -0.00004118225 0.0001420076 -0.000227209 -0.00007100308
## L2 -0.00004118225 0.06802013058 -0.0102958512 0.016473133 0.00514787415
## L3 0.00014200758 -0.01029585122 0.0355029029 -0.056803838 -0.01775126826
## L4 -0.00022720897 0.01647313328 -0.0568038377 0.859310580 0.34076063053
## L5 -0.00007100308 0.00514787415 -0.0177512683 0.340760631 0.14798934460
##
## [[3]]$fit
## lavaan 0.6.16 ended normally after 100 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 13
##
## Model Test User Model:
##
## Test statistic 8.000
## Degrees of freedom 4
## P-value (Chi-square) 0.092
##
##
## [[4]]
## [[4]]$coefs
## L1 L2 L3 L4 L5
## 0.9307451 0.8345907 0.5483165 0.2979739 0.9061197
##
## [[4]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0133971910 0.0031000754 0.0029840845 0.0002603574 0.0007917301
## L2 0.0031000754 0.0140002467 0.0045808305 0.0002334600 0.0007099371
## L3 0.0029840845 0.0045808305 0.0155867000 0.0001533806 0.0004664207
## L4 0.0002603574 0.0002334600 0.0001533806 0.0495590200 -0.0959194740
## L5 0.0007917301 0.0007099371 0.0004664207 -0.0959194740 0.3231228568
##
## [[4]]$fit
## lavaan 0.6.16 ended normally after 28 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 60
##
## Model Test User Model:
##
## Test statistic 17.089
## Degrees of freedom 4
## P-value (Chi-square) 0.002
##
##
## [[5]]
## [[5]]$coefs
## L1 L2 L3 L4 L5
## 0.9713847 0.8820594 0.8347220 1.0000000 0.5399999
##
## [[5]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.022556747 0.016059201 0.015329809 0.002324316 0.001255129
## L2 0.016059201 0.025618528 0.015237255 0.002110579 0.001139712
## L3 0.015329809 0.015237255 0.027130156 0.001997311 0.001078547
## L4 0.002324316 0.002110579 0.001997311 0.019965296 0.010781250
## L5 0.001255129 0.001139712 0.001078547 0.010781250 0.034108672
##
## [[5]]$fit
## lavaan 0.6.16 ended normally after 57 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 24
##
## Model Test User Model:
##
## Test statistic 5.102
## Degrees of freedom 4
## P-value (Chi-square) 0.277## Extract relevant model parameters
## FIT INDICES
## Chi-squares and p values of the parameter-based models
tfm2_model_fit <- t(sapply(tfm2.fit.all, function(x) {
fitMeasures(x$fit, c("chisq", "pvalue", "ntotal",
"cfi", "srmr"))}))
row.names(tfm2_model_fit) <- names(TTCT1$data)
options(scipen=50)
round(tfm2_model_fit,4)## chisq pvalue ntotal cfi srmr
## Auth 3.8267 0.4300 30 1.0000 0.0672
## Barkul 209.5178 0.0000 599 0.9269 0.0537
## Berman 7.9998 0.0916 13 0.7175 0.1554
## Callans 17.0886 0.0019 60 0.8567 0.1120
## Carter 5.1019 0.2770 24 0.9829 0.0709
## Carter 2.0598 0.7248 24 1.0000 0.0336
## Chi 114.0734 0.0000 203 0.5952 0.1638
## Cho 11.5357 0.0212 24 0.8333 0.0795
## Cho 6.5131 0.1640 35 0.9621 0.0579
## Cockcroft 8.8535 0.0649 36 0.8559 0.0942
## Conway 7.1166 0.1299 25 0.9478 0.0920
## Crawford 8.7763 0.0669 21 0.9320 0.0901
## Digranes 4.2876 0.3685 17 0.9946 0.0556
## Fishkin 2.7655 0.5978 116 1.0000 0.0510
## Forsyth 8.6404 0.0707 45 0.9709 0.0401
## Garcia 17.6336 0.0015 95 0.9101 0.0555
## Gollmar 50.4486 0.0000 128 0.6918 0.1480
## Hokanson 282.8336 0.0000 1758 0.9153 0.0723
## Houtz 6.6986 0.1527 42 0.9375 0.0732
## Humble 12.5814 0.0135 125 0.9567 0.0586
## Ibrahim 43.1922 0.0000 99 0.7265 0.1273
## Kim_1 133.7546 0.0000 500 0.8927 0.0786
## Kim_2 144.5767 0.0000 1000 0.9456 0.0508
## Kim_2 52.4440 0.0000 1000 0.9664 0.0483
## Kim_2 168.1729 0.0000 1000 0.9310 0.0628
## Kim_3 141.7916 0.0000 125 0.3822 0.2546
## Kim_3 60.1716 0.0000 137 0.7698 0.1375
## Miranda 0.9114 0.9229 12 1.0000 0.0521
## Nguyen 3.0839 0.5439 187 1.0000 0.0252
## Zbarskaya 27.6169 0.0000 125 0.8204 0.1117
## Rose 8.1898 0.0849 19 0.8886 0.0939
## Rose 6.0483 0.1956 12 0.8015 0.1636
## Roskos 8.8310 0.0655 39 0.8507 0.0857
## Roskos 1.9012 0.7539 31 1.0000 0.0603
## Samuels 9.0344 0.0602 51 0.9109 0.1071
## Shore 7.1004 0.1307 18 0.8387 0.1371
## Stephens 5.9531 0.2027 84 0.9885 0.0482
## Storer 18.5349 0.0010 43 0.8513 0.0939
## Storer 9.3526 0.0529 46 0.9249 0.0692
## Tannehill 36.5296 0.0000 199 0.9134 0.0929
## Tisone 9.7572 0.0447 24 0.7178 0.1732
## Trigani 32.3158 0.0000 107 0.8553 0.0814
## Voss 6.6525 0.1554 120 0.9910 0.0213
## Warne 52.9054 0.0000 432 0.9396 0.0496
## Yoon 121.9903 0.0000 163 0.6124 0.2126
## Zhang 19.0140 0.0008 1067 0.9939 0.0208
## Acar 57.6867 0.0000 477 0.9189 0.0565
## Acaretal 28.1507 0.0000 319 0.9567 0.0507
## Balcia 69.2343 0.0000 1047 0.8079 0.0569
## Balcib 4.5384 0.3380 95 0.9860 0.0468
## Gao 4.4435 0.3493 264 0.9975 0.0243
## Liu 55.8626 0.0000 105 0.8639 0.1704
## Wan 30.5920 0.0000 375 0.8442 0.0581## Write out these indices
write.csv(round(tfm2_model_fit,4), "TwoFactorModel2-Fit.csv")
write.table(round(tfm2_model_fit,4), "TwoFactorModel2-Fit.txt", sep='\t')
head(tfm2_model_fit, 5)## chisq pvalue ntotal cfi srmr
## Auth 3.826686 0.42996912 30 1.0000000 0.06722991
## Barkul 209.517841 0.00000000 599 0.9268554 0.05371925
## Berman 7.999803 0.09158543 13 0.7175163 0.15535802
## Callans 17.088567 0.00185784 60 0.8566965 0.11201201
## Carter 5.101863 0.27700454 24 0.9828550 0.07092265## Extract the scale reliability coefficients from the primary studies
## Point estimates
tfm2.srl1 <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[14,5]}))
tfm2.srl2 <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[15,5]}))
tfm2.srlcr <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[16,5]}))
## Standard errors
tfm2.srl1.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[14,6]}))
tfm2.srl2.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[15,6]}))
tfm2.srlcr.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[16,6]}))
## Lower 95% CI bound
tfm2.srl1.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[14,9]}))
tfm2.srl2.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[15,9]}))
tfm2.srlcr.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[16,9]}))
## Upper 95% CI bound
tfm2.srl1.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[14,10]}))
tfm2.srl2.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[15,10]}))
tfm2.srlcr.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[16,10]}))
## Factor correlation
tfm2.fcorr <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[13,5]}))
## Save these statistics
tfm2.srl.summary <- data.frame(names.studies,
TTCT1$n,
t(tfm2.srl1 ),
t(tfm2.srl1.se),
t(tfm2.srl1.ci.low),
t(tfm2.srl1.ci.upp),
t(tfm2.srl2),
t(tfm2.srl2.se),
t(tfm2.srl2.ci.low),
t(tfm2.srl2.ci.upp),
t(tfm2.srlcr),
t(tfm2.srlcr.se),
t(tfm2.srlcr.ci.low),
t(tfm2.srlcr.ci.upp),
t(tfm2.fcorr))
colnames(tfm2.srl.summary) <- c("Study",
"N",
"Omega Factor 1",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Omega Factor 2",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Composite reliability",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Factor correlation")
tfm2.srl.summary## Study N Omega Factor 1 SE Lower 95% CI Upper 95% CI
## 1 Auth 30 0.65564386 0.104524431 0.45077974 0.8605080
## 2 Barkul 599 0.87684294 0.009058061 0.85908947 0.8945964
## 3 Berman 13 0.21016567 0.195514311 -0.17303533 0.5933667
## 4 Callans 60 0.82486789 0.038333729 0.74973516 0.9000006
## 5 Carter_a 24 0.92550812 0.026368277 0.87382725 0.9771890
## 6 Carter_b 24 0.87840114 0.041440901 0.79717846 0.9596238
## 7 Chi 203 0.67031672 0.034570247 0.60256028 0.7380732
## 8 Cho_F 24 0.78113939 0.075683560 0.63280234 0.9294764
## 9 Cho_M 35 0.88259986 0.033779342 0.81639357 0.9488062
## 10 Cockcroft 36 0.51583279 0.135877965 0.24951687 0.7821487
## 11 Conway 25 0.83865184 0.055255865 0.73035233 0.9469513
## 12 Crawford 21 0.92207022 0.030206796 0.86286599 0.9812745
## 13 Digranes 17 0.91767077 0.034578505 0.84989815 0.9854434
## 14 Fishkin 116 0.67544008 0.052009029 0.57350426 0.7773759
## 15 Forsyth 45 0.94055748 0.015555705 0.91006886 0.9710461
## 16 Garcia 95 0.77668460 0.039474752 0.69931551 0.8540537
## 17 Gollmar 128 0.69572184 0.041101217 0.61516494 0.7762787
## 18 Hokanson 1758 0.82110347 0.007243849 0.80690579 0.8353012
## 19 Houtz 42 0.75289536 0.063984429 0.62748818 0.8783025
## 20 Humble 125 0.79567253 0.030606303 0.73568528 0.8556598
## 21 Ibrahim 99 0.75192054 0.039023666 0.67543556 0.8284055
## 22 Kim_1 500 0.88283380 0.008954717 0.86528287 0.9003847
## 23 Kim_2a 1000 0.88851515 0.006118823 0.87652248 0.9005078
## 24 Kim_2b 1000 0.82563158 0.009412889 0.80718266 0.8440805
## 25 Kim_2c 1000 0.88656108 0.006190075 0.87442875 0.8986934
## 26 Kim_3a 125 0.58458079 0.052557496 0.48156999 0.6875916
## 27 Kim_3b 137 0.23957472 0.093681148 0.05596305 0.4231864
## 28 Miranda 12 0.66708273 0.162981604 0.34764466 0.9865208
## 29 Nguyen 187 0.77821059 0.026939478 0.72541019 0.8310110
## 30 Zbarskaya 125 0.65602250 0.050572601 0.55690203 0.7551430
## 31 Rose 19 0.68572571 0.114188180 0.46192099 0.9095304
## 32 Rose_b 12 0.73783670 0.118126130 0.50631374 0.9693597
## 33 Roskos_Y 39 0.66456878 0.089855121 0.48845598 0.8406816
## 34 Roskos_O 31 0.51431965 0.194303603 0.13349158 0.8951477
## 35 Samuels 51 0.64164977 0.074671111 0.49529708 0.7880025
## 36 Shore 18 0.04879865 0.194405933 -0.33222998 0.4298273
## 37 Stephens 84 0.88711473 0.021369179 0.84523191 0.9289976
## 38 Storer_a 43 0.68572584 0.075903808 0.53695711 0.8344946
## 39 Storer_b 46 0.82236445 0.044859980 0.73444050 0.9102884
## 40 Tannehill 199 0.80535849 0.023291147 0.75970868 0.8510083
## 41 Tisone 24 0.74438228 0.081760928 0.58413380 0.9046308
## 42 Trigani 107 0.85280675 0.023974059 0.80581846 0.8997950
## 43 Voss 120 0.88620094 0.017965305 0.85098959 0.9214123
## 44 Warne 432 0.84503235 0.012736174 0.82006991 0.8699948
## 45 Yoon 163 0.74419757 0.031567238 0.68232692 0.8060682
## 46 Zhang 1067 0.91182228 0.004665094 0.90267886 0.9209657
## 47 Acar 477 0.79342944 0.016278481 0.76152421 0.8253347
## 48 Acaretal 319 0.84145744 0.015254573 0.81155902 0.8713559
## 49 Balcia 1047 0.52510058 0.019826928 0.48624052 0.5639606
## 50 Balcib 95 0.38902701 0.104286830 0.18462858 0.5934254
## 51 Gao 264 0.50809527 0.052198613 0.40578787 0.6104027
## 52 Liu 105 0.85566248 0.025016074 0.80663188 0.9046931
## 53 Wan 375 0.43441826 0.050394311 0.33564723 0.5331893
## Omega Factor 2 SE Lower 95% CI Upper 95% CI Composite reliability
## 1 0.56496187 0.25729832 0.060666434 1.0692573 0.7062665
## 2 0.76734112 0.01859382 0.730897899 0.8037843 0.9067247
## 3 0.13125388 0.63906542 -1.121291339 1.3837991 0.3792193
## 4 0.57080369 0.37412291 -0.162463739 1.3040711 0.7960050
## 5 0.77000008 0.08239435 0.608510111 0.9314900 0.9062086
## 6 0.44168307 0.22914851 -0.007439755 0.8908059 0.8502797
## 7 0.59393298 0.05735172 0.481525674 0.7063403 0.7389661
## 8 0.74362049 0.10602301 0.535819218 0.9514218 0.8521182
## 9 0.52951075 0.15915172 0.217579111 0.8414424 0.8288963
## 10 0.57158106 0.16899674 0.240353530 0.9028086 0.7019128
## 11 0.81823414 0.07401261 0.673172080 0.9632962 0.8918299
## 12 0.55176587 0.19595122 0.167708530 0.9358232 0.8802762
## 13 0.19431940 0.44525652 -0.678367348 1.0670062 0.8539556
## 14 0.55499989 0.06156127 0.434342023 0.6756578 0.6332173
## 15 0.69824220 0.09756896 0.507010559 0.8894739 0.9148728
## 16 0.66395267 0.06913231 0.528455821 0.7994495 0.8312741
## 17 0.61601475 0.14217430 0.337358237 0.8946713 0.5904046
## 18 0.68896747 0.01872603 0.652265130 0.7256698 0.8376745
## 19 0.37844715 0.19970361 -0.012964733 0.7698590 0.7544145
## 20 0.12418332 0.17032751 -0.209652468 0.4580191 0.7245921
## 21 0.49703327 0.12591109 0.250252063 0.7438145 0.7632648
## 22 0.64133584 0.04348908 0.556098816 0.7265729 0.8553685
## 23 0.66067493 0.02159081 0.618357717 0.7029921 0.8796064
## 24 0.46688762 0.03601213 0.396305141 0.5374701 0.7621709
## 25 0.59277500 0.02756163 0.538755195 0.6467948 0.8475895
## 26 0.72052879 0.07125025 0.580880865 0.8601767 0.7188807
## 27 0.66982318 0.05641769 0.559246536 0.7803998 0.6533906
## 28 0.34500000 0.22212131 -0.090349758 0.7803498 0.5295116
## 29 0.26798772 0.11342589 0.045677058 0.4902984 0.6832934
## 30 0.75199998 0.03847119 0.676597836 0.8274021 0.7536246
## 31 0.78500003 0.08740302 0.613693262 0.9563068 0.8199745
## 32 0.34999851 0.22201730 -0.085147407 0.7851444 0.7224728
## 33 0.33423467 0.21365718 -0.084525713 0.7529950 0.7051059
## 34 0.07834996 0.33674747 -0.581662952 0.7383629 0.5443966
## 35 0.74000003 0.06263745 0.617232885 0.8627672 0.6581952
## 36 0.47991626 0.24917184 -0.008451579 0.9682841 0.5015645
## 37 0.23396191 0.18273740 -0.124196812 0.5921206 0.7925324
## 38 0.78499986 0.05809909 0.671127741 0.8988720 0.8199745
## 39 0.49852653 0.16718496 0.170850040 0.8262030 0.8118961
## 40 0.55369507 0.06374239 0.428762276 0.6786279 0.8048860
## 41 0.64999993 0.11519912 0.424213802 0.8757861 0.6900958
## 42 0.57668943 0.10559243 0.369732059 0.7836468 0.8213932
## 43 0.70101044 0.05527658 0.592670348 0.8093505 0.8902267
## 44 0.60592141 0.03833412 0.530787909 0.6810549 0.8427107
## 45 0.73500001 0.03558981 0.665245250 0.8047548 0.7849944
## 46 0.62011383 0.08280858 0.457811995 0.7824157 0.8619198
## 47 0.54442539 0.04311695 0.459917724 0.6289331 0.7995981
## 48 0.71460989 0.06454992 0.588094373 0.8411254 0.8473550
## 49 0.48524886 0.03780843 0.411145694 0.5593520 0.5777128
## 50 0.59196858 0.14094157 0.315728178 0.8682090 0.5673409
## 51 0.40557543 0.07389201 0.260749746 0.5504011 0.6626396
## 52 0.82925718 0.04431111 0.742409002 0.9161054 0.8909229
## 53 0.42685866 0.05944903 0.310340699 0.5433766 0.5986157
## SE Lower 95% CI Upper 95% CI Factor correlation
## 1 0.092312134 0.5253380 0.8871949 0.48204516
## 2 0.006367321 0.8942449 0.9192044 0.90362658
## 3 0.292905012 -0.1948640 0.9533026 -1.99997082
## 4 0.084711215 0.6299741 0.9620360 0.33847042
## 5 0.030774514 0.8458916 0.9665255 0.34618970
## 6 0.051826648 0.7487013 0.9518581 1.06267836
## 7 0.028113255 0.6838652 0.7940671 0.60827484
## 8 0.049000598 0.7560788 0.9481577 0.78415097
## 9 0.048676434 0.7334922 0.9243004 0.41362514
## 10 0.081491839 0.5421917 0.8616339 1.05894673
## 11 0.035242566 0.8227557 0.9609040 0.70916554
## 12 0.045274721 0.7915393 0.9690130 0.74551392
## 13 0.068403324 0.7198875 0.9880236 1.86818896
## 14 0.047527870 0.5400644 0.7263702 -0.02750486
## 15 0.022620103 0.8705382 0.9592074 0.65616223
## 16 0.028347792 0.7757135 0.8868348 0.80388201
## 17 0.084898049 0.4240075 0.7568017 -0.29942308
## 18 0.006533163 0.8248697 0.8504793 0.51334010
## 19 0.062922580 0.6310885 0.8777404 0.92952069
## 20 0.042791799 0.6407217 0.8084624 1.38626219
## 21 0.039552275 0.6857437 0.8407858 0.67185316
## 22 0.011844659 0.8321534 0.8785836 0.42054700
## 23 0.006389031 0.8670841 0.8921287 0.68298986
## 24 0.012648486 0.7373803 0.7869614 0.29332002
## 25 0.008232262 0.8314545 0.8637244 0.43867231
## 26 0.038151808 0.6441045 0.7936568 0.43386323
## 27 0.047508286 0.5602760 0.7465051 1.51035460
## 28 0.183120296 0.1706024 0.8884208 0.39013455
## 29 0.038323521 0.6081807 0.7584062 0.36799956
## 30 0.033105707 0.6887387 0.8185106 0.36486522
## 31 0.064210127 0.6941250 0.9458241 0.71164813
## 32 0.114449578 0.4981557 0.9467898 -0.55999978
## 33 0.077116973 0.5539595 0.8562524 1.11841425
## 34 0.133724762 0.2823009 0.8064923 2.07692958
## 35 0.060411229 0.5397913 0.7765990 -0.10999984
## 36 0.161694096 0.1846499 0.8184791 3.33168104
## 37 0.040502569 0.7131489 0.8719160 0.74456919
## 38 0.042682138 0.7363191 0.9036300 0.71164801
## 39 0.047358438 0.7190753 0.9047170 0.74711475
## 40 0.022718967 0.7603577 0.8494144 0.64817413
## 41 0.082728037 0.5279519 0.8522398 -0.11999973
## 42 0.030549877 0.7615165 0.8812698 0.41817045
## 43 0.016719483 0.8574572 0.9229963 0.74367211
## 44 0.012553997 0.8181054 0.8673161 0.66580678
## 45 0.024549758 0.7368778 0.8331111 0.29000067
## 46 0.017322974 0.8279674 0.8958722 0.28115578
## 47 0.015202535 0.7698017 0.8293946 0.67112035
## 48 0.017402647 0.8132464 0.8814636 0.40766667
## 49 0.019136542 0.5402059 0.6152198 0.32801365
## 50 0.074883057 0.4205728 0.7141090 0.42054211
## 51 0.034017154 0.5959672 0.7293120 1.30264434
## 52 0.018060142 0.8555256 0.9263201 0.50835576
## 53 0.033849719 0.5322715 0.6649599 0.99740458## Write out the results
write.table(tfm2.srl.summary, "TwoFactorModel2-Reliability.txt", sep='\t')
## Note: The standard errors of the derived parameters are identical
## to the asymptotic sampling errors.2.1.2.3 Model 3:
Two-factor model with a cross-loading of Res
## Model 3
## Two-factor model with a cross-loading of RES
## Model estimation
tfmc.fit.all <- lapply(1:k, FUN = tra_study_lavaan,
data = TTCT1,
para_names = para_names,
model = TwoFactorModelC,
estimator = "ML",
fixed.x = FALSE)
head(tfmc.fit.all, 5)## [[1]]
## [[1]]$coefs
## L1 L2 L3 L4 L5
## 0.5841981 0.9243441 0.2905938 0.8505294 0.3762364
##
## [[1]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0472744385 -0.020955050 0.0149970190 0.001811084 0.0008011436
## L2 -0.0209550502 0.069905399 -0.0173292962 0.002865578 0.0012676049
## L3 0.0149970190 -0.017329296 0.0603273242 0.004479300 -0.0008967692
## L4 0.0018110844 0.002865578 0.0044793005 0.187477810 -0.0567234757
## L5 0.0008011436 0.001267605 -0.0008967692 -0.056723476 0.0626024237
##
## [[1]]$fit
## lavaan 0.6.16 ended normally after 27 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 30
##
## Model Test User Model:
##
## Test statistic 3.824
## Degrees of freedom 3
## P-value (Chi-square) 0.281
##
##
## [[2]]
## [[2]]$coefs
## L1 L2 L3 L4 L5
## 0.9846454 0.9648144 -0.3436892 0.5755731 0.9729435
##
## [[2]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0008728441 0.0007796787 -0.0002618658 0.0003795444 0.0006415783
## L2 0.0007796787 0.0009044976 -0.0002901420 0.0003719003 0.0006286568
## L3 -0.0002618658 -0.0002901420 0.0217819949 -0.0006943377 0.0015061441
## L4 0.0003795444 0.0003719003 -0.0006943377 0.0014633668 0.0003967688
## L5 0.0006415783 0.0006286568 0.0015061441 0.0003967688 0.0010857613
##
## [[2]]$fit
## lavaan 0.6.16 ended normally after 39 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 599
##
## Model Test User Model:
##
## Test statistic 105.953
## Degrees of freedom 3
## P-value (Chi-square) 0.000
##
##
## [[3]]
## [[3]]$coefs
## L1 L2 L3 L4 L5
## 0.3900001 1.0000000 -0.0369997 0.7995635 0.2577819
##
## [[3]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0656059229 0.0138461529 -0.00051230348 0.00072316879 0.00023315188
## L2 0.0138461529 0.0355029412 -0.00131359810 0.00185427814 0.00059782508
## L3 -0.0005123035 -0.0013135981 0.04042756743 -0.00006860764 -0.00002211932
## L4 0.0007231688 0.0018542781 -0.00006860764 0.04830886299 0.00731762625
## L5 0.0002331519 0.0005978251 -0.00002211932 0.00731762625 0.06864667893
##
## [[3]]$fit
## lavaan 0.6.16 ended normally after 84 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 13
##
## Model Test User Model:
##
## Test statistic 6.722
## Degrees of freedom 3
## P-value (Chi-square) 0.081
##
##
## [[4]]
## [[4]]$coefs
## L1 L2 L3 L4 L5
## 0.9543728 0.8172905 0.3321244 0.5717039 0.4722738
##
## [[4]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0145420153 0.0018926695 0.0009926216 0.0004149356 0.0003427705
## L2 0.0018926695 0.0150344590 0.0034681932 0.0003553357 0.0002935362
## L3 0.0009926216 0.0034681932 0.0252735672 0.0086084156 0.0054244604
## L4 0.0004149356 0.0003553357 0.0086084156 0.0284483805 0.0059111611
## L5 0.0003427705 0.0002935362 0.0054244604 0.0059111611 0.0246184116
##
## [[4]]$fit
## lavaan 0.6.16 ended normally after 20 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 60
##
## Model Test User Model:
##
## Test statistic 4.379
## Degrees of freedom 3
## P-value (Chi-square) 0.223
##
##
## [[5]]
## [[5]]$coefs
## L1 L2 L3 L4 L5
## 0.9840361 0.8739518 0.7425230 1.0000000 0.5400000
##
## [[5]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.022195703 0.0159245264 0.013379938 0.001956134 0.0010563119
## L2 0.015924526 0.0259417568 0.013868937 0.001737302 0.0009381423
## L3 0.013379938 0.0138689368 0.024062723 0.001476038 0.0007970600
## L4 0.001956134 0.0017373016 0.001476038 0.019965294 0.0107812522
## L5 0.001056312 0.0009381423 0.000797060 0.010781252 0.0341086691
##
## [[5]]$fit
## lavaan 0.6.16 ended normally after 56 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 24
##
## Model Test User Model:
##
## Test statistic 0.644
## Degrees of freedom 3
## P-value (Chi-square) 0.886## Extract relevant model parameters
## FIT INDICES
## Chi-squares and p values of the parameter-based models
tfmc_model_fit <- t(sapply(tfmc.fit.all, function(x) {
fitMeasures(x$fit, c("chisq", "pvalue", "ntotal",
"cfi", "srmr"))}))
row.names(tfmc_model_fit) <- names(TTCT1$data)
options(scipen=50)
round(tfmc_model_fit,4)## chisq pvalue ntotal cfi srmr
## Auth 3.8239 0.2811 30 0.9432 0.0678
## Barkul 105.9531 0.0000 599 0.9634 0.0207
## Berman 6.7223 0.0813 13 0.7371 0.1304
## Callans 4.3791 0.2233 60 0.9849 0.0658
## Carter 0.6436 0.8864 24 1.0000 0.0421
## Carter 2.0061 0.5711 24 1.0000 0.0328
## Chi 55.4909 0.0000 203 0.8070 0.1078
## Cho 4.5596 0.2070 24 0.9655 0.0536
## Cho 6.4872 0.0902 35 0.9475 0.0580
## Cockcroft 8.2605 0.0409 36 0.8439 0.0962
## Conway 0.7394 0.8639 25 1.0000 0.0151
## Crawford 3.8836 0.2743 21 0.9874 0.0518
## Digranes 4.1299 0.2478 17 0.9790 0.0554
## Fishkin 0.6283 0.8899 116 1.0000 0.0151
## Forsyth 4.2433 0.2364 45 0.9922 0.0310
## Garcia 11.5125 0.0093 95 0.9438 0.0469
## Gollmar 33.1378 0.0000 128 0.8000 0.1100
## Hokanson 196.2533 0.0000 1758 0.9413 0.0444
## Houtz 2.9690 0.3964 42 1.0000 0.0447
## Humble 6.2968 0.0980 125 0.9834 0.0406
## Ibrahim 40.7638 0.0000 99 0.7365 0.1105
## Kim_1 81.5590 0.0000 500 0.9350 0.0973
## Kim_2 117.9371 0.0000 1000 0.9555 0.0404
## Kim_2 12.0904 0.0071 1000 0.9937 0.0131
## Kim_2 107.9193 0.0000 1000 0.9559 0.0864
## Kim_3 43.2068 0.0000 125 0.8197 0.1415
## Kim_3 36.3656 0.0000 137 0.8633 0.1240
## Miranda 0.6569 0.8833 12 1.0000 0.0430
## Nguyen 3.0489 0.3841 187 0.9997 0.0239
## Zbarskaya 4.3840 0.2229 125 0.9895 0.0362
## Rose 7.9763 0.0465 19 0.8677 0.0878
## Rose 7.1287 0.0679 12 0.5999 0.1509
## Roskos 8.8309 0.0316 39 0.8198 0.0855
## Roskos 2.1785 0.5362 31 1.0000 0.0618
## Samuels 4.1263 0.2481 51 0.9801 0.0680
## Shore 7.5574 0.0561 18 0.7629 0.1392
## Stephens 1.8276 0.6090 84 1.0000 0.0280
## Storer 18.0516 0.0004 43 0.8460 0.0878
## Storer 9.1835 0.0269 46 0.9132 0.0768
## Tannehill 6.4306 0.0924 199 0.9909 0.0214
## Tisone 1.9791 0.5768 24 1.0000 0.0637
## Trigani 7.1323 0.0678 107 0.9789 0.0529
## Voss 2.8902 0.4089 120 1.0000 0.0183
## Warne 28.0935 0.0000 432 0.9690 0.0439
## Yoon 23.1185 0.0000 163 0.9339 0.0907
## Zhang 17.3737 0.0006 1067 0.9942 0.0187
## Acar 44.7849 0.0000 477 0.9369 0.0470
## Acaretal 23.1567 0.0000 319 0.9639 0.0372
## Balcia 69.1219 0.0000 1047 0.8053 0.0568
## Balcib 4.2004 0.2406 95 0.9688 0.0417
## Gao 4.2199 0.2387 264 0.9931 0.0240
## Liu 3.6774 0.2985 105 0.9982 0.0300
## Wan 27.6775 0.0000 375 0.8554 0.0551## Write out these indices
write.csv(round(tfmc_model_fit,4), "TwoFactorModel3-Fit.csv")
write.table(round(tfmc_model_fit,4), "TwoFactorModel3-Fit.txt", sep='\t')
head(tfmc_model_fit, 5)## chisq pvalue ntotal cfi srmr
## Auth 3.8238669 0.2811222 30 0.9431856 0.06779564
## Barkul 105.9531146 0.0000000 599 0.9633586 0.02069093
## Berman 6.7223298 0.0812949 13 0.7371126 0.13038098
## Callans 4.3791012 0.2233310 60 0.9849006 0.06584712
## Carter 0.6436149 0.8863791 24 1.0000000 0.04213503## Extract the scale reliability coefficients from the primary studies
## Point estimates
tfmc.srl1 <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[15,5]}))
tfmc.srl2 <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[16,5]}))
tfmc.srlcr <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[17,5]}))
## Standard errors
tfmc.srl1.se <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[15,6]}))
tfmc.srl2.se <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[16,6]}))
tfmc.srlcr.se <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[17,6]}))
## Lower 95% CI bound
tfmc.srl1.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[15,9]}))
tfmc.srl2.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[16,9]}))
tfmc.srlcr.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[17,9]}))
## Upper 95% CI bound
tfmc.srl1.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[15,10]}))
tfmc.srl2.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[16,10]}))
tfmc.srlcr.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[17,10]}))
## Factor correlation
tfmc.fcorr <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[14,5]}))
## Save these statistics
tfmc.srl.summary <- data.frame(names.studies,
TTCT1$n,
t(tfmc.srl1 ),
t(tfmc.srl1.se),
t(tfmc.srl1.ci.low),
t(tfmc.srl1.ci.upp),
t(tfmc.srl2),
t(tfmc.srl2.se),
t(tfmc.srl2.ci.low),
t(tfmc.srl2.ci.upp),
t(tfmc.srlcr),
t(tfmc.srlcr.se),
t(tfmc.srlcr.ci.low),
t(tfmc.srlcr.ci.upp),
t(tfmc.fcorr))
colnames(tfmc.srl.summary) <- c("Study",
"N",
"Omega Factor 1",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Omega Factor 2",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Composite reliability",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Factor correlation")
tfmc.srl.summary## Study N Omega Factor 1 SE Lower 95% CI Upper 95% CI
## 1 Auth 30 0.653670574 0.107856677 0.4422754 0.8650658
## 2 Barkul 599 0.787772957 0.028512584 0.7318893 0.8436566
## 3 Berman 13 0.683443320 0.159833437 0.3701755 0.9967111
## 4 Callans 60 0.856329746 0.040851555 0.7762622 0.9363973
## 5 Carter_a 24 0.927154283 0.025691873 0.8767991 0.9775094
## 6 Carter_b 24 0.847019015 0.227871788 0.4003985 1.2936395
## 7 Chi 203 0.593002702 0.049954691 0.4950933 0.6909121
## 8 Cho_F 24 0.817798140 0.062998939 0.6943225 0.9412738
## 9 Cho_M 35 0.882090358 0.034132973 0.8151910 0.9489898
## 10 Cockcroft 36 0.343777202 0.472594250 -0.5824905 1.2700449
## 11 Conway 25 0.807463911 0.069678278 0.6708970 0.9440308
## 12 Crawford 21 0.933745960 0.031927839 0.8711685 0.9963234
## 13 Digranes 17 0.913968974 0.038167446 0.8391622 0.9887758
## 14 Fishkin 116 0.669786264 0.051302281 0.5692356 0.7703369
## 15 Forsyth 45 0.937871024 0.016425512 0.9056776 0.9700644
## 16 Garcia 95 0.735378067 0.051965699 0.6335272 0.8372290
## 17 Gollmar 128 0.766058096 0.038435615 0.6907257 0.8413905
## 18 Hokanson 1758 0.804897564 0.008142107 0.7889393 0.8208558
## 19 Houtz 42 0.458329744 1.281344374 -2.0530591 2.9697186
## 20 Humble 125 0.001128471 0.106290290 -0.2071967 0.2094536
## 21 Ibrahim 99 0.758951858 0.043049709 0.6745760 0.8433277
## 22 Kim_1 500 0.889613867 0.008285288 0.8733750 0.9058527
## 23 Kim_2a 1000 0.878281651 0.006987244 0.8645869 0.8919764
## 24 Kim_2b 1000 0.824550915 0.009605933 0.8057236 0.8433782
## 25 Kim_2c 1000 0.888960436 0.005933615 0.8773308 0.9005901
## 26 Kim_3a 125 0.564555188 0.065229490 0.4367077 0.6924026
## 27 Kim_3b 137 0.596009927 0.061019415 0.4764141 0.7156058
## 28 Miranda 12 0.664389696 0.166030810 0.3389753 0.9898041
## 29 Nguyen 187 0.776273251 0.028415615 0.7205797 0.8319668
## 30 Zbarskaya 125 0.648665206 0.047054876 0.5564393 0.7408911
## 31 Rose 19 0.716436267 0.119989951 0.4812603 0.9516122
## 32 Rose_b 12 0.993535779 0.080098040 0.8365465 1.1505251
## 33 Roskos_Y 39 0.658265502 0.504882906 -0.3312868 1.6478178
## 34 Roskos_O 31 0.314200161 0.418032551 -0.5051286 1.1335289
## 35 Samuels 51 0.659250852 0.072295634 0.5175540 0.8009477
## 36 Shore 18 0.105086770 0.441498475 -0.7602343 0.9704079
## 37 Stephens 84 0.871663728 0.051241046 0.7712331 0.9720943
## 38 Storer_a 43 0.716436134 0.079760447 0.5601085 0.8727637
## 39 Storer_b 46 0.835509219 0.049907087 0.7376931 0.9333253
## 40 Tannehill 199 0.772246618 0.031778095 0.7099627 0.8345305
## 41 Tisone 24 0.750704490 0.080076475 0.5937575 0.9076515
## 42 Trigani 107 0.857182140 0.022762679 0.8125681 0.9017962
## 43 Voss 120 0.877239537 0.020103505 0.8378374 0.9166417
## 44 Warne 432 0.832182178 0.014113254 0.8045207 0.8598436
## 45 Yoon 163 0.807233506 0.028921209 0.7505490 0.8639180
## 46 Zhang 1067 0.912546014 0.004657329 0.9034178 0.9216742
## 47 Acar 477 0.768526179 0.019834695 0.7296509 0.8074015
## 48 Acaretal 319 0.835859365 0.016044373 0.8044130 0.8673058
## 49 Balcia 1047 0.523362679 0.020534407 0.4831160 0.5636094
## 50 Balcib 95 0.410740294 0.111844169 0.1915298 0.6299508
## 51 Gao 264 0.544652556 0.094784192 0.3588790 0.7304262
## 52 Liu 105 0.881255287 0.022466718 0.8372213 0.9252892
## 53 Wan 375 0.149122509 0.293522612 -0.4261712 0.7244163
## Omega Factor 2 SE Lower 95% CI Upper 95% CI Composite reliability
## 1 0.4304457 0.22889696 -0.01818406 0.8790755 0.7080218
## 2 0.8229706 0.02047607 0.78283822 0.8631029 0.9178338
## 3 0.7640593 0.10566824 0.55695338 0.9711653 0.6825557
## 4 0.6209308 0.08270351 0.45883493 0.7830267 0.8126529
## 5 0.7684078 0.07699323 0.61750383 0.9193117 0.9125341
## 6 0.4775192 0.73579760 -0.96461763 1.9196559 0.8488549
## 7 0.7425099 0.03375371 0.67635385 0.8086659 0.8036901
## 8 0.7974172 0.07274661 0.65483647 0.9399979 0.8789835
## 9 0.5326083 0.16591194 0.20742687 0.8577897 0.8283102
## 10 0.6753116 0.19549267 0.29215303 1.0584702 0.7281700
## 11 0.8246381 0.06286488 0.70142522 0.9478510 0.9091115
## 12 0.7211523 0.10708496 0.51126967 0.9310350 0.9031175
## 13 0.1955976 0.45213661 -0.69057391 1.0817690 0.8537584
## 14 0.4145217 0.06674312 0.28370756 0.5453358 0.6674900
## 15 0.7290754 0.07478038 0.58250858 0.8756423 0.9219511
## 16 0.7041132 0.05642252 0.59352706 0.8146993 0.8415844
## 17 0.6126398 0.05983735 0.49536079 0.7299189 0.6615778
## 18 0.6345411 0.01541796 0.60432247 0.6647598 0.8430808
## 19 0.7879058 0.43981282 -0.07411152 1.6499230 0.8128315
## 20 0.8493046 0.07297606 0.70627414 0.9923350 0.8239503
## 21 0.5848694 0.07905407 0.42992630 0.7398126 0.7830406
## 22 0.7072821 0.02020063 0.66768961 0.7468746 0.8731967
## 23 0.6546655 0.02028131 0.61491491 0.6944162 0.8832819
## 24 0.4826084 0.02931695 0.42514820 0.5400685 0.7778928
## 25 0.6794377 0.01538088 0.64929172 0.7095837 0.8706154
## 26 0.8472008 0.02375504 0.80064182 0.8937598 0.8387767
## 27 0.8589660 0.02029488 0.81918874 0.8987432 0.8404802
## 28 0.1303795 0.93840119 -1.70885300 1.9696121 0.5333744
## 29 0.2079270 0.10271652 0.00660629 0.4092477 0.6841580
## 30 0.7088408 0.04021720 0.63001658 0.7876651 0.7958535
## 31 0.5457922 0.16855638 0.21542774 0.8761566 0.8180477
## 32 0.9882623 0.11155613 0.76961626 1.2069083 0.7823291
## 33 0.3066629 0.90301354 -1.46321117 2.0765369 0.7052351
## 34 0.6699684 0.11288397 0.44871988 0.8912169 0.6355349
## 35 0.6433924 0.07396237 0.49842882 0.7883560 0.7680445
## 36 0.9003439 0.05294429 0.79657497 1.0041128 0.8075012
## 37 0.5430463 0.39655382 -0.23418488 1.3202775 0.8198361
## 38 0.5457923 0.11204374 0.32619063 0.7653940 0.8180476
## 39 0.3700022 0.22709589 -0.07509756 0.8151020 0.8158777
## 40 0.6777027 0.04460754 0.59027352 0.7651319 0.8392724
## 41 0.6648445 0.10273171 0.46349403 0.8661949 0.8010826
## 42 0.7005870 0.04481718 0.61274692 0.7884270 0.8574797
## 43 0.7123495 0.04952693 0.61527846 0.8094205 0.8922357
## 44 0.6426562 0.03068659 0.58251154 0.7028008 0.8524045
## 45 0.7939201 0.02771560 0.73959848 0.8482416 0.8556028
## 46 0.5389738 0.06830205 0.40510422 0.6728433 0.8581444
## 47 0.5661430 0.03723139 0.49317081 0.6391152 0.8049032
## 48 0.6079822 0.04093878 0.52774369 0.6882208 0.8424831
## 49 0.3589216 0.03532058 0.28969453 0.4281487 0.5792981
## 50 0.3911986 0.12714698 0.14199514 0.6404021 0.5543793
## 51 0.2629380 0.11025555 0.04684107 0.4790349 0.6621669
## 52 0.8470109 0.02635796 0.79535023 0.8986715 0.9254371
## 53 0.6114610 0.17720180 0.26415188 0.9587702 0.6171623
## SE Lower 95% CI Upper 95% CI Factor correlation
## 1 0.093818456 0.52414099 0.8919026 0.4756440
## 2 0.005606417 0.90684544 0.9288222 0.8964628
## 3 0.116558945 0.45410435 0.9110070 -0.2555810
## 4 0.039229646 0.73576425 0.8895416 0.3046393
## 5 0.028867430 0.85595499 0.9691132 0.3155412
## 6 0.052077286 0.74678534 0.9509245 1.0626498
## 7 0.021633441 0.76128929 0.8460908 0.6051602
## 8 0.039679215 0.80121365 0.9567533 0.4926070
## 9 0.049017303 0.73223802 0.9243823 0.3944023
## 10 0.073562141 0.58399081 0.8723491 0.8396080
## 11 0.029530824 0.85123213 0.9669908 0.6654713
## 12 0.035981513 0.83259500 0.9736399 0.6492101
## 13 0.068628757 0.71924852 0.9882683 1.8872193
## 14 0.044358320 0.58054932 0.7544307 0.1409425
## 15 0.021161247 0.88047580 0.9634264 0.5991126
## 16 0.026490306 0.78966433 0.8935044 0.6792078
## 17 0.046748235 0.56995292 0.7532026 -0.3256691
## 18 0.006154373 0.83101841 0.8551431 0.5311592
## 19 0.092697734 0.63114733 0.9945158 0.8470350
## 20 0.024863841 0.77521808 0.8726825 0.9188289
## 21 0.037837034 0.70888135 0.8571998 0.4052477
## 22 0.008823746 0.85590243 0.8904909 0.1960032
## 23 0.006193187 0.87114352 0.8954204 0.6547173
## 24 0.011914261 0.75454131 0.8012444 0.2602787
## 25 0.006371702 0.85812707 0.8831037 0.2200001
## 26 0.022318890 0.79503245 0.8825209 0.4653317
## 27 0.020846923 0.79962097 0.8813394 0.4123712
## 28 0.311078740 -0.07632876 1.1430775 0.3290446
## 29 0.038487946 0.60872302 0.7595930 0.3713846
## 30 0.027128831 0.74268192 0.8490250 0.3269998
## 31 0.065195767 0.69026632 0.9458290 0.7106701
## 32 0.095298156 0.59554813 0.9691100 0.9965016
## 33 0.077288864 0.55375168 0.8567185 1.1226288
## 34 0.107921399 0.42401286 0.8470570 0.5035671
## 35 0.047824175 0.67431084 0.8617782 0.2400000
## 36 0.068962607 0.67233699 0.9426654 0.8649308
## 37 0.043676233 0.73423222 0.9054399 0.7635129
## 38 0.043337322 0.73310804 0.9029872 0.7106699
## 39 0.047857357 0.72207900 0.9096764 0.7013148
## 40 0.018905308 0.80221867 0.8763261 0.6336887
## 41 0.060289851 0.68291666 0.9192485 0.2600004
## 42 0.021163632 0.81599972 0.8989596 0.2800012
## 43 0.016376975 0.86013739 0.9243339 0.6807858
## 44 0.011747424 0.82937996 0.8754290 0.5602003
## 45 0.018001022 0.82032143 0.8908841 0.3064697
## 46 0.014981635 0.82878090 0.8875078 0.2958872
## 47 0.014734698 0.77602375 0.8337827 0.6221182
## 48 0.015013955 0.81305629 0.8719099 0.4256407
## 49 0.019721613 0.54064442 0.6179517 0.3286024
## 50 0.075349720 0.40669651 0.7020620 0.4286474
## 51 0.034014145 0.59550045 0.7288334 1.2698901
## 52 0.012212749 0.90150054 0.9493736 0.5317692
## 53 0.033075235 0.55233604 0.6819886 0.8782378## Write out the results
write.table(tfmc.srl.summary, "TwoFactorModel3-Reliability.txt", sep='\t')
## Note: The standard errors of the derived parameters are identical
## to the asymptotic sampling errors.2.1.2.4 Model 4:
Two-factor model with Res assigned to
Adapt
## Model 4
## Two-factor model with two correlated traits and Res assigned to Adapt
## Model estimation
tfm.fit.all <- lapply(1:k, FUN = tra_study_lavaan,
data = TTCT1,
para_names = para_names,
model = TwoFactorModel,
estimator = "ML",
fixed.x = FALSE)
head(tfm.fit.all, 5)## [[1]]
## [[1]]$coefs
## L1 L2 L3 L4 L5
## 0.5400006 1.0000000 0.7651308 0.3827568 0.2172019
##
## [[1]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0275242359 0.0087000184 0.001834831 0.0009178736 0.0005208637
## L2 0.0087000184 0.0161111333 0.003397832 0.0016997646 0.0009645616
## L3 0.0018348309 0.0033978323 0.098934551 -0.0177614703 -0.0084516627
## L4 0.0009178736 0.0016997646 -0.017761470 0.0517805816 0.0040403316
## L5 0.0005208637 0.0009645616 -0.008451663 0.0040403316 0.0488336913
##
## [[1]]$fit
## lavaan 0.6.16 ended normally after 43 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 30
##
## Model Test User Model:
##
## Test statistic 4.964
## Degrees of freedom 4
## P-value (Chi-square) 0.291
##
##
## [[2]]
## [[2]]$coefs
## L1 L2 L3 L4 L5
## 0.9862142 0.9632796 0.5600021 1.0000000 0.6000011
##
## [[2]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0008705508 0.0007793315 0.0003499718 0.0006249483 0.0003749692
## L2 0.0007793315 0.0009071488 0.0003418333 0.0006104153 0.0003662494
## L3 0.0003499718 0.0003418333 0.0014053266 0.0004666676 0.0002800007
## L4 0.0006249483 0.0006104153 0.0004666676 0.0008333332 0.0005000003
## L5 0.0003749692 0.0003662494 0.0002800007 0.0005000003 0.0013666619
##
## [[2]]$fit
## lavaan 0.6.16 ended normally after 70 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 599
##
## Model Test User Model:
##
## Test statistic 114.018
## Degrees of freedom 4
## P-value (Chi-square) 0.000
##
##
## [[3]]
## [[3]]$coefs
## L1 L2 L3 L4 L5
## 0.3900009 1.0000000 0.8000005 0.2500011 1.0000000
##
## [[3]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0656059346 0.0138461869 0.0009315682 0.0002911170 0.001164458
## L2 0.0138461869 0.0355029321 0.0023886292 0.0007464516 0.002985780
## L3 0.0009315682 0.0023886292 0.0482838580 0.0071006269 0.028402263
## L4 0.0002911170 0.0007464516 0.0071006269 0.0687871724 0.008875766
## L5 0.0011644579 0.0029857805 0.0284022630 0.0088757656 0.035502755
##
## [[3]]$fit
## lavaan 0.6.16 ended normally after 97 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 13
##
## Model Test User Model:
##
## Test statistic 6.752
## Degrees of freedom 4
## P-value (Chi-square) 0.150
##
##
## [[4]]
## [[4]]$coefs
## L1 L2 L3 L4 L5
## 0.9618307 0.8109534 0.4200000 0.3800001 1.0000000
##
## [[4]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0147972551 0.0016112889 0.0009307011 0.0008420632 0.002215956
## L2 0.0016112889 0.0152574253 0.0007847067 0.0007099730 0.001868350
## L3 0.0009307011 0.0007847067 0.0149433848 0.0013078335 0.003441668
## L4 0.0008420632 0.0007099730 0.0013078335 0.0152056072 0.003113891
## L5 0.0022159562 0.0018683503 0.0034416678 0.0031138911 0.008194451
##
## [[4]]$fit
## lavaan 0.6.16 ended normally after 54 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 60
##
## Model Test User Model:
##
## Test statistic 5.793
## Degrees of freedom 4
## P-value (Chi-square) 0.215
##
##
## [[5]]
## [[5]]$coefs
## L1 L2 L3 L4 L5
## 0.9836156 0.8743249 0.4899993 0.2100005 1.0000000
##
## [[5]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.022280592 0.015866532 0.006525511 0.002796660 0.013317397
## L2 0.015866532 0.025984911 0.005800456 0.002485920 0.011837689
## L3 0.006525511 0.005800456 0.035136823 0.002054426 0.009782957
## L4 0.002796660 0.002485920 0.002054426 0.039050023 0.004192714
## L5 0.013317397 0.011837689 0.009782957 0.004192714 0.019965261
##
## [[5]]$fit
## lavaan 0.6.16 ended normally after 62 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 24
##
## Model Test User Model:
##
## Test statistic 8.537
## Degrees of freedom 4
## P-value (Chi-square) 0.074## Extract relevant model parameters
## FIT INDICES
## Chi-squares and p values of the parameter-based models
tfm_model_fit <- t(sapply(tfm.fit.all, function(x) {
fitMeasures(x$fit, c("chisq", "pvalue", "ntotal",
"cfi", "srmr"))}))
row.names(tfm_model_fit) <- names(TTCT1$data)
options(scipen=50)
round(tfm_model_fit,4)## chisq pvalue ntotal cfi srmr
## Auth 4.9636 0.2911 30 0.9336 0.0983
## Barkul 114.0180 0.0000 599 0.9608 0.0288
## Berman 6.7520 0.1496 13 0.8056 0.1312
## Callans 5.7929 0.2152 60 0.9804 0.0720
## Carter 8.5375 0.0738 24 0.9294 0.1234
## Carter 2.0228 0.7316 24 1.0000 0.0333
## Chi 56.2894 0.0000 203 0.8077 0.1096
## Cho 7.8447 0.0974 24 0.9149 0.0717
## Cho 10.2295 0.0367 35 0.9062 0.0962
## Cockcroft 8.3021 0.0811 36 0.8723 0.0992
## Conway 0.8890 0.9261 25 1.0000 0.0184
## Crawford 4.1617 0.3846 21 0.9977 0.0513
## Digranes 6.6504 0.1556 17 0.9507 0.0747
## Fishkin 2.0369 0.7290 116 1.0000 0.0310
## Forsyth 10.3382 0.0351 45 0.9603 0.0653
## Garcia 13.5111 0.0090 95 0.9373 0.0476
## Gollmar 43.1792 0.0000 128 0.7400 0.1268
## Hokanson 283.9768 0.0000 1758 0.9149 0.0735
## Houtz 3.4475 0.4859 42 1.0000 0.0473
## Humble 8.2194 0.0839 125 0.9787 0.0538
## Ibrahim 42.7677 0.0000 99 0.7295 0.0975
## Kim_1 167.2994 0.0000 500 0.8649 0.0992
## Kim_2 192.5836 0.0000 1000 0.9270 0.0605
## Kim_2 77.5064 0.0000 1000 0.9491 0.0616
## Kim_2 236.0753 0.0000 1000 0.9024 0.0848
## Kim_3 64.7562 0.0000 125 0.7276 0.1798
## Kim_3 51.0455 0.0000 137 0.8072 0.1601
## Miranda 1.3888 0.8461 12 1.0000 0.0638
## Nguyen 5.9355 0.2040 187 0.9898 0.0384
## Zbarskaya 4.3965 0.3550 125 0.9970 0.0372
## Rose 7.9987 0.0916 19 0.8937 0.0882
## Rose 8.8010 0.0663 12 0.5348 0.1686
## Roskos 8.9535 0.0623 39 0.8469 0.0889
## Roskos 2.1785 0.7030 31 1.0000 0.0618
## Samuels 4.9267 0.2949 51 0.9836 0.0861
## Shore 17.9608 0.0013 18 0.2736 0.2218
## Stephens 1.8280 0.7674 84 1.0000 0.0280
## Storer 18.1023 0.0012 43 0.8557 0.0882
## Storer 10.3482 0.0350 46 0.9109 0.0710
## Tannehill 7.1674 0.1273 199 0.9916 0.0201
## Tisone 3.5012 0.4777 24 1.0000 0.0752
## Trigani 21.8630 0.0002 107 0.9087 0.0915
## Voss 16.1637 0.0028 120 0.9589 0.0549
## Warne 56.1459 0.0000 432 0.9356 0.0552
## Yoon 23.1916 0.0001 163 0.9369 0.0918
## Zhang 129.4983 0.0000 1067 0.9493 0.0806
## Acar 58.2737 0.0000 477 0.9180 0.0561
## Acaretal 72.5066 0.0000 319 0.8772 0.0887
## Balcia 87.3886 0.0000 1047 0.7544 0.0587
## Balcib 4.8293 0.3053 95 0.9785 0.0421
## Gao 10.6000 0.0314 264 0.9629 0.0356
## Liu 3.6785 0.4513 105 1.0000 0.0299
## Wan 28.6877 0.0000 375 0.8553 0.0558## Write out these indices
write.csv(round(tfm_model_fit,4), "TwoFactorModel4-Fit.csv")
write.table(round(tfm_model_fit,4), "TwoFactorModel4-Fit.txt", sep='\t')
head(tfm_model_fit, 5)## chisq pvalue ntotal cfi srmr
## Auth 4.963578 0.29105512 30 0.9335511 0.09825559
## Barkul 114.017957 0.00000000 599 0.9608442 0.02876380
## Berman 6.752014 0.14958800 13 0.8056406 0.13120072
## Callans 5.792855 0.21516132 60 0.9803705 0.07195184
## Carter 8.537495 0.07375882 24 0.9293966 0.12344312## Extract the scale reliability coefficients from the primary studies
## Point estimates
tfm.srl1 <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[14,5]}))
tfm.srl2 <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[15,5]}))
tfm.srlcr <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[16,5]}))
## Standard errors
tfm.srl1.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[14,6]}))
tfm.srl2.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[15,6]}))
tfm.srlcr.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[16,6]}))
## Lower 95% CI bound
tfm.srl1.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[14,9]}))
tfm.srl2.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[15,9]}))
tfm.srlcr.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[16,9]}))
## Upper 95% CI bound
tfm.srl1.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[14,10]}))
tfm.srl2.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[15,10]}))
tfm.srlcr.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[16,10]}))
## Factor correlation
tfm.fcorr <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[13,5]}))
## Save these statistics
tfm.srl.summary <- data.frame(names.studies,
TTCT1$n,
t(tfm.srl1 ),
t(tfm.srl1.se),
t(tfm.srl1.ci.low),
t(tfm.srl1.ci.upp),
t(tfm.srl2),
t(tfm.srl2.se),
t(tfm.srl2.ci.low),
t(tfm.srl2.ci.upp),
t(tfm.srlcr),
t(tfm.srlcr.se),
t(tfm.srlcr.ci.low),
t(tfm.srlcr.ci.upp),
t(tfm.fcorr))
colnames(tfm.srl.summary) <- c("Study",
"N",
"Omega Factor 1",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Omega Factor 2",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Composite reliability",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Factor correlation")
tfm.srl.summary## Study N Omega Factor 1 SE Lower 95% CI Upper 95% CI
## 1 Auth 30 0.7700003 0.073695681 0.62555945 0.9144412
## 2 Barkul 599 0.9744939 0.002080806 0.97041555 0.9785722
## 3 Berman 13 0.6950003 0.141042586 0.41856188 0.9714387
## 4 Callans 60 0.8827988 0.031949253 0.82017946 0.9454182
## 5 Carter_a 24 0.9279418 0.029174066 0.87076173 0.9851220
## 6 Carter_b 24 0.9099999 0.035050009 0.84130312 0.9786966
## 7 Chi 203 0.7165000 0.033685510 0.65047763 0.7825224
## 8 Cho_F 24 0.8050006 0.071425899 0.66500843 0.9449928
## 9 Cho_M 35 0.7476911 0.085012742 0.58106919 0.9143130
## 10 Cockcroft 36 0.5554435 0.188006684 0.18695714 0.9239298
## 11 Conway 25 0.9011436 0.039548341 0.82363032 0.9786570
## 12 Crawford 21 0.9700000 0.012895183 0.94472590 0.9952741
## 13 Digranes 17 0.8640321 0.065895715 0.73487891 0.9931854
## 14 Fishkin 116 0.7595687 0.052606411 0.65646200 0.8626753
## 15 Forsyth 45 0.9538664 0.013684640 0.92704499 0.9806878
## 16 Garcia 95 0.7710162 0.047072858 0.67875512 0.8632773
## 17 Gollmar 128 0.8359994 0.026507783 0.78404511 0.8879537
## 18 Hokanson 1758 0.8949419 0.005770018 0.88363288 0.9062509
## 19 Houtz 42 0.7823017 0.080033070 0.62543972 0.9391636
## 20 Humble 125 0.9132729 0.016844923 0.88025750 0.9462884
## 21 Ibrahim 99 0.7596899 0.048448655 0.66473228 0.8546475
## 22 Kim_1 500 0.9209904 0.007201180 0.90687639 0.9351045
## 23 Kim_2a 1000 0.9130688 0.005498046 0.90229281 0.9238448
## 24 Kim_2b 1000 0.8830057 0.007417871 0.86846695 0.8975445
## 25 Kim_2c 1000 0.9282173 0.004592202 0.91921677 0.9372179
## 26 Kim_3a 125 0.7300000 0.041266747 0.64911862 0.8108813
## 27 Kim_3b 137 0.7000003 0.042888383 0.61594059 0.7840600
## 28 Miranda 12 0.6674744 0.201382393 0.27277217 1.0621766
## 29 Nguyen 187 0.8623294 0.021582766 0.82002798 0.9046309
## 30 Zbarskaya 125 0.8070000 0.031014799 0.74621216 0.8677879
## 31 Rose 19 0.8959023 0.047807892 0.80220051 0.9896040
## 32 Rose_b 12 0.8350000 0.087049574 0.66438593 1.0056140
## 33 Roskos_Y 39 0.4299804 0.188549800 0.06042962 0.7995313
## 34 Roskos_O 31 0.3140816 0.300832780 -0.27553980 0.9037030
## 35 Samuels 51 0.8750002 0.032745994 0.81081927 0.9391812
## 36 Shore 18 0.6299999 0.138441342 0.35865989 0.9013400
## 37 Stephens 84 0.9262282 0.016156380 0.89456231 0.9578942
## 38 Storer_a 43 0.8959024 0.031779111 0.83361644 0.9581883
## 39 Storer_b 46 0.8439369 0.046020784 0.75373784 0.9341360
## 40 Tannehill 199 0.9161898 0.012441007 0.89180584 0.9405737
## 41 Tisone 24 0.8400000 0.059866512 0.72266381 0.9573362
## 42 Trigani 107 0.8600006 0.025102314 0.81080093 0.9092002
## 43 Voss 120 0.8378753 0.029424061 0.78020522 0.8955454
## 44 Warne 432 0.8564004 0.013961649 0.82903610 0.8837648
## 45 Yoon 163 0.8650001 0.019668799 0.82644992 0.9035502
## 46 Zhang 1067 0.8821890 0.007137126 0.86820044 0.8961775
## 47 Acar 477 0.7879230 0.019422465 0.74985562 0.8259903
## 48 Acaretal 319 0.8659268 0.015029260 0.83647000 0.8953836
## 49 Balcia 1047 0.6460001 0.017586365 0.61153148 0.6804688
## 50 Balcib 95 0.6047810 0.083199766 0.44171246 0.7678495
## 51 Gao 264 0.4732834 0.064834453 0.34621024 0.6003566
## 52 Liu 105 0.9692173 0.006039498 0.95738007 0.9810545
## 53 Wan 375 0.4282572 0.059613730 0.31141646 0.5450980
## Omega Factor 2 SE Lower 95% CI Upper 95% CI Composite reliability
## 1 0.45624463 0.17409530 0.11502411 0.7974652 0.6874343
## 2 0.77863942 0.01432646 0.75056007 0.8067188 0.9172739
## 3 0.76409048 0.10526202 0.55778071 0.9704002 0.6762777
## 4 0.65864387 0.06537841 0.53050455 0.7867832 0.8232824
## 5 0.62746975 0.11075333 0.41039721 0.8445423 0.8533497
## 6 0.60965463 0.13709507 0.34095323 0.8783560 0.8498451
## 7 0.73092237 0.03255818 0.66710951 0.7947352 0.8028454
## 8 0.78704960 0.07139832 0.64711146 0.9269877 0.8653867
## 9 0.53593998 0.12201920 0.29678675 0.7750932 0.7712868
## 10 0.64197605 0.09590072 0.45401408 0.8299380 0.7316390
## 11 0.83260782 0.05680361 0.72127479 0.9439409 0.9084899
## 12 0.75090462 0.08896888 0.57652882 0.9252804 0.9017108
## 13 0.74279966 0.10223453 0.54242367 0.9431757 0.8840397
## 14 0.13501415 0.13384830 -0.12732370 0.3973520 0.5451633
## 15 0.72290929 0.06714584 0.59130587 0.8545127 0.8942712
## 16 0.73147909 0.04742468 0.63852843 0.8244298 0.8370563
## 17 0.57767217 0.06370018 0.45282212 0.7025222 0.7373011
## 18 0.66848336 0.01363646 0.64175639 0.6952103 0.8356207
## 19 0.63128965 0.09716736 0.44084511 0.8217342 0.7921521
## 20 0.45739326 0.08042479 0.29976356 0.6150230 0.7720023
## 21 0.59575313 0.06800033 0.46247492 0.7290313 0.7734210
## 22 0.60299308 0.02887455 0.54640001 0.6595862 0.8330267
## 23 0.69120730 0.01660130 0.65866936 0.7237452 0.8681365
## 24 0.45982149 0.03034301 0.40035029 0.5192927 0.7583341
## 25 0.57274705 0.02227366 0.52909149 0.6164026 0.8274552
## 26 0.81414657 0.02790440 0.75945495 0.8688382 0.7397523
## 27 0.83913813 0.02259066 0.79486124 0.8834150 0.7616337
## 28 0.04610459 0.17784906 -0.30247317 0.3946823 0.4571518
## 29 0.22396085 0.10291225 0.02225656 0.4256651 0.6642839
## 30 0.70948779 0.03970823 0.63166108 0.7873145 0.7953206
## 31 0.55870035 0.14077852 0.28277951 0.8346212 0.8163590
## 32 0.59599430 0.16621832 0.27021238 0.9217762 0.7568527
## 33 0.64891319 0.09356723 0.46552478 0.8323016 0.7144250
## 34 0.66999465 0.08855887 0.49642245 0.8435668 0.6355428
## 35 0.63343611 0.07502833 0.48638329 0.7804889 0.7764478
## 36 0.68826146 0.12894239 0.43553903 0.9409839 0.5965216
## 37 0.55117737 0.08116852 0.39208999 0.7102647 0.8204643
## 38 0.55870048 0.09357911 0.37528880 0.7421122 0.8163591
## 39 0.53121564 0.11713183 0.30164147 0.7607898 0.7952131
## 40 0.65969490 0.04177264 0.57782203 0.7415678 0.8386434
## 41 0.62199784 0.12765061 0.37180724 0.8721884 0.7710646
## 42 0.62354154 0.05290719 0.51984535 0.7272377 0.8171664
## 43 0.74098227 0.03907849 0.66438983 0.8175747 0.8692816
## 44 0.65742730 0.02780843 0.60292377 0.7119308 0.8352745
## 45 0.79446255 0.02747778 0.74060709 0.8483180 0.8557819
## 46 0.38759036 0.03173293 0.32539496 0.4497858 0.7712278
## 47 0.61657373 0.02990038 0.55797007 0.6751774 0.7961029
## 48 0.56882185 0.04065881 0.48913204 0.6485117 0.8033695
## 49 0.38402612 0.03195386 0.32139770 0.4466545 0.5647673
## 50 0.32904399 0.10989756 0.11364873 0.5444392 0.5565458
## 51 0.52320575 0.05035568 0.42451043 0.6219011 0.6757915
## 52 0.84682712 0.02577615 0.79630679 0.8973475 0.9254409
## 53 0.49301651 0.04512961 0.40456410 0.5814689 0.6127853
## SE Lower 95% CI Upper 95% CI Factor correlation
## 1 0.089172197 0.51266000 0.8622086 0.5250131
## 2 0.005562633 0.90637138 0.9281765 0.8720216
## 3 0.114536395 0.45179054 0.9007650 -0.2899996
## 4 0.034783979 0.75510706 0.8914578 0.5302387
## 5 0.047287275 0.76066833 0.9460310 0.8234923
## 6 0.050779068 0.75031999 0.9493703 1.0336723
## 7 0.021735104 0.76024536 0.8454454 0.5716607
## 8 0.043172322 0.78077054 0.9500029 0.7090588
## 9 0.061062399 0.65160666 0.8909669 0.9957951
## 10 0.072698823 0.58915192 0.8741261 0.7845056
## 11 0.029817101 0.85004946 0.9669303 0.6827169
## 12 0.035560859 0.83201280 0.9714088 0.7643346
## 13 0.045706132 0.79445729 0.9736220 1.0398209
## 14 0.070251030 0.40747384 0.6828528 0.6704365
## 15 0.026008062 0.84329635 0.9452461 0.8915926
## 16 0.027244187 0.78365869 0.8904539 0.7791342
## 17 0.036354007 0.66604852 0.8085536 0.2946672
## 18 0.006418958 0.82303978 0.8482016 0.6169859
## 19 0.051780409 0.69066433 0.8936398 0.7257213
## 20 0.033500218 0.70634311 0.8376615 0.6891124
## 21 0.036703874 0.70148273 0.8453593 0.7498885
## 22 0.012075119 0.80935995 0.8566935 0.7292714
## 23 0.006883575 0.85464495 0.8816281 0.8568125
## 24 0.012683112 0.73347561 0.7831925 0.6521377
## 25 0.008944629 0.80992409 0.8449864 0.7642222
## 26 0.032991000 0.67509111 0.8044135 -0.2595952
## 27 0.028219152 0.70632520 0.8169422 -0.2178581
## 28 0.225939266 0.01431894 0.8999846 0.6841114
## 29 0.041725233 0.58250396 0.7460639 0.8047517
## 30 0.026779849 0.74283305 0.8478081 0.3269999
## 31 0.064886960 0.68918285 0.9435351 0.7107625
## 32 0.100135516 0.56059070 0.9531147 0.3400007
## 33 0.074331273 0.56873837 0.8601116 0.9707069
## 34 0.105000135 0.42974630 0.8413393 0.5037385
## 35 0.045035613 0.68817960 0.8647160 0.2399999
## 36 0.136849912 0.32830071 0.8647425 -0.2906585
## 37 0.032228515 0.75729762 0.8836311 0.7700793
## 38 0.043132015 0.73182188 0.9008963 0.7107627
## 39 0.049547402 0.69810200 0.8923242 0.9521721
## 40 0.018983592 0.80143623 0.8758505 0.6041018
## 41 0.072799427 0.62838038 0.9137489 0.3688393
## 42 0.026912805 0.76441824 0.8699145 0.6899985
## 43 0.019220322 0.83161048 0.9069528 0.9260633
## 44 0.012890119 0.81001034 0.8605387 0.8128623
## 45 0.017943182 0.82061395 0.8909499 0.3176184
## 46 0.012018395 0.74767218 0.7947834 1.0418061
## 47 0.015185967 0.76633894 0.8258668 0.8029838
## 48 0.017997045 0.76809598 0.8386431 0.7857492
## 49 0.019922710 0.52571955 0.6038151 0.3553703
## 50 0.070746154 0.41788587 0.6952057 0.4998183
## 51 0.032542279 0.61200981 0.7395732 1.0997318
## 52 0.012211446 0.90150695 0.9493749 0.5310279
## 53 0.032670096 0.54875314 0.6768176 0.8372851## Write out the results
write.table(tfm.srl.summary, "TwoFactorModel4-Reliability.txt", sep='\t')
## Note: The standard errors of the derived parameters are identical
## to the asymptotic sampling errors.2.2 Stage-2 Analysis: Meta-Analysis of the Reliability Coefficients
2.2.1 Data Preparation
## Create effect size and study identifiers
esid <- as.vector(seq(1:length(TTCT1$data)))
studid <- as.vector(as.numeric(as.factor(names.studid)))
## Create a meta-analytic data set with effect sizes,
## sampling variances, and moderators
srel.meta1 <- data.frame(esid, studid, names.studies,
TTCT1$n,
t(tfm.srl1), t(tfm.srl1.se),
t(tfm.srl2), t(tfm.srl2.se),
t(tfm.srlcr), t(tfm.srlcr.se),
t(tfmc.srl1), t(tfmc.srl1.se),
t(tfmc.srl2), t(tfmc.srl2.se),
t(tfmc.srlcr), t(tfmc.srlcr.se),
t(tfm.fcorr), t(tfmc.fcorr),
mod.adults, mod.forms,
mod.scores, mod.validity)
## Column names
colnames(srel.meta1) <- c("ESID", "STUDID", "Reference",
"N",
"SREL1", "SE.SREL1",
"SREL2", "SE.SREL2",
"SRELCR", "SE.SRELCR",
"SREL1C", "SE.SREL1C",
"SREL2C", "SE.SREL2C",
"SRELCRC", "SE.SRELCRC",
"Fcorr", "FcorrC",
"Adults", "Forms",
"Scores", "Validity")
## Create sampling variances
srel.meta1$SREL1.vg <- srel.meta1$SE.SREL1^2
srel.meta1$SREL2.vg <- srel.meta1$SE.SREL2^2
srel.meta1$SRELCR.vg <- srel.meta1$SE.SRELCR^2
srel.meta1$SREL1C.vg <- srel.meta1$SE.SREL1C^2
srel.meta1$SREL2C.vg <- srel.meta1$SE.SREL2C^2
srel.meta1$SRELCRC.vg <- srel.meta1$SE.SRELCRC^2
## Identify the studies with poor model fit
tfm_model_fit## chisq pvalue ntotal cfi srmr
## Auth 4.9635776 0.2910551211913343649940 30 0.9335511 0.09825559
## Barkul 114.0179567 0.0000000000000000000000 599 0.9608442 0.02876380
## Berman 6.7520142 0.1495880041679560346068 13 0.8056406 0.13120072
## Callans 5.7928546 0.2151613150101536753311 60 0.9803705 0.07195184
## Carter 8.5374948 0.0737588150887573679171 24 0.9293966 0.12344312
## Carter 2.0227827 0.7315683246859836508236 24 1.0000000 0.03325786
## Chi 56.2893634 0.0000000000174372738471 203 0.8077218 0.10964189
## Cho 7.8447267 0.0974345320442262519123 24 0.9149484 0.07170580
## Cho 10.2295462 0.0367335260865379042983 35 0.9061654 0.09617379
## Cockcroft 8.3021446 0.0811166222855801422043 36 0.8723006 0.09924377
## Conway 0.8889755 0.9261371061244291214010 25 1.0000000 0.01843786
## Crawford 4.1616946 0.3845649220312452598591 21 0.9976996 0.05125261
## Digranes 6.6504236 0.1555558127019357472776 17 0.9506538 0.07473993
## Fishkin 2.0368505 0.7289809937310370857588 116 1.0000000 0.03097716
## Forsyth 10.3381919 0.0351003685949631272223 45 0.9602724 0.06526681
## Garcia 13.5111291 0.0090304420932041384873 95 0.9372565 0.04764523
## Gollmar 43.1791907 0.0000000094987471221941 128 0.7400054 0.12680085
## Hokanson 283.9767705 0.0000000000000000000000 1758 0.9149149 0.07349493
## Houtz 3.4474977 0.4859062122440924724742 42 1.0000000 0.04730142
## Humble 8.2194499 0.0838622784140150567822 125 0.9787311 0.05375296
## Ibrahim 42.7677064 0.0000000115623308705182 99 0.7294550 0.09754886
## Kim_1 167.2993976 0.0000000000000000000000 500 0.8649442 0.09923085
## Kim_2 192.5835800 0.0000000000000000000000 1000 0.9270304 0.06051507
## Kim_2 77.5063974 0.0000000000000005551115 1000 0.9490579 0.06163331
## Kim_2 236.0752688 0.0000000000000000000000 1000 0.9024048 0.08477319
## Kim_3 64.7562446 0.0000000000002896571871 125 0.7275914 0.17983244
## Kim_3 51.0455158 0.0000000002183865310812 137 0.8072334 0.16014229
## Miranda 1.3887585 0.8461464699435590208765 12 1.0000000 0.06378383
## Nguyen 5.9355052 0.2040168367815882177752 187 0.9897562 0.03840829
## Zbarskaya 4.3964860 0.3549986090242673864026 125 0.9969846 0.03721615
## Rose 7.9986995 0.0916258465548294154246 19 0.8936954 0.08815120
## Rose 8.8009741 0.0662713307274384044732 12 0.5347775 0.16864982
## Roskos 8.9535133 0.0622719885770935954739 39 0.8469183 0.08886656
## Roskos 2.1784895 0.7029692688101858433214 31 1.0000000 0.06181563
## Samuels 4.9266837 0.2949032714564562285275 51 0.9835969 0.08610035
## Shore 17.9607613 0.0012560801072869498540 18 0.2735955 0.22179542
## Stephens 1.8279847 0.7673585947114496264021 84 1.0000000 0.02798716
## Storer 18.1023205 0.0011785479448310764994 43 0.8557475 0.08815127
## Storer 10.3481802 0.0349537821149176197721 46 0.9109395 0.07096107
## Tannehill 7.1674124 0.1273013297674244759250 199 0.9915642 0.02005184
## Tisone 3.5011952 0.4776966312815904558420 24 1.0000000 0.07522753
## Trigani 21.8629651 0.0002134085732177215533 107 0.9087319 0.09152566
## Voss 16.1637294 0.0028071510715869107955 120 0.9589320 0.05493900
## Warne 56.1459071 0.0000000000186879400843 432 0.9355730 0.05516412
## Yoon 23.1916233 0.0001159388910124681260 163 0.9369490 0.09184230
## Zhang 129.4982963 0.0000000000000000000000 1067 0.9492744 0.08064859
## Acar 58.2737384 0.0000000000066852079428 477 0.9179879 0.05611845
## Acaretal 72.5065989 0.0000000000000066613381 319 0.8771856 0.08868595
## Balcia 87.3885778 0.0000000000000000000000 1047 0.7544349 0.05869789
## Balcib 4.8292585 0.3052694832740436847729 95 0.9784702 0.04205625
## Gao 10.6000200 0.0314467764778070257847 264 0.9629362 0.03563730
## Liu 3.6784591 0.4512666752414976611973 105 1.0000000 0.02992893
## Wan 28.6876679 0.0000090466118810583751 375 0.8553399 0.05579743### EXCLUSIONS DUE TO POOR MODEL FIT
## Exclude samples with at least two fails on the following three criteria:
## Significant chi-square value (p>.05), CFI >= .90, SRMR <= .10
fitexclude <- c(3,7,17,21,22,26,27,32,36,38, 49, 53)
## Data exclusions due to poor model fit
srel.meta <- srel.meta1[-fitexclude,]
## ADD NEW MODERATORS (see Table 1 in the manuscript)
srel.meta$LanguageEnglish <- c(1,0,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,0,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,0,1,0,
1,1,0,0,0)
srel.meta$Culture <- c("Western","Eastern","Western","Western",
"Western","Western","Western","African",
"Western","Western","Western","Western",
"Western","Western","Western","Western",
"African","Western","Western","Western",
"Western","Mixed","Western","Western",
"Western","Western","Western","Western",
"Western","Western","Western","Western",
"Western","Eastern","Western","Eastern",
"Mixed","Western","Eastern","Eastern",
"Eastern")
table(srel.meta$Culture)##
## African Eastern Mixed Western
## 2 6 2 31srel.meta$CultureWestern <- c(1,0,1,1,1,1,1,0,1,1,1,1,1,
1,1,1,0,1,1,1,1,0,1,1,1,1,
1,1,1,1,1,1,1,0,1,0,0,1,0,
0,0)
srel.meta$PropFemale <- c(53,62.1,75,54.17,79.17,100,0,
27.8,60,52,47.05,38.79,NA,NA,
50,69.05,54,51.27,51.27,51.27,
58.33,70.59,75,49,73,94.12,45,
50,50,50,100,46,75,39.9,100,
44.7,69.83,52.94,52,31.66,47.94)
## FURTHER EXCLUSIONS DUE TO SMALL SAMPLE SIZE
## Some power analysis to set a criterion
## Source for the size of parameters: Said-Metwaly et al. (2018), https://doi.org/10.1080/10400419.2018.1530534
## Run CFA power analysis
## Source: https://cran.r-project.org/web/packages/semPower/semPower.pdf
powerCFA <- semPower.powerCFA(type = 'a-priori',
Phi = .6,
loadings = list(c(.7, .8, .1),
c(.6, .6, .6)),
alpha = .05,
beta = .05)
summary(powerCFA)##
## semPower: A priori power analysis
##
## F0 0.181217
## RMSEA 0.425696
## Mc 0.913375
##
## df 1
## Required Num Observations 73
##
## Critical Chi-Square 3.841459
## NCP 13.04764
## Alpha 0.050000
## Beta 0.049248
## Power (1 - Beta) 0.950752
## Implied Alpha/Beta Ratio 1.015269
## Criterion: N < 75 for the exclusion
which(srel.meta$N < 75)## [1] 1 3 4 5 6 7 8 9 10 11 13 16 21 24 25 26 27 29 31length(which(srel.meta$N < 75))## [1] 19srel.meta <- srel.meta[!(srel.meta$N < 75),]
## Inspect the data again
head(srel.meta, 5)## ESID STUDID Reference N SREL1 SE.SREL1 SREL2 SE.SREL2
## 2 2 6 Barkul 599 0.9744939 0.002080806 0.7786394 0.01432646
## 14 14 16 Fishkin 116 0.7595687 0.052606411 0.1350142 0.13384830
## 16 16 19 Garcia 95 0.7710162 0.047072858 0.7314791 0.04742468
## 18 18 21 Hokanson 1758 0.8949419 0.005770018 0.6684834 0.01363646
## 20 20 23 Humble 125 0.9132729 0.016844923 0.4573933 0.08042479
## SRELCR SE.SRELCR SREL1C SE.SREL1C SREL2C SE.SREL2C SRELCRC
## 2 0.9172739 0.005562633 0.787772957 0.028512584 0.8229706 0.02047607 0.9178338
## 14 0.5451633 0.070251030 0.669786264 0.051302281 0.4145217 0.06674312 0.6674900
## 16 0.8370563 0.027244187 0.735378067 0.051965699 0.7041132 0.05642252 0.8415844
## 18 0.8356207 0.006418958 0.804897564 0.008142107 0.6345411 0.01541796 0.8430808
## 20 0.7720023 0.033500218 0.001128471 0.106290290 0.8493046 0.07297606 0.8239503
## SE.SRELCRC Fcorr FcorrC Adults Forms Scores Validity SREL1.vg
## 2 0.005606417 0.8720216 0.8964628 1 A 0 0 0.000004329754
## 14 0.044358320 0.6704365 0.1409425 0 A 0 1 0.002767434440
## 16 0.026490306 0.7791342 0.6792078 0 B 1 0 0.002215853937
## 18 0.006154373 0.6169859 0.5311592 0 A 0 0 0.000033293103
## 20 0.024863841 0.6891124 0.9188289 0 A 1 0 0.000283751417
## SREL2.vg SRELCR.vg SREL1C.vg SREL2C.vg SRELCRC.vg
## 2 0.0002052475 0.00003094289 0.00081296742 0.0004192693 0.00003143191
## 14 0.0179153687 0.00493520728 0.00263192408 0.0044546442 0.00196766057
## 16 0.0022491001 0.00074224575 0.00270043388 0.0031835005 0.00070173632
## 18 0.0001859530 0.00004120302 0.00006629391 0.0002377136 0.00003787630
## 20 0.0064681472 0.00112226459 0.01129762564 0.0053255048 0.00061821057
## LanguageEnglish Culture CultureWestern PropFemale
## 2 0 Eastern 0 62.10
## 14 1 Western 1 38.79
## 16 1 Western 1 NA
## 18 1 Western 1 50.00
## 20 0 African 0 54.002.2.2 Description of the Meta-Analytic Data
## Number of effect sizes
length(srel.meta$ESID)## [1] 22## Number of primary studies
length(unique(srel.meta$STUDID))## [1] 20## Proportion adult samples
prop.table(table(srel.meta$Adults))##
## 0 1
## 0.6818182 0.3181818## Proportion evidence against validity
prop.table(table(srel.meta$Validity))##
## 0 1
## 0.6818182 0.3181818## Distribution of test forms
prop.table(table(srel.meta$Forms))##
## A B Both
## 0.85 0.10 0.05## Distribution of test score types
prop.table(table(srel.meta$Scores))##
## 0 1
## 0.7 0.3## Distribution of the sample sizes
psych::describe(srel.meta$N)## vars n mean sd median trimmed mad min max range skew kurtosis
## X1 1 22 428.95 454.25 193 357.39 145.29 84 1758 1674 1.37 0.97
## se
## X1 96.85sum(unique(srel.meta$N))## [1] 7217## Distribution of English language
prop.table(table(srel.meta$LanguageEnglish))##
## 0 1
## 0.3181818 0.6818182## Distribution of cultures
prop.table(table(srel.meta$Culture))##
## African Eastern Mixed Western
## 0.04545455 0.27272727 0.09090909 0.59090909prop.table(table(srel.meta$CultureWestern))##
## 0 1
## 0.4090909 0.5909091## Distribution of the proportion of women in the samples
psych::describe(srel.meta$PropFemale)## vars n mean sd median trimmed mad min max range skew kurtosis se
## X1 1 21 55.44 15.39 51.27 54.05 7.81 31.66 100 68.34 1.11 1.09 3.36The meta-analytic data are hierarchical with multiple effect sizes derived from independent samples that are nested in primary studies.
2.2.3 Meta-Analysis of the Two-Factor Model without Cross-Loadings
2.2.3.1 Meta-Analytic Baseline Models
## Reliability of factor 1
## Three-level REM
## Model specification and estimation
MLREMCSC.tfm.srel1 <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfm.srel1, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 19.5773 -39.1546 -33.1546 -30.0210 -31.7428
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0079 0.0887 20 no STUDID
## sigma^2.2 0.0005 0.0221 22 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 21) = 799.8024, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.8508 0.0214 39.7561 21 <.0001 0.8063 0.8953 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfm.srel1, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0079 0.0000 0.0197
## sigma.1 0.0887 0.0000 0.1405
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0005 0.0001 0.0101
## sigma.2 0.0221 0.0099 0.1003## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfm.srel1))## % of total variance I2
## Level 1 0.8079376 ---
## Level 2 5.8222292 5.82
## Level 3 93.3698332 93.37
## Total I2: 99.19%plot(dmetar::var.comp(MLREMCSC.tfm.srel1))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfm.srel1, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.851 0.0213 18.6 0.806 0.895# Profile plots
profile.rma.mv(MLREMCSC.tfm.srel1, sigma2=1)
profile.rma.mv(MLREMCSC.tfm.srel1, sigma2=2)
## Reliability of factor 2
## Three-level REM
## Model specification and estimation
MLREMCSC.tfm.srel2 <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfm.srel2, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 7.1266 -14.2532 -8.2532 -5.1196 -6.8414
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0232 0.1522 22 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 21) = 351.2706, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.5977 0.0343 17.4048 21 <.0001 0.5263 0.6691 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfm.srel2, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0000 0.0000 0.0438
## sigma.1 0.0000 0.0000 0.2094
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0232 0.0027 0.0511
## sigma.2 0.1522 0.0522 0.2260## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfm.srel2))## % of total variance I2
## Level 1 3.5719489094137 ---
## Level 2 96.4280509097080 96.43
## Level 3 0.0000001808782 0
## Total I2: 96.43%plot(dmetar::var.comp(MLREMCSC.tfm.srel2))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfm.srel2, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.598 0.0334 15.4 0.527 0.669# Profile plots
profile.rma.mv(MLREMCSC.tfm.srel2, sigma2=1)
profile.rma.mv(MLREMCSC.tfm.srel2, sigma2=2)
## Composite reliability
## Three-level REM
## Model specification and estimation
MLREMCSC.tfm.srelcr <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfm.srelcr, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 20.9723 -41.9446 -35.9446 -32.8110 -34.5328
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0056 0.0749 22 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 21) = 416.1807, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.8027 0.0169 47.4154 21 <.0001 0.7675 0.8379 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfm.srelcr, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0000 0.0000 0.0117
## sigma.1 0.0000 0.0000 0.1080
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0056 0.0006 0.0130
## sigma.2 0.0749 0.0255 0.1139## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfm.srelcr))## % of total variance I2
## Level 1 2.966847478574 ---
## Level 2 97.033151498557 97.03
## Level 3 0.000001022869 0
## Total I2: 97.03%plot(dmetar::var.comp(MLREMCSC.tfm.srelcr))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfm.srelcr, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.803 0.0165 15.3 0.768 0.838# Profile plots
profile.rma.mv(MLREMCSC.tfm.srel1, sigma2=1)
profile.rma.mv(MLREMCSC.tfm.srel1, sigma2=2)
2.2.3.2 Moderator Analyses for the Reliability of Factor 1
2.2.3.2.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.age <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.5870 -37.1739 -29.1739 -25.1910 -26.5073
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0081 0.0900 20 no STUDID
## sigma^2.2 0.0005 0.0222 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 488.7636, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.8066, p-val = 0.3798
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8648 0.0269 32.1288 20 <.0001 0.8087 0.9210 ***
## Adults -0.0408 0.0454 -0.8981 20 0.3798 -0.1355 0.0539
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8648 0.0210 11.7 0.819 0.911
## Adults -0.0408 0.0524 12.2 -0.155 0.073## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8648 0.0269 0.8087 0.9210 0.6635 1.0661
## 2 0.8240 0.0366 0.7477 0.9003 0.6162 1.03192.2.3.2.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.validity <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.8885 -37.7770 -29.7770 -25.7941 -27.1103
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0076 0.0872 20 no STUDID
## sigma^2.2 0.0005 0.0222 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 662.9947, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 1.4415, p-val = 0.2439
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8690 0.0258 33.6579 20 <.0001 0.8151 0.9228 ***
## Validity -0.0537 0.0447 -1.2006 20 0.2439 -0.1470 0.0396
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8690 0.0273 11.9 0.809 0.9286
## Validity -0.0537 0.0407 11.6 -0.143 0.0354## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8690 0.0258 0.8151 0.9228 0.6738 1.0641
## 2 0.8153 0.0365 0.7391 0.8915 0.6128 1.01782.2.3.2.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.forms <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfm.srel1.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 26.7174 -53.4348 -43.4348 -39.2687 -37.9803
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0018 0.0424 18 no STUDID
## sigma^2.2 0.0005 0.0226 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 17) = 663.1370, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 17) = 4.5983, p-val = 0.0253
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8783 0.0133 65.9135 17 <.0001 0.8502 0.9064 ***
## factor(Forms)B -0.0523 0.0423 -1.2364 17 0.2331 -0.1415 0.0369
## factor(Forms)Both -0.2735 0.0970 -2.8200 17 0.0118 -0.4781 -0.0689 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8783 0.0133 13.53 0.850 0.907
## factor(Forms)B -0.0523 0.0430 1.23 -0.407 0.302
## factor(Forms)Both -0.2735 0.0133 13.53 -0.302 -0.245## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8783 0.0133 0.8502 0.9064 0.7731 0.9834
## 2 0.8260 0.0401 0.7413 0.9107 0.6940 0.9580
## 3 0.6048 0.0961 0.4021 0.8074 0.3782 0.83142.2.3.2.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.scores <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 23.8397 -47.6794 -39.6794 -36.1179 -36.6025
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0027 0.0521 18 no STUDID
## sigma^2.2 0.0005 0.0222 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 18) = 688.2868, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 18) = 0.1153, p-val = 0.7382
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8622 0.0177 48.6442 18 <.0001 0.8250 0.8995 ***
## Scores 0.0105 0.0308 0.3395 18 0.7382 -0.0543 0.0753
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8622 0.0193 10.29 0.8195 0.9050
## Scores 0.0105 0.0270 9.78 -0.0499 0.0708## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8622 0.0177 0.8250 0.8995 0.7375 0.9870
## 2 0.8727 0.0252 0.8197 0.9258 0.7424 1.00302.2.3.2.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.lang <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.2426 -36.4852 -28.4852 -24.5023 -25.8186
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0088 0.0936 20 no STUDID
## sigma^2.2 0.0005 0.0221 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 428.9993, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.0803, p-val = 0.7799
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8409 0.0386 21.7593 20 <.0001 0.7603 0.9215 ***
## LanguageEnglish 0.0135 0.0475 0.2833 20 0.7799 -0.0856 0.1125
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8409 0.0605 5.77 0.691 0.99
## LanguageEnglish 0.0135 0.0623 11.68 -0.123 0.15## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8409 0.0386 0.7603 0.9215 0.6246 1.0572
## 2 0.8543 0.0276 0.7968 0.9119 0.6455 1.06312.2.3.2.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.cult <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.3337 -36.6674 -28.6674 -24.6845 -26.0008
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0087 0.0932 20 no STUDID
## sigma^2.2 0.0005 0.0221 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 491.9159, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.2513, p-val = 0.6216
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8373 0.0336 24.8854 20 <.0001 0.7671 0.9075 ***
## CultureWestern 0.0226 0.0450 0.5013 20 0.6216 -0.0713 0.1165
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8373 0.0462 7.76 0.7302 0.944
## CultureWestern 0.0226 0.0489 16.72 -0.0807 0.126## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8373 0.0336 0.7671 0.9075 0.6255 1.0491
## 2 0.8598 0.0299 0.7974 0.9223 0.6505 1.06922.2.3.2.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfm.srel1.female <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfm.srel1.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 21; method: REML)
##
## logLik Deviance AIC BIC AICc
## 17.3263 -34.6526 -26.6526 -22.8748 -23.7955
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0089 0.0941 19 no STUDID
## sigma^2.2 0.0005 0.0221 21 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 19) = 674.2991, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 19) = 0.3759, p-val = 0.5471
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.7942 0.0992 8.0053 19 <.0001 0.5865
## asin(sqrt(PropFemale/100)) 0.0683 0.1114 0.6131 19 0.5471 -0.1649
## ci.ub
## intrcpt 1.0018 ***
## asin(sqrt(PropFemale/100)) 0.3015
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7942 0.115 3.18 0.438 1.150
## asin(sqrt(PropFemale/100)) 0.0683 0.117 2.23 -0.389 0.5262.2.3.3 Moderator Analyses for the Reliability of Factor 2
2.2.3.3.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.age <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.5433 -13.0867 -5.0867 -1.1037 -2.4200
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0237 0.1539 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 303.9563, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.6440, p-val = 0.4317
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.5780 0.0423 13.6784 20 <.0001 0.4899 0.6662 ***
## Adults 0.0594 0.0740 0.8025 20 0.4317 -0.0949 0.2137
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.5780 0.0391 9.07 0.490 0.666
## Adults 0.0594 0.0745 11.32 -0.104 0.223## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.5780 0.0423 0.4899 0.6662 0.2451 0.9110
## 2 0.6374 0.0607 0.5107 0.7641 0.2923 0.98252.2.3.3.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.validity <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.5082 -13.0164 -5.0164 -1.0335 -2.3497
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0244 0.1562 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 341.9569, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.5222, p-val = 0.4783
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6138 0.0421 14.5761 20 <.0001 0.5260 0.7016 ***
## Validity -0.0553 0.0765 -0.7227 20 0.4783 -0.2147 0.1042
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6138 0.0411 9.36 0.521 0.706
## Validity -0.0553 0.0751 10.68 -0.221 0.111## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6138 0.0421 0.5260 0.7016 0.2764 0.9512
## 2 0.5585 0.0638 0.4254 0.6917 0.2066 0.91052.2.3.3.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.forms <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfm.srel2.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.4594 -12.9189 -2.9189 1.2472 2.5357
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 18 no STUDID
## sigma^2.2 0.0208 0.1441 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 17) = 285.3357, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 17) = 1.5179, p-val = 0.2474
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.5874 0.0371 15.8368 17 <.0001 0.5092 0.6657 ***
## factor(Forms)B 0.1058 0.1118 0.9461 17 0.3573 -0.1301 0.3417
## factor(Forms)Both -0.2584 0.1850 -1.3969 17 0.1804 -0.6487 0.1319
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.587 0.0365 10.86 0.507 0.668
## factor(Forms)B 0.106 0.0520 1.26 -0.305 0.517
## factor(Forms)Both -0.258 0.0365 10.86 -0.339 -0.178## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.5874 0.0371 0.5092 0.6657 0.2735 0.9014
## 2 0.6932 0.1055 0.4707 0.9158 0.3165 1.0700
## 3 0.3290 0.1812 -0.0533 0.7114 -0.1594 0.81752.2.3.3.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.scores <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 5.8309 -11.6617 -3.6617 -0.1002 -0.5848
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 18 no STUDID
## sigma^2.2 0.0235 0.1533 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 18) = 290.6924, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 18) = 0.0037, p-val = 0.9523
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.5890 0.0429 13.7341 18 <.0001 0.4989 0.6792 ***
## Scores -0.0049 0.0807 -0.0607 18 0.9523 -0.1744 0.1646
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.5890 0.0379 8.39 0.502 0.676
## Scores -0.0049 0.0879 8.77 -0.204 0.195## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.5890 0.0429 0.4989 0.6792 0.2546 0.9235
## 2 0.5841 0.0683 0.4406 0.7277 0.2315 0.93682.2.3.3.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.lang <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.2629 -12.5258 -4.5258 -0.5429 -1.8592
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0247 0.1571 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 330.0254, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.0545, p-val = 0.8178
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.5848 0.0628 9.3173 20 <.0001 0.4538 0.7157 ***
## LanguageEnglish 0.0177 0.0759 0.2335 20 0.8178 -0.1407 0.1762
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.5848 0.0744 5.9 0.402 0.768
## LanguageEnglish 0.0177 0.0832 11.1 -0.165 0.201## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.5848 0.0628 0.4538 0.7157 0.2319 0.9377
## 2 0.6025 0.0428 0.5133 0.6917 0.2628 0.94222.2.3.3.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.cult <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.6749 -13.3498 -5.3498 -1.3668 -2.6831
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0044 0.0664 20 no STUDID
## sigma^2.2 0.0195 0.1395 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 341.0787, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.9487, p-val = 0.3417
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.5574 0.0550 10.1386 20 <.0001 0.4427 0.6720 ***
## CultureWestern 0.0702 0.0720 0.9740 20 0.3417 -0.0801 0.2204
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.5574 0.0671 7.83 0.4020 0.713
## CultureWestern 0.0702 0.0767 15.76 -0.0925 0.233## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.5574 0.0550 0.4427 0.6720 0.2152 0.8995
## 2 0.6275 0.0465 0.5304 0.7246 0.2908 0.96422.2.3.3.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfm.srel2.female <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfm.srel2.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 21; method: REML)
##
## logLik Deviance AIC BIC AICc
## 7.0886 -14.1772 -6.1772 -2.3995 -3.3201
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 19 no STUDID
## sigma^2.2 0.0212 0.1454 21 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 19) = 302.3661, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 19) = 3.0541, p-val = 0.0967
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.3421 0.1472 2.3237 19 0.0314 0.0340
## asin(sqrt(PropFemale/100)) 0.2913 0.1667 1.7476 19 0.0967 -0.0576
## ci.ub
## intrcpt 0.6502 *
## asin(sqrt(PropFemale/100)) 0.6402 .
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.342 0.0980 3.01 0.031 0.653
## asin(sqrt(PropFemale/100)) 0.291 0.0985 2.15 -0.105 0.6882.2.3.4 Moderator Analyses for the Composite Reliability
2.2.3.4.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.age <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 19.3912 -38.7823 -30.7823 -26.7994 -28.1157
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0018 0.0427 20 no STUDID
## sigma^2.2 0.0044 0.0665 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 340.6877, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.0841, p-val = 0.7748
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7971 0.0226 35.2411 20 <.0001 0.7499 0.8443 ***
## Adults 0.0112 0.0385 0.2900 20 0.7748 -0.0691 0.0914
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7971 0.0209 10.8 0.751 0.8431
## Adults 0.0112 0.0405 12.1 -0.077 0.0993## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7971 0.0226 0.7499 0.8443 0.6255 0.9687
## 2 0.8083 0.0311 0.7434 0.8731 0.6310 0.98552.2.3.4.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.validity <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 20.3844 -40.7687 -32.7687 -28.7858 -30.1020
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0057 0.0753 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 382.4132, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 2.0012, p-val = 0.1726
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8182 0.0202 40.4205 20 <.0001 0.7760 0.8604 ***
## Validity -0.0528 0.0373 -1.4146 20 0.1726 -0.1306 0.0250
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8182 0.0178 9.38 0.778 0.8582
## Validity -0.0528 0.0403 10.30 -0.142 0.0366## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8182 0.0202 0.7760 0.8604 0.6556 0.9808
## 2 0.7654 0.0313 0.7001 0.8308 0.5953 0.93552.2.3.4.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.forms <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 21.0431 -42.0863 -32.0863 -27.9202 -26.6317
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 18 no STUDID
## sigma^2.2 0.0031 0.0556 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 17) = 331.0524, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 17) = 4.1365, p-val = 0.0344
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8100 0.0146 55.5020 17 <.0001 0.7792 0.8408 ***
## factor(Forms)B 0.0261 0.0444 0.5883 17 0.5640 -0.0676 0.1198
## factor(Forms)Both -0.2534 0.0911 -2.7806 17 0.0128 -0.4457 -0.0611 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8100 0.0141 10.53 0.7788 0.841
## factor(Forms)B 0.0261 0.0141 1.25 -0.0863 0.139
## factor(Forms)Both -0.2534 0.0141 10.53 -0.2846 -0.222## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8100 0.0146 0.7792 0.8408 0.6887 0.9312
## 2 0.8361 0.0419 0.7476 0.9246 0.6892 0.9830
## 3 0.5565 0.0900 0.3667 0.7463 0.3335 0.77962.2.3.4.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.scores <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.7631 -37.5262 -29.5262 -25.9647 -26.4493
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 18 no STUDID
## sigma^2.2 0.0044 0.0665 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 18) = 342.0831, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 18) = 0.0371, p-val = 0.8494
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8059 0.0189 42.6621 18 <.0001 0.7662 0.8455 ***
## Scores -0.0068 0.0353 -0.1926 18 0.8494 -0.0809 0.0673
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8059 0.0186 8.09 0.7631 0.8486
## Scores -0.0068 0.0320 8.87 -0.0795 0.0659## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8059 0.0189 0.7662 0.8455 0.6607 0.9510
## 2 0.7991 0.0298 0.7364 0.8617 0.6460 0.95212.2.3.4.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.lang <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 19.3985 -38.7971 -30.7971 -26.8141 -28.1304
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0061 0.0780 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 333.8084, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.0363, p-val = 0.8507
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7971 0.0314 25.3860 20 <.0001 0.7316 0.8626 ***
## LanguageEnglish 0.0072 0.0379 0.1906 20 0.8507 -0.0718 0.0862
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.79715 0.0429 5.81 0.6912 0.903
## LanguageEnglish 0.00722 0.0458 10.88 -0.0937 0.108## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7971 0.0314 0.7316 0.8626 0.6218 0.9725
## 2 0.8044 0.0212 0.7602 0.8485 0.6358 0.97292.2.3.4.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.cult <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 19.7400 -39.4800 -31.4800 -27.4970 -28.8133
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0059 0.0766 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 366.3880, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.7281, p-val = 0.4036
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7843 0.0273 28.7247 20 <.0001 0.7274 0.8413 ***
## CultureWestern 0.0301 0.0353 0.8533 20 0.4036 -0.0435 0.1037
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7843 0.0359 7.78 0.7011 0.868
## CultureWestern 0.0301 0.0387 14.78 -0.0526 0.113## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7843 0.0273 0.7274 0.8413 0.6146 0.9541
## 2 0.8144 0.0223 0.7679 0.8610 0.6479 0.98102.2.3.4.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfm.srelcr.female <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 21; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.5123 -37.0246 -29.0246 -25.2469 -26.1675
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0022 0.0473 19 no STUDID
## sigma^2.2 0.0041 0.0641 21 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 19) = 393.4763, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 19) = 1.2779, p-val = 0.2724
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.7092 0.0816 8.6922 19 <.0001 0.5384
## asin(sqrt(PropFemale/100)) 0.1039 0.0919 1.1304 19 0.2724 -0.0885
## ci.ub
## intrcpt 0.8799 ***
## asin(sqrt(PropFemale/100)) 0.2963
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.709 0.0657 3.12 0.504 0.914
## asin(sqrt(PropFemale/100)) 0.104 0.0654 2.20 -0.154 0.3622.2.3.5 Publication Bias and Infuential Effect Sizes
## Reliability of factor 1
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfm.srel1.pet <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SREL1.vg))
## Summarize the results
summary(MLMEMCSC.tfm.srel1.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 35.4183 -70.8366 -62.8366 -58.8537 -60.1699
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0009 0.0304 20 no STUDID
## sigma^2.2 0.0002 0.0147 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 302.1113, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 58.1024, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.9575 0.0141 67.9768 20 <.0001 0.9282 0.9869 ***
## sqrt(SREL1.vg) -4.8739 0.6394 -7.6225 20 <.0001 -6.2077 -3.5401 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel1.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.958 0.0157 61.17 9.95 <0.001 ***
## sqrt(SREL1.vg) -4.874 0.8356 -5.83 6.82 <0.001 ***## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfm.srel1.fpt <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.6932 -37.3864 -29.3864 -25.4034 -26.7197
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0082 0.0905 20 no STUDID
## sigma^2.2 0.0005 0.0222 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 675.9245, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.9393, p-val = 0.3440
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8315 0.0293 28.4077 20 <.0001 0.7704 0.8925 ***
## N 0.0000 0.0000 0.9692 20 0.3440 -0.0001 0.0002
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel1.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.8314540 0.0288306 28.84 14.61 <0.001 ***
## N 0.0000482 0.0000275 1.75 2.61 0.191## Contour-enhanced funnel plot
funnel(MLREMCSC.tfm.srel1,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfm.srel1.peese <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SREL1.vg)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 32.4621 -64.9242 -56.9242 -52.9413 -54.2576
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0011 0.0328 20 no STUDID
## sigma^2.2 0.0005 0.0224 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 560.1821, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 44.3059, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.9060 0.0115 78.9420 20 <.0001 0.8821 0.9300 ***
## SREL1.vg -65.6666 9.8654 -6.6563 20 <.0001 -86.2454 -45.0878 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel1.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.906 0.0128 70.84 14.18 <0.001 ***
## SREL1.vg -65.667 19.5515 -3.36 2.66 0.0524 .## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfm.srel1 <- influence.rma.uni(rma(SREL1,
SREL1.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfm.srel1)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfm.srel1)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 1.4394 0.2972 0.0826 0.9896 0.0069 317.8785 0.0499 4.9946 0.2965
## 2 -0.9979 -0.2009 0.0399 1.0226 0.0072 787.6459 0.0363 3.6316 -0.2012
## 3 -0.9016 -0.1810 0.0327 1.0382 0.0073 786.4538 0.0384 3.8408 -0.1810
## 4 0.4382 0.1377 0.0206 1.1349 0.0080 725.6504 0.0497 4.9750 0.1381
## 5 0.6415 0.1740 0.0322 1.1147 0.0078 796.6815 0.0481 4.8119 0.1742
## 6 0.6497 0.1785 0.0340 1.1174 0.0079 767.9581 0.0498 4.9771 0.1789
## 7 0.3008 0.1091 0.0130 1.1416 0.0080 731.7872 0.0496 4.9604 0.1094
## 8 0.8316 0.2109 0.0465 1.0961 0.0077 788.3333 0.0498 4.9832 0.2112
## 9 0.0633 0.0546 0.0032 1.1379 0.0080 785.7941 0.0470 4.7000 0.0546
## 10 -0.5493 -0.0960 0.0096 1.0929 0.0077 780.5537 0.0442 4.4197 -0.0959
## 11 0.7944 0.2013 0.0424 1.0982 0.0077 798.7267 0.0483 4.8263 0.2015
## 12 0.6810 0.1827 0.0354 1.1124 0.0078 795.1204 0.0489 4.8945 0.1830
## 13 0.0369 0.0479 0.0025 1.1349 0.0080 788.8536 0.0460 4.6033 0.0479
## 14 -0.2058 -0.0093 0.0001 1.1229 0.0079 787.0258 0.0447 4.4714 -0.0092
## 15 -0.0025 0.0407 0.0018 1.1417 0.0080 760.9567 0.0487 4.8685 0.0408
## 16 0.0935 0.0618 0.0041 1.1394 0.0080 784.0190 0.0475 4.7479 0.0618
## 17 0.2915 0.1072 0.0125 1.1420 0.0080 724.0538 0.0496 4.9631 0.1075
## 18 -0.8025 -0.1718 0.0300 1.0647 0.0075 735.7398 0.0475 4.7538 -0.1719
## 19 0.1050 0.0652 0.0046 1.1422 0.0080 773.2985 0.0485 4.8487 0.0653
## 20 -2.2678 -0.4471 0.1808 0.8475 0.0060 783.2802 0.0257 2.5750 -0.4675
## 21 -4.6711 -1.2237 0.9410 0.4622 0.0031 747.3058 0.0318 3.1805 -1.4435 *
## 22 1.3605 0.2875 0.0787 1.0061 0.0070 779.5789 0.0497 4.9728 0.2870## Reliability of factor 2
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfm.srel2.pet <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SREL2.vg))
## Summarize the results
summary(MLMEMCSC.tfm.srel2.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 14.8921 -29.7841 -21.7841 -17.8012 -19.1174
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0069 0.0829 20 no STUDID
## sigma^2.2 0.0052 0.0722 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 270.2438, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 22.1807, p-val = 0.0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8079 0.0498 16.2090 20 <.0001 0.7039 0.9118 ***
## sqrt(SREL2.vg) -4.5563 0.9674 -4.7096 20 0.0001 -6.5744 -2.5383 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel2.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.808 0.0444 18.21 10.89 <0.001 ***
## sqrt(SREL2.vg) -4.556 0.6162 -7.39 6.05 <0.001 ***## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfm.srel2.fpt <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.2174 -12.4348 -4.4348 -0.4519 -1.7681
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0247 0.1572 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 331.5083, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.0136, p-val = 0.9082
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6010 0.0501 12.0054 20 <.0001 0.4966 0.7054 ***
## N -0.0000 0.0001 -0.1167 20 0.9082 -0.0002 0.0002
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel2.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.60099263 0.053307 11.274 15.00 <0.001 ***
## N -0.00000908 0.000068 -0.134 3.14 0.902## Contour-enhanced funnel plot
funnel(MLREMCSC.tfm.srel2,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfm.srel2.peese <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SREL2.vg)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 15.0124 -30.0248 -22.0248 -18.0419 -19.3582
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0039 0.0621 20 no STUDID
## sigma^2.2 0.0078 0.0884 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 276.1135, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 21.8420, p-val = 0.0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6963 0.0320 21.7446 20 <.0001 0.6295 0.7631 ***
## SREL2.vg -33.4811 7.1640 -4.6735 20 0.0001 -48.4249 -18.5374 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel2.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.696 0.0333 20.93 13.33 < 0.001 ***
## SREL2.vg -33.481 4.3036 -7.78 3.38 0.00287 **## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfm.srel2 <- influence.rma.uni(rma(SREL2,
SREL2.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfm.srel2)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfm.srel2)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 1.2250 0.2748 0.0740 1.0336 0.0227 283.8733 0.0504 5.0435 0.2746
## 2 -2.4560 -0.4760 0.2034 0.8578 0.0190 335.1432 0.0287 2.8699 -0.4936
## 3 0.8504 0.1982 0.0403 1.0762 0.0238 349.6709 0.0464 4.6381 0.1983
## 4 0.4660 0.1297 0.0179 1.1142 0.0247 351.1885 0.0505 5.0477 0.1300
## 5 -0.8278 -0.1652 0.0275 1.0503 0.0234 344.1107 0.0398 3.9781 -0.1651
## 6 0.6150 0.1601 0.0270 1.1038 0.0244 349.7256 0.0503 5.0284 0.1605
## 7 -0.9107 -0.2066 0.0427 1.0515 0.0232 300.3628 0.0489 4.8938 -0.2066
## 8 -0.1552 -0.0105 0.0001 1.1194 0.0248 329.8443 0.0498 4.9816 -0.0106
## 9 -2.2254 -0.4876 0.2084 0.8649 0.0190 332.2524 0.0349 3.4924 -0.4993
## 10 0.7174 0.1758 0.0321 1.0906 0.0242 350.3575 0.0476 4.7641 0.1759
## 11 -0.2659 -0.0354 0.0013 1.0910 0.0244 349.0430 0.0396 3.9620 -0.0353
## 12 0.3959 0.1110 0.0131 1.1108 0.0247 351.1821 0.0473 4.7320 0.1110
## 13 0.1646 0.0601 0.0038 1.1125 0.0248 350.4211 0.0454 4.5399 0.0601
## 14 0.9269 0.2152 0.0472 1.0701 0.0237 348.0770 0.0477 4.7737 0.2153
## 15 0.3890 0.1119 0.0133 1.1154 0.0247 350.9829 0.0492 4.9239 0.1121
## 16 1.3241 0.2881 0.0802 1.0183 0.0224 330.3947 0.0493 4.9277 0.2878
## 17 -1.4597 -0.3716 0.1259 0.9635 0.0210 267.9315 0.0488 4.8764 -0.3707
## 18 0.1264 0.0545 0.0032 1.1216 0.0249 347.6900 0.0490 4.8992 0.0546
## 19 -0.1769 -0.0159 0.0003 1.1131 0.0247 344.6873 0.0475 4.7495 -0.0159
## 20 -1.4935 -0.3006 0.0863 0.9703 0.0216 341.5031 0.0335 3.3452 -0.3033
## 21 -0.4619 -0.0840 0.0074 1.0935 0.0243 342.4135 0.0459 4.5864 -0.0840
## 22 1.7400 0.3521 0.1108 0.9477 0.0206 302.5884 0.0495 4.9464 0.3507## Reliability of factor 1
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfm.srelcr.pet <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SRELCR.vg))
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 31.2702 -62.5404 -54.5404 -50.5575 -51.8738
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0009 0.0296 20 no STUDID
## sigma^2.2 0.0013 0.0358 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 267.3998, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 31.2357, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.9073 0.0208 43.5288 20 <.0001 0.8639 0.9508 ***
## sqrt(SRELCR.vg) -4.7230 0.8451 -5.5889 20 <.0001 -6.4857 -2.9602 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srelcr.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.907 0.0191 47.50 8.78 <0.001 ***
## sqrt(SRELCR.vg) -4.723 0.6082 -7.77 5.56 <0.001 ***## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfm.srelcr.fpt <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 19.6112 -39.2225 -31.2225 -27.2396 -28.5558
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0060 0.0777 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 398.1385, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.5229, p-val = 0.4780
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7894 0.0248 31.8074 20 <.0001 0.7377 0.8412 ***
## N 0.0000 0.0000 0.7231 20 0.4780 -0.0001 0.0001
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srelcr.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.7894383 0.0265932 29.69 14.95 <0.001 ***
## N 0.0000278 0.0000254 1.09 3.16 0.351## Contour-enhanced funnel plot
funnel(MLREMCSC.tfm.srelcr,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfm.srelcr.peese <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SRELCR.vg)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 31.2702 -62.5405 -54.5405 -50.5575 -51.8738
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0021 0.0456 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 301.7706, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 30.3711, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8484 0.0129 65.9421 20 <.0001 0.8215 0.8752 ***
## SRELCR.vg -65.5396 11.8925 -5.5110 20 <.0001 -90.3469 -40.7323 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srelcr.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.848 0.0125 67.67 10.58 <0.001 ***
## SRELCR.vg -65.540 7.7955 -8.41 2.19 0.0105 *## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfm.srelcr <- influence.rma.uni(rma(SRELCR,
SRELCR.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfm.srelcr)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfm.srelcr)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 1.6392 0.3294 0.0975 0.9557 0.0050 236.3897 0.0508 5.0778 0.3279
## 2 -2.7422 -0.5509 0.2646 0.7999 0.0043 397.0933 0.0272 2.7174 -0.5823
## 3 0.4341 0.1247 0.0165 1.1144 0.0060 415.8825 0.0451 4.5096 0.1246
## 4 0.4401 0.1339 0.0193 1.1273 0.0061 408.4356 0.0507 5.0686 0.1343
## 5 -0.3684 -0.0530 0.0029 1.0992 0.0059 410.4615 0.0426 4.2553 -0.0529
## 6 0.8810 0.2179 0.0491 1.0856 0.0058 409.5987 0.0506 5.0631 0.2182
## 7 -0.5819 -0.1129 0.0133 1.0950 0.0059 359.2740 0.0496 4.9636 -0.1131
## 8 0.3313 0.1107 0.0132 1.1318 0.0061 408.0050 0.0503 5.0341 0.1110
## 9 -1.7581 -0.4219 0.1587 0.9054 0.0048 395.8874 0.0390 3.8974 -0.4269
## 10 -0.0850 0.0140 0.0002 1.1204 0.0060 411.6781 0.0453 4.5275 0.0140
## 11 0.2224 0.0794 0.0067 1.1164 0.0060 415.2250 0.0431 4.3087 0.0792
## 12 0.4677 0.1356 0.0196 1.1193 0.0060 415.6860 0.0480 4.7978 0.1358
## 13 0.1868 0.0742 0.0059 1.1226 0.0061 414.5021 0.0452 4.5224 0.0741
## 14 0.8714 0.2104 0.0457 1.0828 0.0058 415.3431 0.0479 4.7906 0.2105
## 15 0.4310 0.1305 0.0183 1.1252 0.0060 414.4503 0.0496 4.9590 0.1308
## 16 0.6937 0.1795 0.0339 1.1025 0.0059 416.1319 0.0483 4.8289 0.1797
## 17 -0.4068 -0.0654 0.0045 1.1129 0.0060 368.8243 0.0498 4.9777 -0.0656
## 18 -0.0775 0.0173 0.0003 1.1304 0.0061 402.2678 0.0490 4.9044 0.0174
## 19 0.0162 0.0389 0.0016 1.1305 0.0061 408.7567 0.0483 4.8273 0.0390
## 20 -2.5921 -0.5153 0.2349 0.8212 0.0044 398.7309 0.0270 2.6995 -0.5415
## 21 -1.7132 -0.4388 0.1687 0.9044 0.0048 386.7081 0.0430 4.2956 -0.4414
## 22 1.7539 0.3384 0.1006 0.9336 0.0049 378.0483 0.0497 4.9737 0.33682.2.3.6 Forest Plots
## Reliability of factor 1
#pdf(file = "ForestPlots-TwoFactorModel4.pdf", onefile = TRUE)
forest.rma(MLREMCSC.tfm.srel1,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfm.srel1$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Reliability Factor 1 (Model 4)")
## Reliability of factor 2
forest.rma(MLREMCSC.tfm.srel2,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfm.srel2$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Reliability Factor 2 (Model 4)")
## Composite reliability
forest.rma(MLREMCSC.tfm.srelcr,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfm.srelcr$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Composite Reliability (Model 4)")
#dev.off()2.2.4 Meta-Analysis of the Two-Factor Model with Cross-Loadings
2.2.4.1 Meta-Analytic Baseline Models
## Reliability of factor 1
## Three-level REM
## Model specification and estimation
MLREMCSC.tfmc.srel1 <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfmc.srel1, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 7.6334 -15.2668 -9.2668 -6.1332 -7.8551
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0269 0.1641 20 no STUDID
## sigma^2.2 0.0011 0.0334 22 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 21) = 392.7456, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.7514 0.0388 19.3793 21 <.0001 0.6707 0.8320 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfmc.srel1, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0269 0.0034 0.0657
## sigma.1 0.1641 0.0583 0.2563
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0011 0.0002 0.0230
## sigma.2 0.0334 0.0154 0.1516## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfmc.srel1))## % of total variance I2
## Level 1 0.623620 ---
## Level 2 3.948782 3.95
## Level 3 95.427598 95.43
## Total I2: 99.38%plot(dmetar::var.comp(MLREMCSC.tfmc.srel1))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfmc.srel1, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.751 0.0386 18.8 0.671 0.832# Profile plots
profile.rma.mv(MLREMCSC.tfmc.srel1, sigma2=1)
profile.rma.mv(MLREMCSC.tfmc.srel1, sigma2=2)
## Reliability of factor 2
## Three-level REM
## Model specification and estimation
MLREMCSC.tfmc.srel2 <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfmc.srel2, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 7.3930 -14.7861 -8.7861 -5.6525 -7.3743
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0210 0.1450 22 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 21) = 242.7908, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.6305 0.0336 18.7549 21 <.0001 0.5605 0.7004 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfmc.srel2, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0000 0.0000 0.0428
## sigma.1 0.0000 0.0000 0.2069
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0210 0.0021 0.0478
## sigma.2 0.1450 0.0460 0.2186## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfmc.srel2))## % of total variance I2
## Level 1 4.691245234104 ---
## Level 2 95.308754439909 95.31
## Level 3 0.000000325987 0
## Total I2: 95.31%plot(dmetar::var.comp(MLREMCSC.tfmc.srel2))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfmc.srel2, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.63 0.0327 14.6 0.561 0.7# Profile plots
profile.rma.mv(MLREMCSC.tfmc.srel2, sigma2=1)
profile.rma.mv(MLREMCSC.tfmc.srel2, sigma2=2)
## Composite reliability
## Three-level REM
## Model specification and estimation
MLREMCSC.tfmc.srelcr <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfmc.srelcr, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 22.5089 -45.0178 -39.0178 -35.8842 -37.6060
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0050 0.0707 22 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 21) = 319.0690, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.8230 0.0160 51.5549 21 <.0001 0.7898 0.8562 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfmc.srelcr, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0000 0.0000 0.0095
## sigma.1 0.0000 0.0000 0.0974
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0050 0.0007 0.0114
## sigma.2 0.0707 0.0268 0.1066## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfmc.srelcr))## % of total variance I2
## Level 1 2.8633835918218 ---
## Level 2 97.1366154090294 97.14
## Level 3 0.0000009991488 0
## Total I2: 97.14%plot(dmetar::var.comp(MLREMCSC.tfmc.srelcr))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfmc.srelcr, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.823 0.0156 15.3 0.79 0.856# Profile plots
profile.rma.mv(MLREMCSC.tfmc.srel1, sigma2=1)
profile.rma.mv(MLREMCSC.tfmc.srel1, sigma2=2)
2.2.4.2 Moderator Analyses for the Reliability of Factor 1
2.2.4.2.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.age <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.9700 -13.9399 -5.9399 -1.9570 -3.2732
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0292 0.1709 20 no STUDID
## sigma^2.2 0.0011 0.0333 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 348.2658, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.2580, p-val = 0.6171
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7351 0.0501 14.6675 20 <.0001 0.6306 0.8396 ***
## Adults 0.0427 0.0841 0.5079 20 0.6171 -0.1326 0.2180
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7351 0.0584 11.9 0.608 0.863
## Adults 0.0427 0.0678 12.4 -0.104 0.190## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7351 0.0501 0.6306 0.8396 0.3571 1.1131
## 2 0.7778 0.0675 0.6370 0.9186 0.3882 1.16742.2.4.2.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.validity <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 7.0093 -14.0187 -6.0187 -2.0358 -3.3520
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0284 0.1686 20 no STUDID
## sigma^2.2 0.0011 0.0334 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 329.2026, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.3317, p-val = 0.5711
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7673 0.0492 15.6011 20 <.0001 0.6647 0.8699 ***
## Validity -0.0481 0.0835 -0.5759 20 0.5711 -0.2223 0.1261
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7673 0.0543 11.9 0.649 0.886
## Validity -0.0481 0.0724 12.2 -0.206 0.109## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7673 0.0492 0.6647 0.8699 0.3943 1.1403
## 2 0.7192 0.0675 0.5784 0.8600 0.3340 1.10452.2.4.2.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.forms <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.9179 -13.8358 -3.8358 0.3302 1.6187
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0257 0.1602 18 no STUDID
## sigma^2.2 0.0011 0.0334 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 17) = 353.1379, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 17) = 1.5719, p-val = 0.2364
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7658 0.0433 17.6841 17 <.0001 0.6744 0.8571 ***
## factor(Forms)B 0.0202 0.1264 0.1596 17 0.8751 -0.2465 0.2868
## factor(Forms)Both -0.3550 0.2029 -1.7495 17 0.0982 -0.7832 0.0731 .
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7658 0.0443 13.91 0.671 0.861
## factor(Forms)B 0.0202 0.0655 1.28 -0.483 0.524
## factor(Forms)Both -0.3550 0.0443 13.91 -0.450 -0.260## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7658 0.0433 0.6744 0.8571 0.4085 1.1230
## 2 0.7859 0.1187 0.5354 1.0364 0.3593 1.2125
## 3 0.4107 0.1983 -0.0075 0.8290 -0.1317 0.95322.2.4.2.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.scores <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.4748 -12.9496 -4.9496 -1.3881 -1.8726
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0295 0.1716 18 no STUDID
## sigma^2.2 0.0011 0.0334 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 18) = 358.0045, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 18) = 0.6513, p-val = 0.4302
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7756 0.0516 15.0343 18 <.0001 0.6672 0.8840 ***
## Scores -0.0732 0.0907 -0.8071 18 0.4302 -0.2636 0.1173
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7756 0.0333 10.9 0.702 0.849
## Scores -0.0732 0.1169 9.8 -0.334 0.188## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7756 0.0516 0.6672 0.8840 0.3926 1.1586
## 2 0.7025 0.0745 0.5458 0.8591 0.3031 1.10182.2.4.2.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.lang <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 8.0257 -16.0513 -8.0513 -4.0684 -5.3847
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0268 0.1636 20 no STUDID
## sigma^2.2 0.0011 0.0334 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 318.2542, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 2.3761, p-val = 0.1389
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6660 0.0676 9.8543 20 <.0001 0.5250 0.8070 ***
## LanguageEnglish 0.1270 0.0824 1.5415 20 0.1389 -0.0449 0.2989
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.666 0.113 5.84 0.388 0.944
## LanguageEnglish 0.127 0.114 11.58 -0.123 0.377## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6660 0.0676 0.5250 0.8070 0.2902 1.0418
## 2 0.7930 0.0471 0.6947 0.8914 0.4311 1.15502.2.4.2.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.cult <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 7.7431 -15.4863 -7.4863 -3.5034 -4.8196
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0272 0.1649 20 no STUDID
## sigma^2.2 0.0011 0.0334 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 349.9256, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 1.7970, p-val = 0.1951
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6914 0.0592 11.6727 20 <.0001 0.5678 0.8150 ***
## CultureWestern 0.1054 0.0786 1.3405 20 0.1951 -0.0586 0.2694
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.691 0.0844 7.83 0.4961 0.887
## CultureWestern 0.105 0.0876 16.71 -0.0798 0.291## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6914 0.0592 0.5678 0.8150 0.3192 1.0636
## 2 0.7968 0.0517 0.6889 0.9047 0.4295 1.16412.2.4.2.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfmc.srel1.female <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 21; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.1238 -12.2477 -4.2477 -0.4699 -1.3906
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0321 0.1793 19 no STUDID
## sigma^2.2 0.0011 0.0334 21 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 19) = 341.2842, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 19) = 0.1277, p-val = 0.7248
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.6859 0.1838 3.7311 19 0.0014 0.3011
## asin(sqrt(PropFemale/100)) 0.0739 0.2067 0.3574 19 0.7248 -0.3588
## ci.ub
## intrcpt 1.0707 **
## asin(sqrt(PropFemale/100)) 0.5065
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6859 0.1089 3.24 0.353 1.019
## asin(sqrt(PropFemale/100)) 0.0739 0.0912 2.27 -0.277 0.4252.2.4.3 Moderator Analyses for the Reliability of Factor 2
2.2.4.3.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.age <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.5900 -13.1799 -5.1799 -1.1970 -2.5133
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0079 0.0888 20 no STUDID
## sigma^2.2 0.0154 0.1242 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 221.2856, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.2824, p-val = 0.6010
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6449 0.0450 14.3404 20 <.0001 0.5511 0.7387 ***
## Adults -0.0406 0.0764 -0.5314 20 0.6010 -0.2000 0.1188
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6449 0.0349 10.4 0.567 0.722
## Adults -0.0406 0.0890 11.8 -0.235 0.154## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6449 0.0450 0.5511 0.7387 0.3129 0.9768
## 2 0.6043 0.0618 0.4754 0.7331 0.2607 0.94782.2.4.3.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.validity <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.8787 -13.7574 -5.7574 -1.7745 -3.0907
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0065 0.0803 20 no STUDID
## sigma^2.2 0.0151 0.1228 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 227.3358, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.8932, p-val = 0.3559
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6557 0.0433 15.1501 20 <.0001 0.5654 0.7460 ***
## Validity -0.0696 0.0736 -0.9451 20 0.3559 -0.2231 0.0840
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6557 0.0478 10.2 0.549 0.7620
## Validity -0.0696 0.0637 11.8 -0.209 0.0694## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6557 0.0433 0.5654 0.7460 0.3366 0.9748
## 2 0.5861 0.0595 0.4619 0.7104 0.2559 0.91642.2.4.3.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.forms <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 8.0692 -16.1384 -6.1384 -1.9724 -0.6839
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 18 no STUDID
## sigma^2.2 0.0159 0.1261 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 17) = 180.7465, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 17) = 1.0047, p-val = 0.3869
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6381 0.0335 19.0581 17 <.0001 0.5675 0.7087 ***
## factor(Forms)B 0.0334 0.1004 0.3325 17 0.7436 -0.1784 0.2451
## factor(Forms)Both -0.2469 0.1822 -1.3550 17 0.1931 -0.6313 0.1375
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6381 0.0332 10.15 0.564 0.712
## factor(Forms)B 0.0334 0.0452 1.26 -0.323 0.389
## factor(Forms)Both -0.2469 0.0332 10.15 -0.321 -0.173## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6381 0.0335 0.5675 0.7087 0.3627 0.9135
## 2 0.6715 0.0946 0.4718 0.8711 0.3388 1.0042
## 3 0.3912 0.1791 0.0133 0.7691 -0.0710 0.85342.2.4.3.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.scores <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 8.2477 -16.4953 -8.4953 -4.9338 -5.4184
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 18 no STUDID
## sigma^2.2 0.0156 0.1249 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 18) = 170.7798, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 18) = 0.5732, p-val = 0.4588
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6210 0.0356 17.4503 18 <.0001 0.5462 0.6957 ***
## Scores 0.0539 0.0712 0.7571 18 0.4588 -0.0957 0.2035
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6210 0.0268 8.22 0.559 0.682
## Scores 0.0539 0.0918 6.90 -0.164 0.272## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6210 0.0356 0.5462 0.6957 0.3480 0.8939
## 2 0.6749 0.0617 0.5453 0.8045 0.3822 0.96762.2.4.3.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.lang <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.5989 -13.1977 -5.1977 -1.2148 -2.5310
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0218 0.1478 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 182.8585, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.2258, p-val = 0.6398
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6544 0.0616 10.6230 20 <.0001 0.5259 0.7828 ***
## LanguageEnglish -0.0352 0.0741 -0.4751 20 0.6398 -0.1897 0.1193
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6544 0.0798 5.78 0.457 0.851
## LanguageEnglish -0.0352 0.0862 10.65 -0.226 0.155## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6544 0.0616 0.5259 0.7828 0.3204 0.9883
## 2 0.6192 0.0411 0.5334 0.7049 0.2993 0.93912.2.4.3.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.cult <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.7171 -13.4342 -5.4342 -1.4513 -2.7676
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0093 0.0966 20 no STUDID
## sigma^2.2 0.0141 0.1187 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 212.2652, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.5083, p-val = 0.4841
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6006 0.0561 10.7085 20 <.0001 0.4836 0.7176 ***
## CultureWestern 0.0527 0.0740 0.7130 20 0.4841 -0.1016 0.2070
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6006 0.0765 7.75 0.423 0.778
## CultureWestern 0.0527 0.0818 15.81 -0.121 0.226## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6006 0.0561 0.4836 0.7176 0.2606 0.9406
## 2 0.6533 0.0482 0.5527 0.7539 0.3186 0.98812.2.4.3.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfmc.srel2.female <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 21; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.6486 -13.2971 -5.2971 -1.5193 -2.4400
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 19 no STUDID
## sigma^2.2 0.0210 0.1449 21 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 19) = 215.0767, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 19) = 2.0870, p-val = 0.1648
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.4174 0.1490 2.8010 19 0.0114 0.1055
## asin(sqrt(PropFemale/100)) 0.2425 0.1679 1.4446 19 0.1648 -0.1089
## ci.ub
## intrcpt 0.7293 *
## asin(sqrt(PropFemale/100)) 0.5940
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.417 0.106 3.04 0.084 0.751
## asin(sqrt(PropFemale/100)) 0.243 0.109 2.14 -0.199 0.6842.2.4.4 Moderator Analyses for the Composite Reliability
2.2.4.4.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.age <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 20.8738 -41.7476 -33.7476 -29.7647 -31.0809
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0054 0.0732 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 291.6755, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.0548, p-val = 0.8173
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8252 0.0201 41.1396 20 <.0001 0.7833 0.8670 ***
## Adults -0.0082 0.0352 -0.2340 20 0.8173 -0.0816 0.0651
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.82519 0.0168 9.01 0.7871 0.8632
## Adults -0.00823 0.0392 11.34 -0.0943 0.0778## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8252 0.0201 0.7833 0.8670 0.6669 0.9835
## 2 0.8170 0.0289 0.7568 0.8772 0.6528 0.98112.2.4.4.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.validity <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 21.9864 -43.9728 -35.9728 -31.9898 -33.3061
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0048 0.0696 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 281.4849, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 2.3014, p-val = 0.1449
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8388 0.0188 44.6171 20 <.0001 0.7996 0.8780 ***
## Validity -0.0521 0.0344 -1.5170 20 0.1449 -0.1238 0.0196
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8388 0.0171 9.3 0.80 0.8773
## Validity -0.0521 0.0351 10.5 -0.13 0.0256## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8388 0.0188 0.7996 0.8780 0.6883 0.9892
## 2 0.7867 0.0288 0.7266 0.8467 0.6295 0.94382.2.4.4.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.forms <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 23.8113 -47.6225 -37.6225 -33.4564 -32.1680
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 18 no STUDID
## sigma^2.2 0.0026 0.0508 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 17) = 240.3683, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 17) = 4.7316, p-val = 0.0232
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8331 0.0133 62.5835 17 <.0001 0.8051 0.8612 ***
## factor(Forms)B 0.0144 0.0408 0.3523 17 0.7289 -0.0716 0.1004
## factor(Forms)Both -0.2788 0.0918 -3.0359 17 0.0075 -0.4725 -0.0850 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8331 0.0127 10.59 0.805 0.861
## factor(Forms)B 0.0144 0.0138 1.25 -0.096 0.125
## factor(Forms)Both -0.2788 0.0127 10.59 -0.307 -0.251## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8331 0.0133 0.8051 0.8612 0.7224 0.9439
## 2 0.8475 0.0385 0.7662 0.9288 0.7130 0.9820
## 3 0.5544 0.0909 0.3627 0.7461 0.3348 0.77392.2.4.4.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.scores <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 20; method: REML)
##
## logLik Deviance AIC BIC AICc
## 20.9422 -41.8844 -33.8844 -30.3229 -30.8075
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 18 no STUDID
## sigma^2.2 0.0035 0.0591 20 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 18) = 249.7669, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 18) = 0.1526, p-val = 0.7006
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8307 0.0168 49.4887 18 <.0001 0.7954 0.8659 ***
## Scores -0.0124 0.0317 -0.3907 18 0.7006 -0.0789 0.0542
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8307 0.0161 8.1 0.7936 0.8677
## Scores -0.0124 0.0291 8.7 -0.0787 0.0539## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8307 0.0168 0.7954 0.8659 0.7017 0.9597
## 2 0.8183 0.0269 0.7618 0.8747 0.6819 0.95462.2.4.4.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.lang <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 19.3985 -38.7971 -30.7971 -26.8141 -28.1304
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0061 0.0780 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 333.8084, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.0363, p-val = 0.8507
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7971 0.0314 25.3860 20 <.0001 0.7316 0.8626 ***
## LanguageEnglish 0.0072 0.0379 0.1906 20 0.8507 -0.0718 0.0862
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.79715 0.0429 5.81 0.6912 0.903
## LanguageEnglish 0.00722 0.0458 10.88 -0.0937 0.108## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7971 0.0314 0.7316 0.8626 0.6218 0.9725
## 2 0.8044 0.0212 0.7602 0.8485 0.6358 0.97292.2.4.4.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.cult <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 19.7400 -39.4800 -31.4800 -27.4970 -28.8133
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0059 0.0766 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 366.3880, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.7281, p-val = 0.4036
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7843 0.0273 28.7247 20 <.0001 0.7274 0.8413 ***
## CultureWestern 0.0301 0.0353 0.8533 20 0.4036 -0.0435 0.1037
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7843 0.0359 7.78 0.7011 0.868
## CultureWestern 0.0301 0.0387 14.78 -0.0526 0.113## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7843 0.0273 0.7274 0.8413 0.6146 0.9541
## 2 0.8144 0.0223 0.7679 0.8610 0.6479 0.98102.2.4.4.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfmc.srelcr.female <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 21; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.5123 -37.0246 -29.0246 -25.2469 -26.1675
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0022 0.0473 19 no STUDID
## sigma^2.2 0.0041 0.0641 21 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 19) = 393.4763, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 19) = 1.2779, p-val = 0.2724
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.7092 0.0816 8.6922 19 <.0001 0.5384
## asin(sqrt(PropFemale/100)) 0.1039 0.0919 1.1304 19 0.2724 -0.0885
## ci.ub
## intrcpt 0.8799 ***
## asin(sqrt(PropFemale/100)) 0.2963
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.709 0.0657 3.12 0.504 0.914
## asin(sqrt(PropFemale/100)) 0.104 0.0654 2.20 -0.154 0.3622.2.4.5 Publication Bias and Infuential Effect Sizes
## Reliability of factor 1
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfmc.srel1.pet <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SREL1C.vg))
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 22.9742 -45.9485 -37.9485 -33.9655 -35.2818
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0023 0.0477 20 no STUDID
## sigma^2.2 0.0006 0.0240 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 182.0183, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 56.6286, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.9283 0.0227 40.8949 20 <.0001 0.8809 0.9756 ***
## sqrt(SREL1C.vg) -4.9493 0.6577 -7.5252 20 <.0001 -6.3213 -3.5774 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel1.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.928 0.0291 31.94 10.0 < 0.001 ***
## sqrt(SREL1C.vg) -4.949 1.2289 -4.03 6.7 0.00549 **## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfmc.srel1.fpt <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 7.6173 -15.2347 -7.2347 -3.2518 -4.5680
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0268 0.1637 20 no STUDID
## sigma^2.2 0.0011 0.0334 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 388.4889, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 1.5069, p-val = 0.2339
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7091 0.0518 13.6837 20 <.0001 0.6010 0.8172 ***
## N 0.0001 0.0001 1.2276 20 0.2339 -0.0001 0.0003
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel1.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.709098 0.0582039 12.18 14.7 <0.001 ***
## N 0.000109 0.0000792 1.38 2.6 0.275## Contour-enhanced funnel plot
funnel(MLREMCSC.tfmc.srel1,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfmc.srel1.peese <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SREL1C.vg)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 25.5126 -51.0251 -43.0251 -39.0422 -40.3585
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0018 0.0423 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 229.1239, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 71.7669, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8555 0.0123 69.6609 20 <.0001 0.8299 0.8812 ***
## SREL1C.vg -49.7199 5.8691 -8.4715 20 <.0001 -61.9626 -37.4773 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel1.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.856 0.0125 68.58 9.25 <0.001 ***
## SREL1C.vg -49.720 12.3230 -4.03 3.25 0.0235 *## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfmc.srel1 <- influence.rma.uni(rma(SREL1C,
SREL1C.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfmc.srel1)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfmc.srel1)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 0.1549 0.0666 0.0048 1.1388 0.0272 384.5449 0.0473 4.7273 0.0667
## 2 -0.5613 -0.1026 0.0110 1.0943 0.0262 377.6156 0.0442 4.4160 -0.1026
## 3 -0.1615 -0.0059 0.0000 1.1248 0.0270 386.1092 0.0441 4.4051 -0.0059
## 4 0.2633 0.0914 0.0091 1.1401 0.0272 323.4127 0.0487 4.8682 0.0916
## 5 -6.1861 -1.7680 1.4299 0.2503 0.0053 326.0241 0.0336 3.3613 -2.4122 *
## 6 0.7283 0.1843 0.0358 1.1043 0.0263 390.7255 0.0487 4.8716 0.1846
## 7 0.3856 0.1170 0.0148 1.1343 0.0271 369.5273 0.0486 4.8632 0.1173
## 8 0.7985 0.1972 0.0406 1.0956 0.0261 378.7877 0.0487 4.8743 0.1974
## 9 0.0846 0.0511 0.0028 1.1393 0.0273 381.9874 0.0473 4.7284 0.0512
## 10 -0.7003 -0.1396 0.0201 1.0785 0.0258 370.7359 0.0448 4.4838 -0.1396
## 11 0.6530 0.1620 0.0276 1.1031 0.0264 392.7430 0.0442 4.4170 0.1619
## 12 0.0597 0.0454 0.0022 1.1382 0.0272 383.3971 0.0469 4.6915 0.0454
## 13 0.5869 0.1560 0.0260 1.1173 0.0267 392.4688 0.0478 4.7820 0.1562
## 14 0.7164 0.1808 0.0345 1.1044 0.0264 392.5779 0.0480 4.8035 0.1810
## 15 0.4326 0.1264 0.0173 1.1309 0.0270 385.6668 0.0484 4.8426 0.1266
## 16 0.2739 0.0921 0.0092 1.1358 0.0272 388.1351 0.0472 4.7230 0.0922
## 17 0.9569 0.2250 0.0517 1.0732 0.0255 265.1308 0.0488 4.8769 0.2251
## 18 0.0375 0.0411 0.0018 1.1413 0.0273 366.5885 0.0481 4.8055 0.0412
## 19 0.4552 0.1308 0.0185 1.1291 0.0270 388.3413 0.0483 4.8314 0.1310
## 20 -1.9830 -0.4271 0.1614 0.8638 0.0207 375.9428 0.0325 3.2527 -0.4391
## 21 -1.2360 -0.2564 0.0634 0.9892 0.0238 381.0212 0.0359 3.5902 -0.2578
## 22 0.7409 0.1850 0.0359 1.1012 0.0263 392.4477 0.0478 4.7845 0.1852## Reliability of factor 2
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfmc.srel2.pet <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SREL2C.vg))
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 10.5743 -21.1486 -13.1486 -9.1657 -10.4819
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0065 0.0804 20 no STUDID
## sigma^2.2 0.0071 0.0843 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 221.9369, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 8.8143, p-val = 0.0076
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7512 0.0477 15.7477 20 <.0001 0.6517 0.8507 ***
## sqrt(SREL2C.vg) -2.3491 0.7913 -2.9689 20 0.0076 -3.9997 -0.6986 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel2.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.751 0.0659 11.40 7.93 <0.001 ***
## sqrt(SREL2C.vg) -2.349 1.4378 -1.63 3.61 0.185## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfmc.srel2.fpt <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 6.4915 -12.9831 -4.9831 -1.0002 -2.3164
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0072 0.0851 20 no STUDID
## sigma^2.2 0.0158 0.1257 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 211.7767, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 0.0780, p-val = 0.7829
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6405 0.0499 12.8261 20 <.0001 0.5363 0.7447 ***
## N -0.0000 0.0001 -0.2793 20 0.7829 -0.0002 0.0001
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel2.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.6405029 0.0521098 12.291 14.12 <0.001 ***
## N -0.0000223 0.0000498 -0.449 2.82 0.686## Contour-enhanced funnel plot
funnel(MLREMCSC.tfmc.srel2,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfmc.srel2.peese <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SREL2C.vg)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 8.0477 -16.0954 -8.0954 -4.1124 -5.4287
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0190 0.1379 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 232.1348, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 1.3925, p-val = 0.2518
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6431 0.0336 19.1411 20 <.0001 0.5730 0.7132 ***
## SREL2C.vg -3.0747 2.6056 -1.1800 20 0.2518 -8.5098 2.3604
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel2.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.643 0.0366 17.558 13.31 <0.001 ***
## SREL2C.vg -3.075 7.7912 -0.395 1.17 0.753## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfmc.srel2 <- influence.rma.uni(rma(SREL2C,
SREL2C.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfmc.srel2)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfmc.srel2)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 1.3825 0.3084 0.0903 1.0071 0.0199 187.3832 0.0527 5.2724 0.3077
## 2 -1.4412 -0.3408 0.1081 0.9683 0.0193 226.9599 0.0444 4.4369 -0.3418
## 3 0.4769 0.1262 0.0168 1.1074 0.0223 242.5862 0.0467 4.6700 0.1261
## 4 0.0340 0.0347 0.0013 1.1343 0.0228 232.6962 0.0532 5.3174 0.0348
## 5 1.3984 0.2797 0.0757 1.0062 0.0201 237.2842 0.0429 4.2902 0.2803
## 6 0.1697 0.0661 0.0047 1.1328 0.0227 241.2097 0.0527 5.2743 0.0664
## 7 -1.0385 -0.2500 0.0613 1.0355 0.0206 195.6089 0.0517 5.1662 -0.2499
## 8 0.3386 0.1047 0.0118 1.1289 0.0226 242.7885 0.0532 5.3177 0.1051
## 9 -2.6962 -0.6082 0.3033 0.7833 0.0153 221.6910 0.0358 3.5799 -0.6328
## 10 0.5243 0.1405 0.0209 1.1115 0.0223 242.2160 0.0499 4.9932 0.1407
## 11 -0.2073 -0.0136 0.0002 1.0144 0.0212 242.6737 0.0063 0.6338 -0.0135
## 12 0.3147 0.0952 0.0097 1.1204 0.0226 242.7902 0.0491 4.9124 0.0953
## 13 0.4652 0.1271 0.0171 1.1133 0.0224 242.5485 0.0491 4.9084 0.1272
## 14 0.5382 0.1408 0.0209 1.1068 0.0223 242.3226 0.0482 4.8154 0.1408
## 15 0.0875 0.0464 0.0023 1.1303 0.0227 241.3383 0.0515 5.1469 0.0465
## 16 1.1418 0.2641 0.0691 1.0463 0.0208 224.4802 0.0519 5.1879 0.2640
## 17 -0.5720 -0.1099 0.0125 1.0826 0.0218 238.5621 0.0440 4.4005 -0.1098
## 18 -0.4249 -0.0784 0.0065 1.1079 0.0222 233.3408 0.0504 5.0448 -0.0785
## 19 -0.1425 -0.0084 0.0001 1.1224 0.0226 239.7208 0.0498 4.9803 -0.0084
## 20 -1.2761 -0.2369 0.0549 0.9960 0.0202 237.6616 0.0304 3.0392 -0.2384
## 21 -2.1922 -0.4654 0.1922 0.8686 0.0173 228.5150 0.0341 3.4066 -0.4781
## 22 1.5701 0.3366 0.1037 0.9731 0.0192 199.2949 0.0521 5.2055 0.3354## Reliability of factor 1
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfmc.srelcr.pet <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SRELCRC.vg))
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 32.6305 -65.2610 -57.2610 -53.2781 -54.5943
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0005 0.0220 20 no STUDID
## sigma^2.2 0.0012 0.0340 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 201.6670, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 33.5206, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.9207 0.0185 49.7848 20 <.0001 0.8821 0.9592 ***
## sqrt(SRELCRC.vg) -4.7309 0.8171 -5.7897 20 <.0001 -6.4354 -3.0264 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srelcr.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.921 0.0175 52.66 8.22 < 0.001 ***
## sqrt(SRELCRC.vg) -4.731 0.8378 -5.65 6.11 0.00124 **## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfmc.srelcr.fpt <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 21.3633 -42.7267 -34.7267 -30.7437 -32.0600
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0052 0.0720 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 310.4858, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 1.0811, p-val = 0.3109
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8058 0.0230 35.0240 20 <.0001 0.7578 0.8538 ***
## N 0.0000 0.0000 1.0398 20 0.3109 -0.0000 0.0001
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srelcr.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.8057743 0.0247654 32.54 14.98 <0.001 ***
## N 0.0000371 0.0000251 1.48 3.15 0.232## Contour-enhanced funnel plot
funnel(MLREMCSC.tfmc.srelcr,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfmc.srelcr.peese <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SRELCRC.vg)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 22; method: REML)
##
## logLik Deviance AIC BIC AICc
## 31.6004 -63.2009 -55.2009 -51.2180 -52.5342
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 20 no STUDID
## sigma^2.2 0.0017 0.0415 22 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 20) = 214.8777, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 20) = 29.3528, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8655 0.0118 73.2354 20 <.0001 0.8408 0.8901 ***
## SRELCRC.vg -69.2828 12.7879 -5.4178 20 <.0001 -95.9579 -42.6076 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srelcr.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.865 0.0108 80.39 9.74 <0.001 ***
## SRELCRC.vg -69.283 18.3526 -3.78 2.31 0.0506 .## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfmc.srelcr <- influence.rma.uni(rma(SRELCRC,
SRELCRC.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfmc.srelcr)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfmc.srelcr)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 1.4038 0.3016 0.0862 1.0031 0.0047 216.8109 0.0506 5.0619 0.3009
## 2 -2.0462 -0.4749 0.1970 0.8699 0.0041 298.7771 0.0366 3.6562 -0.4845
## 3 0.2506 0.0832 0.0074 1.1162 0.0054 318.1406 0.0447 4.4673 0.0831
## 4 0.2869 0.0972 0.0102 1.1297 0.0054 301.0381 0.0506 5.0554 0.0975
## 5 0.0194 0.0339 0.0012 1.1191 0.0054 316.0416 0.0453 4.5336 0.0339
## 6 0.8604 0.2119 0.0464 1.0854 0.0052 310.8310 0.0505 5.0549 0.2122
## 7 -0.6294 -0.1277 0.0169 1.0883 0.0052 260.6025 0.0495 4.9532 -0.1279
## 8 0.6755 0.1770 0.0330 1.1050 0.0053 318.6885 0.0505 5.0527 0.1774
## 9 -1.8950 -0.4564 0.1824 0.8872 0.0042 296.4074 0.0393 3.9304 -0.4624
## 10 -0.3521 -0.0525 0.0029 1.1023 0.0053 312.1341 0.0444 4.4407 -0.0524
## 11 -0.0328 0.0181 0.0003 1.0964 0.0053 317.8993 0.0369 3.6880 0.0180
## 12 0.2270 0.0812 0.0071 1.1239 0.0054 316.8809 0.0475 4.7541 0.0812
## 13 0.4705 0.1307 0.0181 1.1125 0.0053 318.8640 0.0468 4.6753 0.1307
## 14 0.9708 0.2266 0.0523 1.0688 0.0051 316.6379 0.0483 4.8346 0.2267
## 15 0.4132 0.1229 0.0162 1.1220 0.0054 317.4542 0.0496 4.9570 0.1232
## 16 0.4500 0.1281 0.0175 1.1161 0.0054 318.6606 0.0478 4.7839 0.1282
## 17 0.4893 0.1374 0.0201 1.1158 0.0053 318.7101 0.0488 4.8750 0.1376
## 18 -0.2416 -0.0263 0.0007 1.1194 0.0054 300.7926 0.0488 4.8819 -0.0263
## 19 0.2737 0.0924 0.0092 1.1257 0.0054 316.3286 0.0487 4.8741 0.0926
## 20 -2.7862 -0.5024 0.2273 0.8234 0.0039 301.8371 0.0239 2.3865 -0.5311
## 21 -2.3983 -0.6413 0.3214 0.7858 0.0037 282.6121 0.0414 4.1373 -0.6527
## 22 1.5096 0.3129 0.0911 0.9844 0.0047 295.1839 0.0495 4.9462 0.31212.2.4.6 Forest Plots
## Reliability of factor 1
#pdf(file = "ForestPlots-TwoFactorModel3.pdf", onefile = TRUE)
forest.rma(MLREMCSC.tfmc.srel1,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfmc.srel1$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Reliability Factor 1 (Model 3)")
## Reliability of factor 2
forest.rma(MLREMCSC.tfmc.srel2,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfmc.srel2$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Reliability Factor 2 (Model 3)")
## Composite reliability
forest.rma(MLREMCSC.tfmc.srelcr,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfmc.srelcr$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Composite Reliability (Model 3)")
#dev.off()3 R session info
sessionInfo()## R version 4.3.1 (2023-06-16)
## Platform: x86_64-apple-darwin20 (64-bit)
## Running under: macOS Ventura 13.6
##
## Matrix products: default
## BLAS: /Library/Frameworks/R.framework/Versions/4.3-x86_64/Resources/lib/libRblas.0.dylib
## LAPACK: /Library/Frameworks/R.framework/Versions/4.3-x86_64/Resources/lib/libRlapack.dylib; LAPACK version 3.11.0
##
## locale:
## [1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
##
## time zone: Europe/Oslo
## tzcode source: internal
##
## attached base packages:
## [1] stats graphics grDevices utils datasets methods base
##
## other attached packages:
## [1] dmetar_0.1.0 semPower_2.0.1 devtools_2.4.5
## [4] usethis_2.1.6 dplyr_1.1.3 lavaan_0.6-16
## [7] clubSandwich_0.5.8 robumeta_2.1 metafor_4.2-0
## [10] numDeriv_2016.8-1.1 metadat_1.2-0 Matrix_1.5-4.1
## [13] metaSEM_1.3.1 OpenMx_2.21.8 reshape2_1.4.4
## [16] psych_2.3.6 pacman_0.5.1
##
## loaded via a namespace (and not attached):
## [1] mathjaxr_1.6-0 rstudioapi_0.14 jsonlite_1.8.7
## [4] magrittr_2.0.3 modeltools_0.2-23 farver_2.1.1
## [7] nloptr_2.0.3 rmarkdown_2.21 fs_1.6.3
## [10] vctrs_0.6.3 memoise_2.0.1 minqa_1.2.6
## [13] CompQuadForm_1.4.3 htmltools_0.5.5 sass_0.4.6
## [16] bslib_0.4.2 htmlwidgets_1.6.2 plyr_1.8.8
## [19] poibin_1.5 sandwich_3.0-2 zoo_1.8-12
## [22] cachem_1.0.8 mime_0.12 lifecycle_1.0.3
## [25] pkgconfig_2.0.3 R6_2.5.1 fastmap_1.1.1
## [28] shiny_1.7.4 magic_1.6-1 digest_0.6.33
## [31] colorspace_2.1-0 ps_1.7.5 pkgload_1.3.2.1
## [34] ellipse_0.5.0 labeling_0.4.3 fansi_1.0.4
## [37] abind_1.4-5 compiler_4.3.1 remotes_2.4.2
## [40] withr_2.5.0 meta_6.5-0 pkgbuild_1.4.0
## [43] highr_0.10 MASS_7.3-60 sessioninfo_1.2.2
## [46] tools_4.3.1 pbivnorm_0.6.0 prabclus_2.3-2
## [49] httpuv_1.6.11 nnet_7.3-19 glue_1.6.2
## [52] quadprog_1.5-8 callr_3.7.3 nlme_3.1-162
## [55] promises_1.2.0.1 grid_4.3.1 cluster_2.1.4
## [58] generics_0.1.3 gtable_0.3.4 class_7.3-22
## [61] xml2_1.3.4 utf8_1.2.3 flexmix_2.3-19
## [64] ggrepel_0.9.3 pillar_1.9.0 stringr_1.5.0
## [67] later_1.3.1 robustbase_0.99-0 splines_4.3.1
## [70] lattice_0.21-8 tidyselect_1.2.0 pbapply_1.7-2
## [73] miniUI_0.1.1.1 knitr_1.42 gridExtra_2.3
## [76] stats4_4.3.1 xfun_0.39 diptest_0.76-0
## [79] DEoptimR_1.1-2 MuMIn_1.47.5 netmeta_2.8-2
## [82] stringi_1.7.12 yaml_2.3.7 boot_1.3-28.1
## [85] evaluate_0.21 codetools_0.2-19 kernlab_0.9-32
## [88] tibble_3.2.1 cli_3.6.1 RcppParallel_5.1.7
## [91] xtable_1.8-4 munsell_0.5.0 processx_3.8.2
## [94] jquerylib_0.1.4 Rcpp_1.0.11 parallel_4.3.1
## [97] ellipsis_0.3.2 ggplot2_3.4.3 prettyunits_1.1.1
## [100] mclust_6.0.0 profvis_0.3.8 urlchecker_1.0.1
## [103] lme4_1.1-34 mvtnorm_1.1-3 scales_1.2.1
## [106] purrr_1.0.2 crayon_1.5.2 fpc_2.2-10
## [109] rlang_1.1.1 mnormt_2.1.1