Supplementary Material
Reliability Generalization of the TTCT: Parameter-Based MASEM
25 October 2023
1 Data Preparation
1.1 Installing and
loading relevant R packages
library(pacman)
pacman::p_load(psych,
reshape2,
metaSEM,
metafor,
robumeta,
clubSandwich,
lavaan,
dplyr,
devtools)
# devtools::install_github("MathiasHarrer/dmetar")
library(dmetar)1.2 Data input
## Data
## Note: If variables are not included, then code them completely as NA,
## including the diagonals (i.e., not 1 but NA in the diagonal for
## completely missing variables)
Auth_mat <-
'
1,
0.54, 1,
0.11, 0.39, 1,
0.08, 0.18, 0.32, 1,
0.36, 0.25, 0.15, -0.07, 1'
Barkul_mat_a <-
'
1,
0.95, 1,
0.47, 0.56, 1,
0.86, 0.84, 0.56, 1,
0.47, 0.49, 0.38, 0.6, 1
'
Barkul_mat_b <- '
1,
0.89, 1,
0.31, 0.47, 1,
0.64, 0.62, 0.38, 1,
0.28, 0.24, NA, 0.43, 1'
Berman_mat <-
'
1,
0.39, 1,
0.14, -0.23, 1,
-0.38, 0.14, 0.05, 1,
0.004, -0.29, 0.80, 0.25, 1
'
# Adjusted
Boynton_mat <-
'1
0.576, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA
'
Callans_mat <-
'1
0.78, 1,
0.09, -0.01, 1,
0.26, 0.26, 0.27, 1,
0.51, 0.43, 0.42, 0.38, 1
'
#Carter Control Group Pre-Test
Carter_mat_a <-
'1
0.86, 1,
0.31, 0.28, 1,
0.05, 0.05, 0.54, 1,
0.81, 0.72, 0.49, 0.21, 1
'
#Carter Experimental Group Pre-Test
Carter_mat_b <-
'1,
0.82, 1,
0.62, 0.39, 1,
0.51, 0.40, 0.28, 1,
0.68, 0.60, 0.40, 0.33, 1
'
Chi_mat <-'
1,
0.433, 1,
-0.02, 0.42, 1,
0.18, 0.37, 0.42, 1,
-0.19, 0.4, 0.54, 0.45, 1,
'
Cho_mat_F <- '
1,
0.61, 1,
0.42, 0.43, 1,
0.32, 0.21, 0.58, 1,
0.69, 0.33, 0.63, 0.54, 1'
#Changed to Male correlation matrices from total
Cho_mat_M <- '
1,
0.59, 1,
0.26, 0.02, 1,
0.14, 0.31, 0.36, 1,
0.82, 0.69, 0.25, 0.24, 1,
'
Cockcroft_mat <- '
1,
0.29, 1,
0.42, 0.12, 1,
0.24, 0.65, 0.31, 1,
0.18, 0.31, 0.31, 0.46, 1 '
Conway_mat <-
'1,
0.82, 1,
0.45, 0.47, 1,
0.56, 0.56, 0.68, 1,
0.50, 0.43, 0.49, 0.65, 1
'
Crawford_mat <-
'1
0.94, 1,
0.38, 0.44, 1,
0.37, 0.48, 0.38, 1,
0.67, 0.72, 0.65, 0.40, 1
'
Digranes_mat <-
'1,
0.76, 1,
0.17, 0.45, 1,
0.80, 0.71, 0.09, 1,
0.81, 0.79, 0.37, 0.74, 1
'
## Adjusted
Dufner_mat <-
'1,
0.45, 1,
0.11, 0.09, 1,
-0.07, 0.42, 0.16, 1,
NA, NA, NA, NA, NA
'
Fishkin_mat <-
'1
0.60, 1,
0, -0.03, 1,
0.13, 0.11, 0.11, 1,
0.19, 0.26, 0.01, 0.09, 1
'
Forsyth_mat <-
'1,
0.91, 1,
0.56, 0.49, 1,
0.35, 0.40, 0.50, 1,
0.82, 0.77, 0.62, 0.33, 1'
#8th grade
Garcia_mat <-
'1,
0.626, 1,
0.392, 0.45, 1,
0.435, 0.387, 0.495, 1,
0.416, 0.525, 0.32, 0.567, 1'
Gollmar_mat <-
'1,
0.672, 1,
-0.26, 0.019, 1,
-0.13, 0.241, 0.377, 1,
0.188, 0.217, 0.22, 0.296, 1'
## Adjusted
Hamlen_mat <-'
1,
0.816, 1,
NA, NA, NA,
NA, NA, NA, NA,
0.319, 0.247, NA, NA, 1 '
Hokanson_mat <-
'1,
0.79, 1,
0.14, 0.28, 1,
0.32, 0.44, 0.48, 1,
0.48, 0.45, 0.36, 0.36, 1'
Houtz_mat <- '
1,
0.59, 1,
0.27, 0.16, 1,
0.49, 0.27, 0.22, 1,
0.5, 0.34, 0.46, 0.37, 1 '
Humble_mat <-
'1,
0.83, 1,
0.22, 0.28 1
0.44 0.38 0.06 1
0.34 0.25 0.07 0.37 1'
Ibrahim_mat_1 <- '
1,
0.611, 1,
0.524, 0.12, 1,
0.23, 0.246, 0.267, 1,
0.46, 0.485, 0.155, 0.484, 1 '
Kiehn_mat <-
'1,
0.8, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA
'
#Kim (2006b)
Kim_mat_1 <- '
1,
0.844, 1,
0.351, 0.332, 1,
0.196, 0.209, 0.428, 1,
0.666, 0.563, 0.212, 0.443, 1
'
#Kim et al.(2006)
Kim_mat_2a <- '
1,
0.84, 1,
0.45, 0.49, 1,
0.39, 0.38, 0.49, 1,
0.65, 0.65, 0.34, 0.51, 1 '
Kim_mat_2b <- '
1,
0.79, 1,
0.08, 0.15, 1,
0.14, 0.15, 0.3, 1,
0.47, 0.48, 0.17, 0.25, 1 '
Kim_mat_2c <- '
1,
0.86, 1,
0.31, 0.32, 1,
0.22, 0.25, 0.41, 1,
0.66, 0.57, 0.2, 0.4, 1
'
#Kim et al. 2016
Kim_mat_3_a <- '
1,
0.46, 1,
-0.28, 0.24, 1,
0.1, 0.4, 0.51, 1,
-0.28, 0.12, 0.6, 0.66, 1'
Kim_mat_3_b <- '
1,
0.4, 1,
-0.27, 0.23, 1,
0.08, 0.34, 0.5, 1,
-0.22, 0.17, 0.64, 0.72, 1
'
Lew_mat <- '
1,
0.46, 1,
-0.15, 0.04, 1,
0.14, 0.36, 0.24, 1,
NA, NA, NA, NA, NA '
## Adjusted
Miranda_mat <-
'1,
0.49, 1,
0.02, 0.11, 1,
-0.25, -0.30, -0.31, 1,
0.41, 0.20, 0.14, -0.29, 1'
## Adjusted
#1st grade
Morrison_mat_a <-
'1,
0.36, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA'
## Adjusted
#3rd grade
Morrison_mat_b <-
'1,
0.28, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA'
## Adjusted
#5th grade
Morrison_mat_c <-
'1,
0.42, 1,
NA, NA, NA,
NA, NA, NA, NA,
NA, NA, NA, NA, NA'
Nguyen_mat <-
'1,
0.75, 1,
0.11, 0.09, 1,
0.17, 0.10, 0.15, 1,
0.38, 0.34, 0.13, 0.03, 1'
Zbarskaya_mat <-
'1,
0.614, 1,
0.008, 0.135, 1,
0.168, 0.327, 0.504, 1,
0.189, 0.153, 0.225, 0.442, 1'
#Children's house
Rose_mat_a <-
'1,
0.81, 1,
0.07, 0.27, 1,
0.66, 0.62, 0.57, 1,
-0.05, -0.06, -0.19, -0.12, 1'
#Montessori
Rose_mat_b <-
'1,
0.67, 1,
0.05, 0.13, 1,
0.56, 0.16, -0.30, 1,
0.34, 0.12, 0.43, 0.19, 1'
Roskos_mat_Y <-'
1,
0.26, 1,
0.15, 0.08, 1,
0.37, 0.47, 0.2, 1,
0.53, 0.31, 0.47, 0.37, 1 '
Roskos_mat_O <- '
1,
0.16, 1,
-0.06, -0.02, 1,
0.06, 0.25, 0.04, 1,
0.14, 0.29, 0.36, 0.47, 1
'
Samuels_mat <-
'1,
0.75, 1,
-0.11, 0.02, 1,
0.10, 0.24, 0.48, 1,
-0.16, -0.06, 0.07, 0.24, 1'
Shore_mat <-
'1,
0.26, 1,
-0.01, 0.33, 1,
-0.12, 0.70, 0.30, 1,
-0.29, 0.12, 0.54, 0.34, 1'
Stephens_mat <-
'1,
0.86, 1,
0.17, 0.16, 1,
0.33, 0.30, 0.12, 1,
0.65, 0.60, 0.30, 0.31, 1'
#Control Pre-Test
Storer_mat_a <-
'1,
0.77, 1,
0.11, 0.29, 1,
0.28, 0.19, 0.35, 1,
0.33, 0.49, -0.02, -0.14, 1'
#Treatment Pre-Test
Storer_mat_b <-
'1,
0.73, 1,
0.23, 0.20, 1,
0.43, 0.55, 0.29, 1,
0.57, 0.48, 0.40, 0.25, 1'
Tannehill_mat <-
'1,
0.84, 1,
0.35, 0.41, 1,
0.28, 0.36, 0.38, 1,
0.35, 0.33, 0.39, 0.41, 1'
Tisone_mat <-
'1,
0.68, 1,
-0.03, -0.12, 1,
0.15, 0.26, 0.30, 1,
0.13, 0.35, 0.11, 0.53, 1'
Trigani_mat <-
'1,
0.72, 1,
0.28, 0.24, 1,
0.17, 0.26, 0.37, 1,
0.69, 0.47, 0.52, 0.16, 1'
Voss_mat <-
'1,
0.71, 1,
0.43, 0.51, 1,
0.38, 0.44, 0.53, 1,
0.65, 0.79, 0.59, 0.42, 1'
Warne_mat <-
'1,
0.74, 1,
0.40, 0.37, 1,
0.35, 0.37, 0.43, 1,
0.63, 0.50, 0.50, 0.26, 1'
Watson_mat <-
'1,
0.64, 1,
NA, NA, NA,
0.41, 0.40, NA, 1,
NA, NA, NA, NA, NA'
Yoon_mat <-
'1,
0.73, 1,
-0.16, 0.08, 1,
0.08, 0.29, 0.47, 1,
0.09, 0.28, 0.53, 0.66, 1'
Zhang_mat <-
'1,
0.78, 1,
0.09, 0.15, 1,
0.25, 0.24, 0.33, 1,
0.83, 0.70, 0.06, 0.21, 1'
## New matrices after the update in 2023
Acar_mat <-
'1,
0.65, 1,
0.188, 0.274, 1,
0.338, 0.409, 0.363, 1,
0.529, 0.477, 0.413, 0.305, 1'
Acaretal_mat <-
'1,
0.763, 1,
0.089, 0.21, 1,
0.294, 0.367, 0.469, 1,
0.567, 0.54, 0.277, 0.318, 1'
Balci_mata <-
'1,
0.292, 1,
0, 0.221, 1,
0.176, 0.148, 0.304, 1,
-0.117, 0.154, 0.035, 0.047, 1'
Balci_matb <-
'1,
0.43, 1,
0.116, 0.166, 1,
0.267, 0.225, 0.363, 1,
-0.19, 0.016, -0.128, -0.107, 1'
Gao_mat <-
'1,
0.31, 1,
0.28, 0.34, 1,
0.39, 0.41, 0.25, 1,
0.28, 0.17, 0.26, 0.31, 1'
Liu_mat <-
'1,
0.94, 1,
0.32, 0.32, 1,
0.49, 0.48, 0.66, 1,
0.41, 0.39, 0.62, .67, 1'
Rubenstein_2020_a <-
'1,
0.828, 1,
0.331, 0.321, 1,
0.47, 0.55, 0.45, 1,
0.681, 0.64, 0.443, 0.526, 1'
Rubenstein_2022_b <-
'1,
0.629, 1,
0.238, 0.351, 1,
0.402, 0.509, 0.38, 1,
0.313, 0.394, 0.385, 0.305, 1'
Wan_mat <-
'1,
0.27, 1,
0.24, 0.35, 1,
0.17, 0.19, 0.27, 1,
0.19, 0.14, 0.12, 0.35, 1'
#This names columns and rows for correlation matrices
Auth_mat1 <-char2num(Auth_mat)
Auth <- getCov(Auth_mat1, diag=T)
colnames(Auth) <- c("Fl","Or", "El","Ab","Res")
rownames(Auth) <- c("Fl","Or", "El","Ab","Res")
#Barkul
Barkul_mat1 <-char2num(Barkul_mat_a)
Barkul <- getCov(Barkul_mat1, diag=T)
colnames(Barkul) <- c("Fl","Or", "El","Ab","Res")
rownames(Barkul) <- c("Fl","Or", "El","Ab","Res")
Barkul_mat2 <-char2num(Barkul_mat_b)
Barkul2 <- getCov(Barkul_mat2, diag=T)
colnames(Barkul2) <- c("Fl","Or", "El","Ab","Res")
rownames(Barkul2) <- c("Fl","Or", "El","Ab","Res")
Berman_mat1 <-char2num(Berman_mat)
Berman <- getCov(Berman_mat1, diag=T)
colnames(Berman) <- c("Fl","Or", "El","Ab","Res")
rownames(Berman) <- c("Fl","Or", "El","Ab","Res")
Boynton_mat1 <-char2num(Boynton_mat)
Boynton <- getCov(Boynton_mat1, diag=T)
colnames(Boynton) <- c("Fl","Or", "El","Ab","Res")
rownames(Boynton) <- c("Fl","Or", "El","Ab","Res")
Callans_mat1 <-char2num(Callans_mat)
Callans <- getCov(Callans_mat1, diag=T)
colnames(Callans) <- c("Fl","Or", "El","Ab","Res")
rownames(Callans) <- c("Fl","Or", "El","Ab","Res")
Carter_mat1 <-char2num(Carter_mat_a)
Carter_a <- getCov(Carter_mat1, diag=T)
colnames(Carter_a) <- c("Fl","Or", "El","Ab","Res")
rownames(Carter_a) <- c("Fl","Or", "El","Ab","Res")
Carter_mat2 <-char2num(Carter_mat_b)
Carter_b <- getCov(Carter_mat2, diag=T)
colnames(Carter_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Carter_b) <- c("Fl","Or", "El","Ab","Res")
Chi_mat1 <-char2num(Chi_mat)
Chi <- getCov(Chi_mat1, diag=T)
colnames(Chi) <- c("Fl","Or", "Ab","El","Res")
rownames(Chi) <- c("Fl","Or", "Ab","El","Res")
Cho_mat1_F <-char2num(Cho_mat_F)
Cho_F <- getCov(Cho_mat1_F, diag=T)
colnames(Cho_F) <- c("Fl","Or", "Ab","El","Res")
rownames(Cho_F) <- c("Fl","Or", "Ab","El","Res")
Cho_mat1_M <-char2num(Cho_mat_M)
Cho_M <- getCov(Cho_mat1_M, diag=T)
colnames(Cho_M) <- c("Fl","Or", "Ab","El","Res")
rownames(Cho_M) <- c("Fl","Or", "Ab","El","Res")
Cockcroft_mat1 <-char2num(Cockcroft_mat)
Cockcroft <- getCov(Cockcroft_mat1, diag=T)
colnames(Cockcroft) <- c("Fl","Or", "Ab","El","Res")
rownames(Cockcroft) <- c("Fl","Or", "Ab","El","Res")
Conway_mat1 <-char2num(Conway_mat)
Conway <- getCov(Conway_mat1, diag=T)
colnames(Conway) <- c("Fl","Or", "El","Ab","Res")
rownames(Conway) <- c("Fl","Or", "El","Ab","Res")
Crawford_mat1 <-char2num(Crawford_mat)
Crawford <- getCov(Crawford_mat1, diag=T)
colnames(Crawford) <- c("Fl","Or", "El","Ab","Res")
rownames(Crawford) <- c("Fl","Or", "El","Ab","Res")
Digranes_mat1 <-char2num(Digranes_mat)
Digranes <- getCov(Digranes_mat1, diag=T)
colnames(Digranes) <- c("Fl","Or", "El","Ab","Res")
rownames(Digranes) <- c("Fl","Or", "El","Ab","Res")
Dufner_mat1 <-char2num(Dufner_mat)
Dufner <- getCov(Dufner_mat1, diag=T)
colnames(Dufner) <- c("Fl","Or", "El","Ab","Res")
rownames(Dufner) <- c("Fl","Or", "El","Ab","Res")
Fishkin_mat1 <-char2num(Fishkin_mat)
Fishkin <- getCov(Fishkin_mat1, diag=T)
colnames(Fishkin) <- c("Fl","Or", "El","Ab","Res")
rownames(Fishkin) <- c("Fl","Or", "El","Ab","Res")
Forsyth_mat1 <-char2num(Forsyth_mat)
Forsyth <- getCov(Forsyth_mat1, diag=T)
colnames(Forsyth) <- c("Fl","Or", "El","Ab","Res")
rownames(Forsyth) <- c("Fl","Or", "El","Ab","Res")
Garcia_mat1 <-char2num(Garcia_mat)
Garcia <- getCov(Garcia_mat1, diag=T)
colnames(Garcia) <- c("Fl","Or", "El","Ab","Res")
rownames(Garcia) <- c("Fl","Or", "El","Ab","Res")
Gollmar_mat1 <-char2num(Gollmar_mat)
Gollmar <- getCov(Gollmar_mat1, diag=T)
colnames(Gollmar) <- c("Fl","Or", "El","Ab","Res")
rownames(Gollmar) <- c("Fl","Or", "El","Ab","Res")
Hamlen_mat1 <-char2num(Hamlen_mat)
Hamlen<- getCov(Hamlen_mat1, diag=T)
colnames(Hamlen) <- c("Fl","Or", "Ab","El","Res")
rownames(Hamlen) <- c("Fl","Or", "Ab","El","Res")
Hokanson_mat1 <-char2num(Hokanson_mat)
Hokanson <- getCov(Hokanson_mat1, diag=T)
colnames(Hokanson) <- c("Fl","Or", "El","Ab","Res")
rownames(Hokanson) <- c("Fl","Or", "El","Ab","Res")
Houtz_mat1 <-char2num(Houtz_mat)
Houtz<- getCov(Houtz_mat1, diag=T)
colnames(Houtz) <- c("Fl","Or", "Ab","El","Res")
rownames(Houtz) <- c("Fl","Or", "Ab","El","Res")
Humble_mat1 <-char2num(Humble_mat)
Humble <- getCov(Humble_mat1, diag=T)
colnames(Humble) <- c("Fl","Or", "El","Ab","Res")
rownames(Humble) <- c("Fl","Or", "El","Ab","Res")
Ibrahim_mat1 <-char2num(Ibrahim_mat_1)
Ibrahim<- getCov(Ibrahim_mat1, diag=T)
colnames(Ibrahim) <- c("Fl","Or", "Ab","El","Res")
rownames(Ibrahim) <- c("Fl","Or", "Ab","El","Res")
Kiehn_mat1 <-char2num(Kiehn_mat)
Kiehn <- getCov(Kiehn_mat1, diag=T)
colnames(Kiehn) <- c("Fl","Or", "El","Ab","Res")
rownames(Kiehn) <- c("Fl","Or", "El","Ab","Res")
Kim_1_mat1 <-char2num(Kim_mat_1)
Kim_1<- getCov(Kim_1_mat1, diag=T)
colnames(Kim_1) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_1) <- c("Fl","Or", "Ab","El","Res")
Kim_2a_mat1 <-char2num(Kim_mat_2a)
Kim_2a<- getCov(Kim_2a_mat1, diag=T)
colnames(Kim_2a) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_2a) <- c("Fl","Or", "Ab","El","Res")
Kim_2b_mat1 <-char2num(Kim_mat_2b)
Kim_2b<- getCov(Kim_2b_mat1, diag=T)
colnames(Kim_2b) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_2b) <- c("Fl","Or", "Ab","El","Res")
Kim_2c_mat1 <-char2num(Kim_mat_2c)
Kim_2c<- getCov(Kim_2c_mat1, diag=T)
colnames(Kim_2c) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_2c) <- c("Fl","Or", "Ab","El","Res")
Kim_3_mat1 <-char2num(Kim_mat_3_a)
Kim_3a <- getCov(Kim_3_mat1, diag=T)
colnames(Kim_3a) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_3a) <- c("Fl","Or", "Ab","El","Res")
Kim_3_mat2 <-char2num(Kim_mat_3_b)
Kim_3b <- getCov(Kim_3_mat2, diag=T)
colnames(Kim_3b) <- c("Fl","Or", "Ab","El","Res")
rownames(Kim_3b) <- c("Fl","Or", "Ab","El","Res")
Lew_mat1 <-char2num(Lew_mat)
Lew<- getCov(Lew_mat1, diag=T)
colnames(Lew) <- c("Fl","Or", "Ab","El","Res")
rownames(Lew) <- c("Fl","Or", "Ab","El","Res")
Miranda_mat1 <-char2num(Miranda_mat)
Miranda <- getCov(Miranda_mat1, diag=T)
colnames(Miranda) <- c("Fl","Or", "El","Ab","Res")
rownames(Miranda) <- c("Fl","Or", "El","Ab","Res")
Morrison_mat1 <-char2num(Morrison_mat_a)
Morrison_a <- getCov(Morrison_mat1, diag=T)
colnames(Morrison_a) <- c("Fl","Or", "El","Ab","Res")
rownames(Morrison_a) <- c("Fl","Or", "El","Ab","Res")
Morrison_mat2 <-char2num(Morrison_mat_b)
Morrison_b <- getCov(Morrison_mat2, diag=T)
colnames(Morrison_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Morrison_b) <- c("Fl","Or", "El","Ab","Res")
Morrison_mat3 <-char2num(Morrison_mat_c)
Morrison_c <- getCov(Morrison_mat3, diag=T)
colnames(Morrison_c) <- c("Fl","Or", "El","Ab","Res")
rownames(Morrison_c) <- c("Fl","Or", "El","Ab","Res")
Nguyen_mat1 <-char2num(Nguyen_mat)
Nguyen <- getCov(Nguyen_mat1, diag=T)
colnames(Nguyen) <- c("Fl","Or", "El","Ab","Res")
rownames(Nguyen) <- c("Fl","Or", "El","Ab","Res")
Zbarskaya_mat1 <-char2num(Zbarskaya_mat)
Zbarskaya <- getCov(Zbarskaya_mat1, diag=T)
colnames(Zbarskaya) <- c("Fl","Or", "El","Ab","Res")
rownames(Zbarskaya) <- c("Fl","Or", "El","Ab","Res")
#Children's Place
Rose_mat1 <- char2num(Rose_mat_a)
Rose <- getCov(Rose_mat1, diag=T)
colnames(Rose) <- c("Fl","Or", "El","Ab","Res")
rownames(Rose) <- c("Fl","Or", "El","Ab","Res")
#Montessori
Rose_mat2 <- char2num(Rose_mat_b)
Rose_b <- getCov(Rose_mat2, diag=T)
colnames(Rose_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Rose_b) <- c("Fl","Or", "El","Ab","Res")
Roskos_mat1_Y <-char2num(Roskos_mat_Y)
Roskos_Y<- getCov(Roskos_mat1_Y, diag=T)
colnames(Roskos_Y) <- c("Fl","Or", "Ab","El","Res")
rownames(Roskos_Y) <- c("Fl","Or", "Ab","El","Res")
Roskos_mat1_O <-char2num(Roskos_mat_O)
Roskos_O<- getCov(Roskos_mat1_O, diag=T)
colnames(Roskos_O) <- c("Fl","Or", "Ab","El","Res")
rownames(Roskos_O) <- c("Fl","Or", "Ab","El","Res")
Samuels_mat1 <-char2num(Samuels_mat)
Samuels <- getCov(Samuels_mat1, diag=T)
colnames(Samuels) <- c("Fl","Or", "El","Ab","Res")
rownames(Samuels) <- c("Fl","Or", "El","Ab","Res")
Shore_mat1 <-char2num(Shore_mat)
Shore <- getCov(Shore_mat1, diag=T)
colnames(Shore) <- c("Fl","Or", "El","Ab","Res")
rownames(Shore) <- c("Fl","Or", "El","Ab","Res")
Stephens_mat1 <-char2num(Stephens_mat)
Stephens <- getCov(Stephens_mat1, diag=T)
colnames(Stephens) <- c("Fl","Or", "El","Ab","Res")
rownames(Stephens) <- c("Fl","Or", "El","Ab","Res")
Storer_mat1 <- char2num(Storer_mat_a)
Storer_a <- getCov(Rose_mat1, diag=T)
colnames(Storer_a) <- c("Fl","Or", "El","Ab","Res")
rownames(Storer_a) <- c("Fl","Or", "El","Ab","Res")
Storer_mat2 <- char2num(Storer_mat_b)
Storer_b <- getCov(Storer_mat2, diag=T)
colnames(Storer_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Storer_b) <- c("Fl","Or", "El","Ab","Res")
Tannehill_mat1 <-char2num(Tannehill_mat)
Tannehill <- getCov(Tannehill_mat1, diag=T)
colnames(Tannehill) <- c("Fl","Or", "El","Ab","Res")
rownames(Tannehill) <- c("Fl","Or", "El","Ab","Res")
Tisone_mat1 <-char2num(Tisone_mat)
Tisone <- getCov(Tisone_mat1, diag=T)
colnames(Tisone) <- c("Fl","Or", "El","Ab","Res")
rownames(Tisone) <- c("Fl","Or", "El","Ab","Res")
Trigani_mat1 <-char2num(Trigani_mat)
Trigani <- getCov(Trigani_mat1, diag=T)
colnames(Trigani) <- c("Fl","Or", "El","Ab","Res")
rownames(Trigani) <- c("Fl","Or", "El","Ab","Res")
Voss_mat1 <-char2num(Voss_mat)
Voss <- getCov(Voss_mat1, diag=T)
colnames(Voss) <- c("Fl","Or", "El","Ab","Res")
rownames(Voss) <- c("Fl","Or", "El","Ab","Res")
Warne_mat1 <-char2num(Warne_mat)
Warne <- getCov(Warne_mat1, diag=T)
colnames(Warne) <- c("Fl","Or", "El","Ab","Res")
rownames(Warne) <- c("Fl","Or", "El","Ab","Res")
Watson_mat1 <-char2num(Watson_mat)
Watson <- getCov(Watson_mat1, diag=T)
colnames(Watson) <- c("Fl","Or", "El","Ab","Res")
rownames(Watson) <- c("Fl","Or", "El","Ab","Res")
Yoon_mat1 <-char2num(Yoon_mat)
Yoon <- getCov(Yoon_mat1, diag=T)
colnames(Yoon) <- c("Fl","Or", "El","Ab","Res")
rownames(Yoon) <- c("Fl","Or", "El","Ab","Res")
Zhang_mat1 <-char2num(Zhang_mat)
Zhang <- getCov(Zhang_mat1, diag=T)
colnames(Zhang) <- c("Fl","Or", "El","Ab","Res")
rownames(Zhang) <- c("Fl","Or", "El","Ab","Res")
## New studies after the update in 2023
Acar_mat1 <-char2num(Acar_mat)
Acar <- getCov(Acar_mat1, diag=T)
colnames(Acar) <- c("Fl","Or", "El","Ab","Res")
rownames(Acar) <- c("Fl","Or", "El","Ab","Res")
Acaretal_mat1 <-char2num(Acaretal_mat)
Acaretal <- getCov(Acaretal_mat1, diag=T)
colnames(Acaretal) <- c("Fl","Or", "El","Ab","Res")
rownames(Acaretal) <- c("Fl","Or", "El","Ab","Res")
Balcia_mat1 <-char2num(Balci_mata)
Balcia <- getCov(Balcia_mat1, diag=T)
colnames(Balcia) <- c("Fl","Or", "El","Ab","Res")
rownames(Balcia) <- c("Fl","Or", "El","Ab","Res")
Balcib_mat1 <-char2num(Balci_matb)
Balcib <- getCov(Balcib_mat1, diag=T)
colnames(Balcib) <- c("Fl","Or", "El","Ab","Res")
rownames(Balcib) <- c("Fl","Or", "El","Ab","Res")
Gao_mat1 <-char2num(Gao_mat)
Gao <- getCov(Gao_mat1, diag=T)
colnames(Gao) <- c("Fl","Or", "El","Ab","Res")
rownames(Gao) <- c("Fl","Or", "El","Ab","Res")
Liu_mat1 <-char2num(Liu_mat)
Liu <- getCov(Liu_mat1, diag=T)
colnames(Liu) <- c("Fl","Or", "El","Ab","Res")
rownames(Liu) <- c("Fl","Or", "El","Ab","Res")
Wan_mat1 <-char2num(Wan_mat)
Wan <- getCov(Wan_mat1, diag=T)
colnames(Wan) <- c("Fl","Or", "El","Ab","Res")
rownames(Wan) <- c("Fl","Or", "El","Ab","Res")
Rubenstein_2020_a1 <-char2num(Rubenstein_2020_a)
Rubenstein_a <- getCov(Rubenstein_2020_a1, diag=T)
colnames(Rubenstein_a) <- c("Fl","Or", "El","Ab","Res")
rownames(Rubenstein_a) <- c("Fl","Or", "El","Ab","Res")
Rubenstein_2022_b1 <-char2num(Rubenstein_2022_b)
Rubenstein_b <- getCov(Rubenstein_2022_b1, diag=T)
colnames(Rubenstein_b) <- c("Fl","Or", "El","Ab","Res")
rownames(Rubenstein_b) <- c("Fl","Or", "El","Ab","Res")
#alphabetical order
data <- list(Acar, Acaretal,Auth, Balcia, Balcib, Barkul, Barkul2, Berman, Boynton, Callans,
Carter_a, Carter_b, Chi, Cho_F, Cho_M, Cockcroft,
Conway, Crawford, Digranes, Dufner, Fishkin, Forsyth, Gao, Garcia, Gollmar,
Hamlen, Hokanson, Houtz, Humble, Ibrahim,
Kiehn, Kim_1, Kim_2a, Kim_2b, Kim_2c, Kim_3a, Kim_3b,
Lew, Liu, Miranda, Morrison_a, Morrison_b, Morrison_c,
Nguyen, Rose, Rose_b, Roskos_Y, Roskos_O, Rubenstein_a, Rubenstein_b, Samuels, Shore, Stephens,
Storer_a, Storer_b, Tannehill, Tisone, Trigani,
Voss, Wan, Warne, Watson, Yoon, Zbarskaya, Zhang)
#Updated sample size counts
n <- c(477, 375, 30, 264, 105, 599, 147, 13, 62, 60, 24, 24, 203, 24, 35, 36, 25, 21, 17, 98, 116, 45, 319, 95, 128, 118, 1758,
42, 125, 99, 89, 500, 1000, 1000, 1000, 125, 137, 135, 1047, 12, 184, 122, 121, 187, 12, 19, 39, 31, 371,371,
51, 18, 84, 43, 46, 199, 24, 107, 120, 95, 432, 6, 163, 125, 1067)
#alphabetical order
names <- c("Acar", "Acaretal","Auth" , "Balcia", "Balcib", "Barkul" , "Barkul2" , "Berman" , "Boynton" , "Callans" ,
"Carter_a" , "Carter_b" , "Chi" , "Cho_F" , "Cho_M" , "Cockcroft" ,
"Conway" , "Crawford",
"Digranes" , "Dufner" , "Fishkin" , "Forsyth" , "Gao", "Garcia" , "Gollmar" ,
"Hamlen" , "Hokanson" , "Houtz" , "Humble" , "Ibrahim",
"Kiehn" , "Kim_1" , "Kim_2a" , "Kim_2b" , "Kim_2c" , "Kim_3a" , "Kim_3b" ,
"Lew" , "Liu", "Miranda" , "Morrison_a" , "Morrison_b" , "Morrison_c" ,
"Nguyen" , "Rose" , "Rose_b" , "Roskos_Y" , "Roskos_O" , "Rubenstein_a","Rubenstein_b", "Samuels" , "Shore" , "Stephens" ,
"Storer_a" , "Storer_b" , "Tannehill" , "Tisone" , "Trigani", "Voss" ,"Wan", "Warne" , "Watson" , "Yoon" , "Zbarskaya" , "Zhang")
#alphabetical order
study.names <- c("Acar", "Acaretal","Auth" , "Balcia", "Balcib", "Barkul" , "Barkul2" , "Berman" , "Boynton" , "Callans" ,
"Carter_a" , "Carter_b" , "Chi" , "Cho_F" , "Cho_M" , "Cockcroft" ,
"Conway" , "Crawford",
"Digranes" , "Dufner" , "Fishkin" , "Forsyth" , "Gao", "Garcia" , "Gollmar" ,
"Hamlen" , "Hokanson" , "Houtz" , "Humble" , "Ibrahim",
"Kiehn" , "Kim_1" , "Kim_2a" , "Kim_2b" , "Kim_2c" , "Kim_3a" , "Kim_3b" ,
"Lew" , "Liu", "Miranda" , "Morrison_a" , "Morrison_b" , "Morrison_c" ,
"Nguyen" , "Rose" , "Rose_b" , "Roskos_Y" , "Roskos_O" , "Rubenstein_a","Rubenstein_b", "Samuels" , "Shore" , "Stephens" , "Storer_a" , "Storer_b" , "Tannehill" , "Tisone" , "Trigani", "Voss" ,"Wan", "Warne" , "Watson" , "Yoon" , "Zbarskaya" , "Zhang")
names(data) <- study.names
## Possible moderators
## Age group
## Code: 1 = Adults, 0 = Kindergarten up to High/Middle school
adults <- c(1,0,1,0,0,1,1,1,1,1,0,0,0,1,1,0,1,0,0,0,0,1,1,0,0,0,0,1,0,1,0,0,
0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,1,0,1,1,1,0,1,0)
## Evidence against discriminant validity
## Coding: 1 = evidence present, 0 = evidence not present/reported
validity <- c(1,1,0,0,1,0,1,0,1,0,1,1,0,1,1,1,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,
0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0)
## Test forms
## Subset the data
formsAB <- c("A","A",NA,"Both","Both","A","A","A","Both","A","A","A","A",NA,NA,"B","A","A","A","B","A","Both",NA,"B","A","A","A","B","A","A","A","A","A","A","A","A","A",NA,"A","A","A","A","A","A","A","A","B","B","A","B","A","A","A","A","A","A","A","A","A",NA,"A","B","A","A","A")
forms <- as.vector(as.numeric(which(formsAB %in% c(NA, "Both")))) # which gives the positions of codes 1
## Standardized test scores
## Subset the data
scoring <- c(0,0,1,0,0,0,0,NA,0,1,0,0,0,0,0,0,1,1,0,0,0,1,1,1,1,1,0,0,1,1,1,1,0,0,0,1,1,1,1,0,
NA,NA,NA,1,0,0,1,1,0,0,1,0,1,1,1,1,0,1,1,NA,0,0,1,1,0)
scores <- as.vector(as.numeric(which(scoring %in% NA))) # which gives the positions of codes 1
head(data, 5)## $Acar
## Fl Or El Ab Res
## Fl 1.000 0.650 0.188 0.338 0.529
## Or 0.650 1.000 0.274 0.409 0.477
## El 0.188 0.274 1.000 0.363 0.413
## Ab 0.338 0.409 0.363 1.000 0.305
## Res 0.529 0.477 0.413 0.305 1.000
##
## $Acaretal
## Fl Or El Ab Res
## Fl 1.000 0.763 0.089 0.294 0.567
## Or 0.763 1.000 0.210 0.367 0.540
## El 0.089 0.210 1.000 0.469 0.277
## Ab 0.294 0.367 0.469 1.000 0.318
## Res 0.567 0.540 0.277 0.318 1.000
##
## $Auth
## Fl Or El Ab Res
## Fl 1.00 0.54 0.11 0.08 0.36
## Or 0.54 1.00 0.39 0.18 0.25
## El 0.11 0.39 1.00 0.32 0.15
## Ab 0.08 0.18 0.32 1.00 -0.07
## Res 0.36 0.25 0.15 -0.07 1.00
##
## $Balcia
## Fl Or El Ab Res
## Fl 1.000 0.292 0.000 0.176 -0.117
## Or 0.292 1.000 0.221 0.148 0.154
## El 0.000 0.221 1.000 0.304 0.035
## Ab 0.176 0.148 0.304 1.000 0.047
## Res -0.117 0.154 0.035 0.047 1.000
##
## $Balcib
## Fl Or El Ab Res
## Fl 1.000 0.430 0.116 0.267 -0.190
## Or 0.430 1.000 0.166 0.225 0.016
## El 0.116 0.166 1.000 0.363 -0.128
## Ab 0.267 0.225 0.363 1.000 -0.107
## Res -0.190 0.016 -0.128 -0.107 1.0002 Parameter-Based Meta-Analytic Structural Equation Modeling (pbMASEM)
Parameter-based MASEM comprises two steps: (1) Model-based generation of the relevant effect sizes and their sampling (co-)variances; and (2) Meta-analysis of the effect sizes.
2.1 Stage-1 Analysis: Model-Based Generation of Effect Sizes
2.1.1 Model Specification
In this step, we specify the analytic models for reliability
estimation. The extant literature indicated that several models may
describe the structure of the TTCT: (a) single-factor model with a
general TTCT factor (gTTCT); (b) two-factor model with a
factor representing innovativeness (INNOV) and a factor
describing adaptability (ADAPT); and (c) the two-factor
model from (b) with a cross-loading of the indicator Res.
In the following section, we specify these models and define the
reliability estimates.
## Model 1
## Single-factor model
SingleFactorModel <- " # Measurement Model
gCT =~ L1*Fl + L2*Or + L3*El + L4*Ab + L5*Res
# Residual variances
Fl ~~ R1*Fl
Or ~~ R2*Or
El ~~ R3*El
Ab ~~ R4*Ab
Res ~~ R5*Res
# Factor variances fixed to 1
gCT ~~ 1*gCT
# Additional constraints
R1 > 0
R2 > 0
R3 > 0
R4 > 0
R5 > 0
"
## Model 4
## Two-factor model with correlated traits ADAPT and INNOV and Res assigned to ADAPT
TwoFactorModel <- " # Measurement Model
Innov =~ L1*Fl + L2*Or
Adapt =~ L3*El + L4*Ab + L5*Res
# Residual variances
Fl ~~ R1*Fl
Or ~~ R2*Or
El ~~ R3*El
Ab ~~ R4*Ab
Res ~~ R5*Res
# Factor variances fixed to 1
Adapt ~~ 1*Adapt
Innov ~~ 1*Innov
# Factor correlation
Adapt ~~ fcorr*Innov
# Reliability coefficients
SREL1 := ((L1+L2)^2)/((L1+L2)^2 +
R1+R2)
SREL2 := ((L3+L4+L5)^2)/((L3+L4+L5)^2 +
R3+R4+R5)
# Composite reliability by Fu et al. (2022)
SRELCR := ((L1+L2)^2+(L3+L4+L5)^2+2*fcorr*(L1+L2)*(L3+L4+L5))/
((L1+L2)^2+(L3+L4+L5)^2+2*fcorr*(L1+L2)*(L3+L4+L5) +
R1+R2+R3+R4+R5)
# Additional constraints
R1 > 0
R2 > 0
R3 > 0
R4 > 0
R5 > 0
"
## Model 2
## Two-factor model with correlated traits ADAPT and INNOV and Res assigned to Innov
TwoFactorModelB <- " # Measurement Model
Innov =~ L1*Fl + L2*Or + L5*Res
Adapt =~ L3*El + L4*Ab
# Residual variances
Fl ~~ R1*Fl
Or ~~ R2*Or
El ~~ R3*El
Ab ~~ R4*Ab
Res ~~ R5*Res
# Factor variances fixed to 1
Adapt ~~ 1*Adapt
Innov ~~ 1*Innov
# Factor correlation
Adapt ~~ fcorr*Innov
# Reliability coefficients
SREL1 := ((L1+L2+L5)^2)/((L1+L2+L5)^2 +
R1+R2+R5)
SREL2 := ((L3+L4)^2)/((L3+L4)^2 +
R3+R4)
# Composite reliability by Fu et al. (2022)
SRELCR := ((L1+L2+L5)^2+(L3+L4)^2+2*fcorr*(L1+L2+L5)*(L3+L4))/
((L1+L2+L5)^2+(L3+L4)^2+2*fcorr*(L1+L2+L5)*(L3+L4) +
R1+R2+R3+R4+R5)
# Additional constraints
R1 > 0
R2 > 0
R3 > 0
R4 > 0
R5 > 0
"
## Model 3
## Two-factor model with correlated traits ADAPT and INNOV and a cross-loading of RES
TwoFactorModelC <- " # Measurement Model
Innov =~ L1*Fl + L2*Or + CL*Res
Adapt =~ L3*El + L4*Ab + L5*Res
# Residual variances
Fl ~~ R1*Fl
Or ~~ R2*Or
El ~~ R3*El
Ab ~~ R4*Ab
Res ~~ R5*Res
# Factor variances fixed to 1
Adapt ~~ 1*Adapt
Innov ~~ 1*Innov
# Factor correlation
Adapt ~~ fcorr*Innov
# Reliability coefficients
SREL1C := ((L1+L2+CL)^2)/((L1+L2+CL)^2 +
R1+R2+R5)
SREL2C := ((L3+L4+L5)^2)/((L3+L4+L5)^2 +
R3+R4+R5)
# Composite reliability by Fu et al. (2022)
SRELCRC := ((L1+L2+CL)^2+(L3+L4+L5)^2 +
2*fcorr*(L1+L2+CL)*(L3+L4+L5))/
((L1+L2+CL)^2+(L3+L4+L5)^2 +
2*fcorr*(L1+L2+CL)*(L3+L4+L5) +
R1+R2+R3+R4+R5)
# Additional constraints
R1 > 0
R2 > 0
R3 > 0
R4 > 0
R5 > 0
"2.2 Data checking and subsetting
## Check for positive definiteness
is.pd(data)## Acar Acaretal Auth Balcia Balcib Barkul
## TRUE TRUE TRUE TRUE TRUE TRUE
## Barkul2 Berman Boynton Callans Carter_a Carter_b
## NA TRUE TRUE TRUE TRUE TRUE
## Chi Cho_F Cho_M Cockcroft Conway Crawford
## TRUE TRUE TRUE TRUE TRUE TRUE
## Digranes Dufner Fishkin Forsyth Gao Garcia
## TRUE TRUE TRUE TRUE TRUE TRUE
## Gollmar Hamlen Hokanson Houtz Humble Ibrahim
## TRUE TRUE TRUE TRUE TRUE TRUE
## Kiehn Kim_1 Kim_2a Kim_2b Kim_2c Kim_3a
## TRUE TRUE TRUE TRUE TRUE TRUE
## Kim_3b Lew Liu Miranda Morrison_a Morrison_b
## TRUE TRUE TRUE TRUE TRUE TRUE
## Morrison_c Nguyen Rose Rose_b Roskos_Y Roskos_O
## TRUE TRUE TRUE TRUE TRUE TRUE
## Rubenstein_a Rubenstein_b Samuels Shore Stephens Storer_a
## TRUE TRUE TRUE TRUE TRUE TRUE
## Storer_b Tannehill Tisone Trigani Voss Wan
## TRUE TRUE TRUE TRUE TRUE TRUE
## Warne Watson Yoon Zbarskaya Zhang
## TRUE TRUE TRUE TRUE TRUE## Simplify the data structure
TTCT <- data
TTCT.n <- n
# Overview of missing data
pattern.na(TTCT, show.na = TRUE)## Fl Or El Ab Res
## Fl 0 0 7 6 8
## Or 0 0 7 6 8
## El 7 7 7 7 10
## Ab 6 6 7 6 9
## Res 8 8 10 9 8# Overview of complete data
pattern.na(TTCT, show.na = FALSE)## Fl Or El Ab Res
## Fl 65 65 58 59 57
## Or 65 65 58 59 57
## El 58 58 58 58 55
## Ab 59 59 58 59 56
## Res 57 57 55 56 57# Which correlation matrices have missing values?
# Search in the pattern matrices for TRUE (i.e., NAs present)
# Create the relevant objects
x <- "TRUE"
y <- c(rep(NA, length(TTCT)))
for (i in 1:length(TTCT)) {
## Check missingness
y[i] <- x %in% as.matrix(is.na(TTCT[[i]]))
}
y## [1] FALSE FALSE FALSE FALSE FALSE FALSE TRUE FALSE TRUE FALSE FALSE FALSE
## [13] FALSE FALSE FALSE FALSE FALSE FALSE FALSE TRUE FALSE FALSE FALSE FALSE
## [25] FALSE TRUE FALSE FALSE FALSE FALSE TRUE FALSE FALSE FALSE FALSE FALSE
## [37] FALSE TRUE FALSE FALSE TRUE TRUE TRUE FALSE FALSE FALSE FALSE FALSE
## [49] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
## [61] FALSE TRUE FALSE FALSE FALSEtable(y)## y
## FALSE TRUE
## 55 10## Identify the correlation matrices with missing values
missexclude <- as.vector(which(y %in% x))
## Exclude the correlation matrices with missing values
TTCT <- TTCT[-missexclude]
TTCT.n <- TTCT.n[-missexclude]
names.studies <- names[-missexclude]
names.studid <- study.names[-missexclude]
mod.adults <- adults[-missexclude]
mod.validity <- validity[-missexclude]
mod.forms <- formsAB[-missexclude]
mod.scores <- scoring[-missexclude]
## Combine everything in an object
TTCT1 <- list(data = TTCT, n = TTCT.n)
TTCT1## $data
## $data$Acar
## Fl Or El Ab Res
## Fl 1.000 0.650 0.188 0.338 0.529
## Or 0.650 1.000 0.274 0.409 0.477
## El 0.188 0.274 1.000 0.363 0.413
## Ab 0.338 0.409 0.363 1.000 0.305
## Res 0.529 0.477 0.413 0.305 1.000
##
## $data$Acaretal
## Fl Or El Ab Res
## Fl 1.000 0.763 0.089 0.294 0.567
## Or 0.763 1.000 0.210 0.367 0.540
## El 0.089 0.210 1.000 0.469 0.277
## Ab 0.294 0.367 0.469 1.000 0.318
## Res 0.567 0.540 0.277 0.318 1.000
##
## $data$Auth
## Fl Or El Ab Res
## Fl 1.00 0.54 0.11 0.08 0.36
## Or 0.54 1.00 0.39 0.18 0.25
## El 0.11 0.39 1.00 0.32 0.15
## Ab 0.08 0.18 0.32 1.00 -0.07
## Res 0.36 0.25 0.15 -0.07 1.00
##
## $data$Balcia
## Fl Or El Ab Res
## Fl 1.000 0.292 0.000 0.176 -0.117
## Or 0.292 1.000 0.221 0.148 0.154
## El 0.000 0.221 1.000 0.304 0.035
## Ab 0.176 0.148 0.304 1.000 0.047
## Res -0.117 0.154 0.035 0.047 1.000
##
## $data$Balcib
## Fl Or El Ab Res
## Fl 1.000 0.430 0.116 0.267 -0.190
## Or 0.430 1.000 0.166 0.225 0.016
## El 0.116 0.166 1.000 0.363 -0.128
## Ab 0.267 0.225 0.363 1.000 -0.107
## Res -0.190 0.016 -0.128 -0.107 1.000
##
## $data$Barkul
## Fl Or El Ab Res
## Fl 1.00 0.95 0.47 0.86 0.47
## Or 0.95 1.00 0.56 0.84 0.49
## El 0.47 0.56 1.00 0.56 0.38
## Ab 0.86 0.84 0.56 1.00 0.60
## Res 0.47 0.49 0.38 0.60 1.00
##
## $data$Berman
## Fl Or El Ab Res
## Fl 1.000 0.39 0.14 -0.38 0.004
## Or 0.390 1.00 -0.23 0.14 -0.290
## El 0.140 -0.23 1.00 0.05 0.800
## Ab -0.380 0.14 0.05 1.00 0.250
## Res 0.004 -0.29 0.80 0.25 1.000
##
## $data$Callans
## Fl Or El Ab Res
## Fl 1.00 0.78 0.09 0.26 0.51
## Or 0.78 1.00 -0.01 0.26 0.43
## El 0.09 -0.01 1.00 0.27 0.42
## Ab 0.26 0.26 0.27 1.00 0.38
## Res 0.51 0.43 0.42 0.38 1.00
##
## $data$Carter_a
## Fl Or El Ab Res
## Fl 1.00 0.86 0.31 0.05 0.81
## Or 0.86 1.00 0.28 0.05 0.72
## El 0.31 0.28 1.00 0.54 0.49
## Ab 0.05 0.05 0.54 1.00 0.21
## Res 0.81 0.72 0.49 0.21 1.00
##
## $data$Carter_b
## Fl Or El Ab Res
## Fl 1.00 0.82 0.62 0.51 0.68
## Or 0.82 1.00 0.39 0.40 0.60
## El 0.62 0.39 1.00 0.28 0.40
## Ab 0.51 0.40 0.28 1.00 0.33
## Res 0.68 0.60 0.40 0.33 1.00
##
## $data$Chi
## Fl Or Ab El Res
## Fl 1.000 0.433 -0.02 0.18 -0.19
## Or 0.433 1.000 0.42 0.37 0.40
## Ab -0.020 0.420 1.00 0.42 0.54
## El 0.180 0.370 0.42 1.00 0.45
## Res -0.190 0.400 0.54 0.45 1.00
##
## $data$Cho_F
## Fl Or Ab El Res
## Fl 1.00 0.61 0.42 0.32 0.69
## Or 0.61 1.00 0.43 0.21 0.33
## Ab 0.42 0.43 1.00 0.58 0.63
## El 0.32 0.21 0.58 1.00 0.54
## Res 0.69 0.33 0.63 0.54 1.00
##
## $data$Cho_M
## Fl Or Ab El Res
## Fl 1.00 0.59 0.26 0.14 0.82
## Or 0.59 1.00 0.02 0.31 0.69
## Ab 0.26 0.02 1.00 0.36 0.25
## El 0.14 0.31 0.36 1.00 0.24
## Res 0.82 0.69 0.25 0.24 1.00
##
## $data$Cockcroft
## Fl Or Ab El Res
## Fl 1.00 0.29 0.42 0.24 0.18
## Or 0.29 1.00 0.12 0.65 0.31
## Ab 0.42 0.12 1.00 0.31 0.31
## El 0.24 0.65 0.31 1.00 0.46
## Res 0.18 0.31 0.31 0.46 1.00
##
## $data$Conway
## Fl Or El Ab Res
## Fl 1.00 0.82 0.45 0.56 0.50
## Or 0.82 1.00 0.47 0.56 0.43
## El 0.45 0.47 1.00 0.68 0.49
## Ab 0.56 0.56 0.68 1.00 0.65
## Res 0.50 0.43 0.49 0.65 1.00
##
## $data$Crawford
## Fl Or El Ab Res
## Fl 1.00 0.94 0.38 0.37 0.67
## Or 0.94 1.00 0.44 0.48 0.72
## El 0.38 0.44 1.00 0.38 0.65
## Ab 0.37 0.48 0.38 1.00 0.40
## Res 0.67 0.72 0.65 0.40 1.00
##
## $data$Digranes
## Fl Or El Ab Res
## Fl 1.00 0.76 0.17 0.80 0.81
## Or 0.76 1.00 0.45 0.71 0.79
## El 0.17 0.45 1.00 0.09 0.37
## Ab 0.80 0.71 0.09 1.00 0.74
## Res 0.81 0.79 0.37 0.74 1.00
##
## $data$Fishkin
## Fl Or El Ab Res
## Fl 1.00 0.60 0.00 0.13 0.19
## Or 0.60 1.00 -0.03 0.11 0.26
## El 0.00 -0.03 1.00 0.11 0.01
## Ab 0.13 0.11 0.11 1.00 0.09
## Res 0.19 0.26 0.01 0.09 1.00
##
## $data$Forsyth
## Fl Or El Ab Res
## Fl 1.00 0.91 0.56 0.35 0.82
## Or 0.91 1.00 0.49 0.40 0.77
## El 0.56 0.49 1.00 0.50 0.62
## Ab 0.35 0.40 0.50 1.00 0.33
## Res 0.82 0.77 0.62 0.33 1.00
##
## $data$Gao
## Fl Or El Ab Res
## Fl 1.00 0.31 0.28 0.39 0.28
## Or 0.31 1.00 0.34 0.41 0.17
## El 0.28 0.34 1.00 0.25 0.26
## Ab 0.39 0.41 0.25 1.00 0.31
## Res 0.28 0.17 0.26 0.31 1.00
##
## $data$Garcia
## Fl Or El Ab Res
## Fl 1.000 0.626 0.392 0.435 0.416
## Or 0.626 1.000 0.450 0.387 0.525
## El 0.392 0.450 1.000 0.495 0.320
## Ab 0.435 0.387 0.495 1.000 0.567
## Res 0.416 0.525 0.320 0.567 1.000
##
## $data$Gollmar
## Fl Or El Ab Res
## Fl 1.000 0.672 -0.260 -0.130 0.188
## Or 0.672 1.000 0.019 0.241 0.217
## El -0.260 0.019 1.000 0.377 0.220
## Ab -0.130 0.241 0.377 1.000 0.296
## Res 0.188 0.217 0.220 0.296 1.000
##
## $data$Hokanson
## Fl Or El Ab Res
## Fl 1.00 0.79 0.14 0.32 0.48
## Or 0.79 1.00 0.28 0.44 0.45
## El 0.14 0.28 1.00 0.48 0.36
## Ab 0.32 0.44 0.48 1.00 0.36
## Res 0.48 0.45 0.36 0.36 1.00
##
## $data$Houtz
## Fl Or Ab El Res
## Fl 1.00 0.59 0.27 0.49 0.50
## Or 0.59 1.00 0.16 0.27 0.34
## Ab 0.27 0.16 1.00 0.22 0.46
## El 0.49 0.27 0.22 1.00 0.37
## Res 0.50 0.34 0.46 0.37 1.00
##
## $data$Humble
## Fl Or El Ab Res
## Fl 1.00 0.83 0.22 0.44 0.34
## Or 0.83 1.00 0.28 0.38 0.25
## El 0.22 0.28 1.00 0.06 0.07
## Ab 0.44 0.38 0.06 1.00 0.37
## Res 0.34 0.25 0.07 0.37 1.00
##
## $data$Ibrahim
## Fl Or Ab El Res
## Fl 1.000 0.611 0.524 0.230 0.460
## Or 0.611 1.000 0.120 0.246 0.485
## Ab 0.524 0.120 1.000 0.267 0.155
## El 0.230 0.246 0.267 1.000 0.484
## Res 0.460 0.485 0.155 0.484 1.000
##
## $data$Kim_1
## Fl Or Ab El Res
## Fl 1.000 0.844 0.351 0.196 0.666
## Or 0.844 1.000 0.332 0.209 0.563
## Ab 0.351 0.332 1.000 0.428 0.212
## El 0.196 0.209 0.428 1.000 0.443
## Res 0.666 0.563 0.212 0.443 1.000
##
## $data$Kim_2a
## Fl Or Ab El Res
## Fl 1.00 0.84 0.45 0.39 0.65
## Or 0.84 1.00 0.49 0.38 0.65
## Ab 0.45 0.49 1.00 0.49 0.34
## El 0.39 0.38 0.49 1.00 0.51
## Res 0.65 0.65 0.34 0.51 1.00
##
## $data$Kim_2b
## Fl Or Ab El Res
## Fl 1.00 0.79 0.08 0.14 0.47
## Or 0.79 1.00 0.15 0.15 0.48
## Ab 0.08 0.15 1.00 0.30 0.17
## El 0.14 0.15 0.30 1.00 0.25
## Res 0.47 0.48 0.17 0.25 1.00
##
## $data$Kim_2c
## Fl Or Ab El Res
## Fl 1.00 0.86 0.31 0.22 0.66
## Or 0.86 1.00 0.32 0.25 0.57
## Ab 0.31 0.32 1.00 0.41 0.20
## El 0.22 0.25 0.41 1.00 0.40
## Res 0.66 0.57 0.20 0.40 1.00
##
## $data$Kim_3a
## Fl Or Ab El Res
## Fl 1.00 0.46 -0.28 0.10 -0.28
## Or 0.46 1.00 0.24 0.40 0.12
## Ab -0.28 0.24 1.00 0.51 0.60
## El 0.10 0.40 0.51 1.00 0.66
## Res -0.28 0.12 0.60 0.66 1.00
##
## $data$Kim_3b
## Fl Or Ab El Res
## Fl 1.00 0.40 -0.27 0.08 -0.22
## Or 0.40 1.00 0.23 0.34 0.17
## Ab -0.27 0.23 1.00 0.50 0.64
## El 0.08 0.34 0.50 1.00 0.72
## Res -0.22 0.17 0.64 0.72 1.00
##
## $data$Liu
## Fl Or El Ab Res
## Fl 1.00 0.94 0.32 0.49 0.41
## Or 0.94 1.00 0.32 0.48 0.39
## El 0.32 0.32 1.00 0.66 0.62
## Ab 0.49 0.48 0.66 1.00 0.67
## Res 0.41 0.39 0.62 0.67 1.00
##
## $data$Miranda
## Fl Or El Ab Res
## Fl 1.00 0.49 0.02 -0.25 0.41
## Or 0.49 1.00 0.11 -0.30 0.20
## El 0.02 0.11 1.00 -0.31 0.14
## Ab -0.25 -0.30 -0.31 1.00 -0.29
## Res 0.41 0.20 0.14 -0.29 1.00
##
## $data$Nguyen
## Fl Or El Ab Res
## Fl 1.00 0.75 0.11 0.17 0.38
## Or 0.75 1.00 0.09 0.10 0.34
## El 0.11 0.09 1.00 0.15 0.13
## Ab 0.17 0.10 0.15 1.00 0.03
## Res 0.38 0.34 0.13 0.03 1.00
##
## $data$Rose
## Fl Or El Ab Res
## Fl 1.00 0.81 0.07 0.66 -0.05
## Or 0.81 1.00 0.27 0.62 -0.06
## El 0.07 0.27 1.00 0.57 -0.19
## Ab 0.66 0.62 0.57 1.00 -0.12
## Res -0.05 -0.06 -0.19 -0.12 1.00
##
## $data$Rose_b
## Fl Or El Ab Res
## Fl 1.00 0.67 0.05 0.56 0.34
## Or 0.67 1.00 0.13 0.16 0.12
## El 0.05 0.13 1.00 -0.30 0.43
## Ab 0.56 0.16 -0.30 1.00 0.19
## Res 0.34 0.12 0.43 0.19 1.00
##
## $data$Roskos_Y
## Fl Or Ab El Res
## Fl 1.00 0.26 0.15 0.37 0.53
## Or 0.26 1.00 0.08 0.47 0.31
## Ab 0.15 0.08 1.00 0.20 0.47
## El 0.37 0.47 0.20 1.00 0.37
## Res 0.53 0.31 0.47 0.37 1.00
##
## $data$Roskos_O
## Fl Or Ab El Res
## Fl 1.00 0.16 -0.06 0.06 0.14
## Or 0.16 1.00 -0.02 0.25 0.29
## Ab -0.06 -0.02 1.00 0.04 0.36
## El 0.06 0.25 0.04 1.00 0.47
## Res 0.14 0.29 0.36 0.47 1.00
##
## $data$Rubenstein_a
## Fl Or El Ab Res
## Fl 1.000 0.828 0.331 0.470 0.681
## Or 0.828 1.000 0.321 0.550 0.640
## El 0.331 0.321 1.000 0.450 0.443
## Ab 0.470 0.550 0.450 1.000 0.526
## Res 0.681 0.640 0.443 0.526 1.000
##
## $data$Rubenstein_b
## Fl Or El Ab Res
## Fl 1.000 0.629 0.238 0.402 0.313
## Or 0.629 1.000 0.351 0.509 0.394
## El 0.238 0.351 1.000 0.380 0.385
## Ab 0.402 0.509 0.380 1.000 0.305
## Res 0.313 0.394 0.385 0.305 1.000
##
## $data$Samuels
## Fl Or El Ab Res
## Fl 1.00 0.75 -0.11 0.10 -0.16
## Or 0.75 1.00 0.02 0.24 -0.06
## El -0.11 0.02 1.00 0.48 0.07
## Ab 0.10 0.24 0.48 1.00 0.24
## Res -0.16 -0.06 0.07 0.24 1.00
##
## $data$Shore
## Fl Or El Ab Res
## Fl 1.00 0.26 -0.01 -0.12 -0.29
## Or 0.26 1.00 0.33 0.70 0.12
## El -0.01 0.33 1.00 0.30 0.54
## Ab -0.12 0.70 0.30 1.00 0.34
## Res -0.29 0.12 0.54 0.34 1.00
##
## $data$Stephens
## Fl Or El Ab Res
## Fl 1.00 0.86 0.17 0.33 0.65
## Or 0.86 1.00 0.16 0.30 0.60
## El 0.17 0.16 1.00 0.12 0.30
## Ab 0.33 0.30 0.12 1.00 0.31
## Res 0.65 0.60 0.30 0.31 1.00
##
## $data$Storer_a
## Fl Or El Ab Res
## Fl 1.00 0.81 0.07 0.66 -0.05
## Or 0.81 1.00 0.27 0.62 -0.06
## El 0.07 0.27 1.00 0.57 -0.19
## Ab 0.66 0.62 0.57 1.00 -0.12
## Res -0.05 -0.06 -0.19 -0.12 1.00
##
## $data$Storer_b
## Fl Or El Ab Res
## Fl 1.00 0.73 0.23 0.43 0.57
## Or 0.73 1.00 0.20 0.55 0.48
## El 0.23 0.20 1.00 0.29 0.40
## Ab 0.43 0.55 0.29 1.00 0.25
## Res 0.57 0.48 0.40 0.25 1.00
##
## $data$Tannehill
## Fl Or El Ab Res
## Fl 1.00 0.84 0.35 0.28 0.35
## Or 0.84 1.00 0.41 0.36 0.33
## El 0.35 0.41 1.00 0.38 0.39
## Ab 0.28 0.36 0.38 1.00 0.41
## Res 0.35 0.33 0.39 0.41 1.00
##
## $data$Tisone
## Fl Or El Ab Res
## Fl 1.00 0.68 -0.03 0.15 0.13
## Or 0.68 1.00 -0.12 0.26 0.35
## El -0.03 -0.12 1.00 0.30 0.11
## Ab 0.15 0.26 0.30 1.00 0.53
## Res 0.13 0.35 0.11 0.53 1.00
##
## $data$Trigani
## Fl Or El Ab Res
## Fl 1.00 0.72 0.28 0.17 0.69
## Or 0.72 1.00 0.24 0.26 0.47
## El 0.28 0.24 1.00 0.37 0.52
## Ab 0.17 0.26 0.37 1.00 0.16
## Res 0.69 0.47 0.52 0.16 1.00
##
## $data$Voss
## Fl Or El Ab Res
## Fl 1.00 0.71 0.43 0.38 0.65
## Or 0.71 1.00 0.51 0.44 0.79
## El 0.43 0.51 1.00 0.53 0.59
## Ab 0.38 0.44 0.53 1.00 0.42
## Res 0.65 0.79 0.59 0.42 1.00
##
## $data$Wan
## Fl Or El Ab Res
## Fl 1.00 0.27 0.24 0.17 0.19
## Or 0.27 1.00 0.35 0.19 0.14
## El 0.24 0.35 1.00 0.27 0.12
## Ab 0.17 0.19 0.27 1.00 0.35
## Res 0.19 0.14 0.12 0.35 1.00
##
## $data$Warne
## Fl Or El Ab Res
## Fl 1.00 0.74 0.40 0.35 0.63
## Or 0.74 1.00 0.37 0.37 0.50
## El 0.40 0.37 1.00 0.43 0.50
## Ab 0.35 0.37 0.43 1.00 0.26
## Res 0.63 0.50 0.50 0.26 1.00
##
## $data$Yoon
## Fl Or El Ab Res
## Fl 1.00 0.73 -0.16 0.08 0.09
## Or 0.73 1.00 0.08 0.29 0.28
## El -0.16 0.08 1.00 0.47 0.53
## Ab 0.08 0.29 0.47 1.00 0.66
## Res 0.09 0.28 0.53 0.66 1.00
##
## $data$Zbarskaya
## Fl Or El Ab Res
## Fl 1.000 0.614 0.008 0.168 0.189
## Or 0.614 1.000 0.135 0.327 0.153
## El 0.008 0.135 1.000 0.504 0.225
## Ab 0.168 0.327 0.504 1.000 0.442
## Res 0.189 0.153 0.225 0.442 1.000
##
## $data$Zhang
## Fl Or El Ab Res
## Fl 1.00 0.78 0.09 0.25 0.83
## Or 0.78 1.00 0.15 0.24 0.70
## El 0.09 0.15 1.00 0.33 0.06
## Ab 0.25 0.24 0.33 1.00 0.21
## Res 0.83 0.70 0.06 0.21 1.00
##
##
## $n
## [1] 477 375 30 264 105 599 13 60 24 24 203 24 35 36 25
## [16] 21 17 116 45 319 95 128 1758 42 125 99 500 1000 1000 1000
## [31] 125 137 1047 12 187 12 19 39 31 371 371 51 18 84 43
## [46] 46 199 24 107 120 95 432 163 125 10672.2.1 Model Estimation
## Define a function to run the SEM in laavan and extract the
## model parameters
tra_study_lavaan <- function(id, data, para_names=NULL, model, ...) {
cor_i <- data$data[[id]]
n_i <- data$n[[id]]
fit_i <- sem(model=model, std.lv = T, sample.cov=cor_i, sample.nobs=n_i, ...)
results <- list()
coefs <- standardizedSolution(fit_i)$est.std[1:5]
vcoefs <- vcov(fit_i)[1:5,1:5]
if (is.null(para_names)) para_names <- names(coefs)
names(coefs) <- para_names
colnames(vcoefs) <- rownames(vcoefs) <- para_names
results$coefs <- coefs
results$vcoefs <- vcoefs
results$fit <- fit_i
results
}
## Specify some more parameters
k <- length(TTCT1$data)
para_names <- c("L1", "L2", "L3", "L4", "L5")
var_names <- c("Fl", "Or", "El", "Ab", "Res")2.2.1.1 Model 1: Single-factor model
## Model 1
## Single-factor model
## Model estimation
sfm.fit.all <- lapply(1:k, FUN = tra_study_lavaan,
data = TTCT1,
para_names = para_names,
model = SingleFactorModel,
estimator = "ML",
fixed.x = FALSE)
head(sfm.fit.all, 5)## [[1]]
## [[1]]$coefs
## L1 L2 L3 L4 L5
## 0.7806757 0.7970722 0.3850632 0.4928744 0.6544310
##
## [[1]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0018747607 0.0004414647 0.0002888422 0.0003672497 0.0004744853
## L2 0.0004414647 0.0018586283 0.0002859710 0.0003626627 0.0004635303
## L3 0.0002888422 0.0002859710 0.0023570479 0.0001926207 0.0002535575
## L4 0.0003672497 0.0003626627 0.0001926207 0.0022354054 0.0003235835
## L5 0.0004744853 0.0004635303 0.0002535575 0.0003235835 0.0020128976
##
## [[1]]$fit
## lavaan 0.6.16 ended normally after 14 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 477
##
## Model Test User Model:
##
## Test statistic 83.333
## Degrees of freedom 5
## P-value (Chi-square) 0.000
##
##
## [[2]]
## [[2]]$coefs
## L1 L2 L3 L4 L5
## 0.8539468 0.8834395 0.2320228 0.4090740 0.6433003
##
## [[2]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0020687540 0.0007157960 0.0002623920 0.0004625366 0.0007268287
## L2 0.0007157960 0.0020239678 0.0002524888 0.0004436589 0.0006875786
## L3 0.0002623920 0.0002524888 0.0029423964 0.0001260761 0.0001982261
## L4 0.0004625366 0.0004436589 0.0001260761 0.0027619526 0.0003494688
## L5 0.0007268287 0.0006875786 0.0001982261 0.0003494688 0.0023771537
##
## [[2]]$fit
## lavaan 0.6.16 ended normally after 15 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 375
##
## Model Test User Model:
##
## Test statistic 108.835
## Degrees of freedom 5
## P-value (Chi-square) 0.000
##
##
## [[3]]
## [[3]]$coefs
## L1 L2 L3 L4 L5
## 0.5976890 0.8960524 0.4174755 0.2081323 0.3135450
##
## [[3]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.042572394 -0.011264727 0.010409972 0.004974506 0.007613374
## L2 -0.011264727 0.054008776 -0.005773406 -0.002480583 -0.003957476
## L3 0.010409972 -0.005773406 0.039023005 0.003161825 0.004834038
## L4 0.004974506 -0.002480583 0.003161825 0.038628760 0.002318153
## L5 0.007613374 -0.003957476 0.004834038 0.002318153 0.038679852
##
## [[3]]$fit
## lavaan 0.6.16 ended normally after 15 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 30
##
## Model Test User Model:
##
## Test statistic 6.068
## Degrees of freedom 5
## P-value (Chi-square) 0.300
##
##
## [[4]]
## [[4]]$coefs
## L1 L2 L3 L4 L5
## 0.3384332 0.5238644 0.4325338 0.4546676 0.1122193
##
## [[4]]$vcoefs
## L1 L2 L3 L4 L5
## L1 7.787576e-03 -0.0006302157 -9.201167e-05 -1.448375e-04 1.826983e-05
## L2 -6.302157e-04 0.0095965837 -1.256685e-03 -1.541087e-03 -1.332804e-04
## L3 -9.201167e-05 -0.0012566850 8.347094e-03 -3.886514e-04 -1.621477e-06
## L4 -1.448375e-04 -0.0015410869 -3.886514e-04 8.596114e-03 -1.388429e-05
## L5 1.826983e-05 -0.0001332804 -1.621477e-06 -1.388429e-05 7.516828e-03
##
## [[4]]$fit
## lavaan 0.6.16 ended normally after 23 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 264
##
## Model Test User Model:
##
## Test statistic 36.623
## Degrees of freedom 5
## P-value (Chi-square) 0.000
##
##
## [[5]]
## [[5]]$coefs
## L1 L2 L3 L4 L5
## 0.6505729 0.5767207 0.3335709 0.4680604 -0.1908895
##
## [[5]]$vcoefs
## L1 L2 L3 L4 L5
## L1 1.718667e-02 -0.0026844610 -0.0002037401 -0.0007658679 3.988347e-05
## L2 -2.684461e-03 0.0159257850 0.0005739521 0.0006749661 -3.492890e-04
## L3 -2.037401e-04 0.0005739521 0.0146980847 0.0005715674 -2.534976e-04
## L4 -7.658679e-04 0.0006749661 0.0005715674 0.0147126561 -3.373881e-04
## L5 3.988347e-05 -0.0003492890 -0.0002534976 -0.0003373881 1.491776e-02
##
## [[5]]$fit
## lavaan 0.6.16 ended normally after 17 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 10
## Number of inequality constraints 5
##
## Number of observations 105
##
## Model Test User Model:
##
## Test statistic 14.475
## Degrees of freedom 5
## P-value (Chi-square) 0.013## Extract relevant model parameters
## FIT INDICES
## Chi-squares and p values of the parameter-based models
## Select CFI and SRMR
## SRMR performs well in small samples (see Pavlov et al., 2021, EPM)
sfm_model_fit <- t(sapply(sfm.fit.all, function(x) {
fitMeasures(x$fit, c("chisq", "pvalue", "ntotal",
"cfi", "srmr"))}))
row.names(sfm_model_fit) <- names(TTCT1$data)
options(scipen=50)
round(sfm_model_fit,4)## chisq pvalue ntotal cfi srmr
## Acar 83.3332 0.0000 477 0.8816 0.0710
## Acaretal 108.8346 0.0000 375 0.8421 0.1082
## Auth 6.0684 0.2996 30 0.9263 0.0915
## Balcia 36.6229 0.0000 264 0.5953 0.0766
## Balcib 14.4754 0.0129 105 0.7828 0.0752
## Barkul 241.9006 0.0000 599 0.9157 0.0556
## Berman 19.7296 0.0014 13 0.0000 0.2533
## Callans 20.9919 0.0008 60 0.8249 0.1265
## Carter_a 13.2179 0.0214 24 0.8721 0.1466
## Carter_b 2.1289 0.8310 24 1.0000 0.0348
## Chi 97.7060 0.0000 203 0.6591 0.1254
## Cho_F 13.9922 0.0157 24 0.8011 0.0947
## Cho_M 10.2306 0.0690 35 0.9212 0.0961
## Cockcroft 8.8922 0.1134 36 0.8845 0.0962
## Conway 14.1535 0.0147 25 0.8468 0.1038
## Crawford 9.7633 0.0822 21 0.9322 0.1001
## Digranes 7.0518 0.2168 17 0.9618 0.0756
## Fishkin 2.2949 0.8070 116 1.0000 0.0336
## Forsyth 15.3126 0.0091 45 0.9354 0.0807
## Gao 14.0567 0.0153 319 0.9583 0.0368
## Garcia 22.4599 0.0004 95 0.8848 0.0630
## Gollmar 68.0533 0.0000 128 0.5816 0.1725
## Hokanson 601.4395 0.0000 1758 0.8187 0.1122
## Houtz 6.7467 0.2402 42 0.9596 0.0725
## Humble 13.0713 0.0227 125 0.9593 0.0605
## Ibrahim 45.6004 0.0000 99 0.7167 0.1149
## Kim_1 212.7346 0.0000 500 0.8282 0.1158
## Kim_2a 262.3095 0.0000 1000 0.9004 0.0802
## Kim_2b 134.6176 0.0000 1000 0.9102 0.0849
## Kim_2c 305.3774 0.0000 1000 0.8737 0.1028
## Kim_3a 181.6475 0.0000 125 0.2080 0.3009
## Kim_3b 69.7880 0.0000 137 0.7345 0.1431
## Liu 1025.4772 0.0000 1047 0.7377 0.2090
## Miranda 1.9213 0.8599 12 1.0000 0.0801
## Nguyen 6.3830 0.2707 187 0.9927 0.0418
## Rose 9.2189 0.1006 12 0.7898 0.1172
## Rose_b 11.3207 0.0454 19 0.7149 0.1517
## Roskos_Y 8.9598 0.1107 39 0.8776 0.0875
## Roskos_O 2.6746 0.7500 31 1.0000 0.0691
## Rubenstein_a 84.3037 0.0000 371 0.9157 0.0721
## Rubenstein_b 36.0357 0.0000 371 0.9332 0.0574
## Samuels 21.1041 0.0008 51 0.7149 0.1485
## Shore 12.9287 0.0241 18 0.5875 0.1627
## Stephens 6.2337 0.2841 84 0.9927 0.0498
## Storer_a 33.0343 0.0000 43 0.7132 0.1172
## Storer_b 10.4611 0.0632 46 0.9234 0.0746
## Tannehill 50.4142 0.0000 199 0.8790 0.1066
## Tisone 10.3408 0.0661 24 0.7382 0.1507
## Trigani 44.2975 0.0000 107 0.7992 0.1077
## Voss 20.2329 0.0011 120 0.9486 0.0594
## Wan 7.7501 0.1706 95 0.9231 0.0581
## Warne 89.9866 0.0000 432 0.8950 0.0731
## Yoon 161.6311 0.0000 163 0.4854 0.2409
## Zbarskaya 60.8402 0.0000 125 0.5753 0.1558
## Zhang 131.0945 0.0000 1067 0.9490 0.0807## Write out these indices
write.csv(round(sfm_model_fit,4), "SingleFactorModel-Fit.csv")
write.table(round(sfm_model_fit,4), "SingleFactorModel-Fit.txt", sep='\t')
head(sfm_model_fit, 5)## chisq pvalue ntotal cfi srmr
## Acar 83.333237 0.0000000000000002220446 477 0.8816320 0.07096241
## Acaretal 108.834597 0.0000000000000000000000 375 0.8420727 0.10816058
## Auth 6.068386 0.2996214212440039270646 30 0.9263234 0.09146972
## Balcia 36.622915 0.0000007127725482058977 264 0.5953345 0.07661383
## Balcib 14.475398 0.0128555988944794030715 105 0.7827929 0.075189842.2.1.2 Model 2:
Two-factor model with Res assigned to
Innov
## Two-factor model with two correlated traits and Res assigned to Innov
## Model estimation
tfm2.fit.all <- lapply(1:k, FUN = tra_study_lavaan,
data = TTCT1,
para_names = para_names,
model = TwoFactorModelB,
estimator = "ML",
fixed.x = FALSE)
head(tfm2.fit.all, 5)## [[1]]
## [[1]]$coefs
## L1 L2 L3 L4 L5
## 0.7969849 0.8003590 0.6446502 0.5195315 0.6987064
##
## [[1]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0018861508 0.0004099329 0.0004707938 0.0001950756 0.0002623528
## L2 0.0004099329 0.0018834415 0.0004673686 0.0001959015 0.0002634636
## L3 0.0004707938 0.0004673686 0.0020258499 0.0001577891 0.0002122070
## L4 0.0001950756 0.0001959015 0.0001577891 0.0031343648 -0.0003250442
## L5 0.0002623528 0.0002634636 0.0002122070 -0.0003250442 0.0039772879
##
## [[1]]$fit
## lavaan 0.6.16 ended normally after 18 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 477
##
## Model Test User Model:
##
## Test statistic 57.687
## Degrees of freedom 4
## P-value (Chi-square) 0.000
##
##
## [[2]]
## [[2]]$coefs
## L1 L2 L3 L4 L5
## 0.8684909 0.8753378 0.6396407 0.4880936 0.9608786
##
## [[2]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.00206802892 0.00069584126 0.00071243600 0.00009368257 0.0001844270
## L2 0.00069584126 0.00205814308 0.00070152449 0.00009442116 0.0001858810
## L3 0.00071243600 0.00070152449 0.00238115270 0.00006899694 0.0001358299
## L4 0.00009368257 0.00009442116 0.00006899694 0.00543615561 -0.0053004314
## L5 0.00018442700 0.00018588101 0.00013582993 -0.00530043142 0.0134203654
##
## [[2]]$fit
## lavaan 0.6.16 ended normally after 21 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 375
##
## Model Test User Model:
##
## Test statistic 33.092
## Degrees of freedom 4
## P-value (Chi-square) 0.000
##
##
## [[3]]
## [[3]]$coefs
## L1 L2 L3 L4 L5
## 0.5895150 0.9156545 0.3050138 0.8405801 0.3806894
##
## [[3]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0453756427 -0.016991967 0.0092965831 0.0018551323 0.0008401689
## L2 -0.0169919669 0.062985668 -0.0068274011 0.0028814532 0.0013049783
## L3 0.0092965831 -0.006827401 0.0385664551 0.0009598415 0.0004347016
## L4 0.0018551323 0.002881453 0.0009598415 0.1800957928 -0.0543381624
## L5 0.0008401689 0.001304978 0.0004347016 -0.0543381624 0.0625523360
##
## [[3]]$fit
## lavaan 0.6.16 ended normally after 24 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 30
##
## Model Test User Model:
##
## Test statistic 3.827
## Degrees of freedom 4
## P-value (Chi-square) 0.430
##
##
## [[4]]
## [[4]]$coefs
## L1 L2 L3 L4 L5
## 0.2920001 1.0000000 0.1540000 0.6737536 0.4512037
##
## [[4]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.00361265564 0.00055093560 0.00008484407 0.00003993787 0.00002674585
## L2 0.00055093560 0.00188676565 0.00029056186 0.00013677351 0.00009159538
## L3 0.00008484407 0.00029056186 0.00372878360 0.00002106312 0.00001410569
## L4 0.00003993787 0.00013677351 0.00002106312 0.02310016732 -0.01024367663
## L5 0.00002674585 0.00009159538 0.00001410569 -0.01024367663 0.01244114141
##
## [[4]]$fit
## lavaan 0.6.16 ended normally after 55 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 264
##
## Model Test User Model:
##
## Test statistic 17.457
## Degrees of freedom 4
## P-value (Chi-square) 0.002
##
##
## [[5]]
## [[5]]$coefs
## L1 L2 L3 L4 L5
## 0.8040935 0.5316154 -0.1983872 0.4340951 0.8362217
##
## [[5]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0325363543 -0.0126544938 0.00306217860 0.00029116216 0.0005608822
## L2 -0.0126544938 0.0197394588 -0.00243904990 0.00019249800 0.0003708198
## L3 0.0030621786 -0.0024390499 0.01364481519 -0.00007183599 -0.0001383817
## L4 0.0002911622 0.0001924980 -0.00007183599 0.02442133368 -0.0257014945
## L5 0.0005608822 0.0003708198 -0.00013838175 -0.02570149452 0.0650521958
##
## [[5]]$fit
## lavaan 0.6.16 ended normally after 27 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 105
##
## Model Test User Model:
##
## Test statistic 5.016
## Degrees of freedom 4
## P-value (Chi-square) 0.286## Extract relevant model parameters
## FIT INDICES
## Chi-squares and p values of the parameter-based models
tfm2_model_fit <- t(sapply(tfm2.fit.all, function(x) {
fitMeasures(x$fit, c("chisq", "pvalue", "ntotal",
"cfi", "srmr"))}))
row.names(tfm2_model_fit) <- names(TTCT1$data)
options(scipen=50)
round(tfm2_model_fit,4)## chisq pvalue ntotal cfi srmr
## Acar 57.6867 0.0000 477 0.9189 0.0565
## Acaretal 33.0925 0.0000 375 0.9558 0.0507
## Auth 3.8267 0.4300 30 1.0000 0.0672
## Balcia 17.4574 0.0016 264 0.8278 0.0569
## Balcib 5.0161 0.2856 105 0.9767 0.0468
## Barkul 209.5178 0.0000 599 0.9269 0.0537
## Berman 7.9998 0.0916 13 0.7175 0.1554
## Callans 17.0886 0.0019 60 0.8567 0.1120
## Carter_a 5.1019 0.2770 24 0.9829 0.0709
## Carter_b 2.0598 0.7248 24 1.0000 0.0336
## Chi 114.0734 0.0000 203 0.5952 0.1638
## Cho_F 11.5357 0.0212 24 0.8333 0.0795
## Cho_M 6.5131 0.1640 35 0.9621 0.0579
## Cockcroft 8.8535 0.0649 36 0.8559 0.0942
## Conway 7.1166 0.1299 25 0.9478 0.0920
## Crawford 8.7763 0.0669 21 0.9320 0.0901
## Digranes 4.2876 0.3685 17 0.9946 0.0556
## Fishkin 2.7655 0.5978 116 1.0000 0.0510
## Forsyth 8.6404 0.0707 45 0.9709 0.0401
## Gao 5.3692 0.2515 319 0.9937 0.0243
## Garcia 17.6336 0.0015 95 0.9101 0.0555
## Gollmar 50.4486 0.0000 128 0.6918 0.1480
## Hokanson 282.8336 0.0000 1758 0.9153 0.0723
## Houtz 6.6986 0.1527 42 0.9375 0.0732
## Humble 12.5814 0.0135 125 0.9567 0.0586
## Ibrahim 43.1922 0.0000 99 0.7265 0.1273
## Kim_1 133.7546 0.0000 500 0.8927 0.0786
## Kim_2a 144.5767 0.0000 1000 0.9456 0.0508
## Kim_2b 52.4440 0.0000 1000 0.9664 0.0483
## Kim_2c 168.1729 0.0000 1000 0.9310 0.0628
## Kim_3a 141.7916 0.0000 125 0.3822 0.2546
## Kim_3b 60.1716 0.0000 137 0.7698 0.1375
## Liu 557.0301 0.0000 1047 0.8578 0.1704
## Miranda 0.9114 0.9229 12 1.0000 0.0521
## Nguyen 3.0839 0.5439 187 1.0000 0.0252
## Rose 5.1725 0.2700 12 0.9416 0.0939
## Rose_b 9.5764 0.0482 19 0.7485 0.1636
## Roskos_Y 8.8310 0.0655 39 0.8507 0.0857
## Roskos_O 1.9012 0.7539 31 1.0000 0.0603
## Rubenstein_a 51.4485 0.0000 371 0.9496 0.0513
## Rubenstein_b 25.1730 0.0000 371 0.9544 0.0524
## Samuels 9.0344 0.0602 51 0.9109 0.1071
## Shore 7.1004 0.1307 18 0.8387 0.1371
## Stephens 5.9531 0.2027 84 0.9885 0.0482
## Storer_a 18.5349 0.0010 43 0.8513 0.0939
## Storer_b 9.3526 0.0529 46 0.9249 0.0692
## Tannehill 36.5296 0.0000 199 0.9134 0.0929
## Tisone 9.7572 0.0447 24 0.7178 0.1732
## Trigani 32.3158 0.0000 107 0.8553 0.0814
## Voss 6.6525 0.1554 120 0.9910 0.0213
## Wan 7.7500 0.1012 95 0.8952 0.0581
## Warne 52.9054 0.0000 432 0.9396 0.0496
## Yoon 121.9903 0.0000 163 0.6124 0.2126
## Zbarskaya 27.6169 0.0000 125 0.8204 0.1117
## Zhang 19.0140 0.0008 1067 0.9939 0.0208## Write out these indices
write.csv(round(tfm2_model_fit,4), "TwoFactorModel2-Fit.csv")
write.table(round(tfm2_model_fit,4), "TwoFactorModel2-Fit.txt", sep='\t')
head(tfm2_model_fit, 5)## chisq pvalue ntotal cfi srmr
## Acar 57.686705 0.000000000008878454 477 0.9188750 0.05648029
## Acaretal 33.092492 0.000001143512293966 375 0.9557518 0.05067900
## Auth 3.826686 0.429969118942402750 30 1.0000000 0.06722991
## Balcia 17.457364 0.001574836742134322 264 0.8277916 0.05693066
## Balcib 5.016106 0.285648861730992576 105 0.9767075 0.04683203## Extract the scale reliability coefficients from the primary studies
## Point estimates
tfm2.srl1 <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[14,5]}))
tfm2.srl2 <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[15,5]}))
tfm2.srlcr <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[16,5]}))
## Standard errors
tfm2.srl1.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[14,6]}))
tfm2.srl2.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[15,6]}))
tfm2.srlcr.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[16,6]}))
## Lower 95% CI bound
tfm2.srl1.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[14,9]}))
tfm2.srl2.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[15,9]}))
tfm2.srlcr.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[16,9]}))
## Upper 95% CI bound
tfm2.srl1.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[14,10]}))
tfm2.srl2.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[15,10]}))
tfm2.srlcr.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[16,10]}))
## Factor correlation
tfm2.fcorr <- t(sapply(1:k, function(x) {parameterEstimates(tfm2.fit.all[[x]]$fit)[13,5]}))
## Save these statistics
tfm2.srl.summary <- data.frame(names.studies,
TTCT1$n,
t(tfm2.srl1 ),
t(tfm2.srl1.se),
t(tfm2.srl1.ci.low),
t(tfm2.srl1.ci.upp),
t(tfm2.srl2),
t(tfm2.srl2.se),
t(tfm2.srl2.ci.low),
t(tfm2.srl2.ci.upp),
t(tfm2.srlcr),
t(tfm2.srlcr.se),
t(tfm2.srlcr.ci.low),
t(tfm2.srlcr.ci.upp),
t(tfm2.fcorr))
colnames(tfm2.srl.summary) <- c("Study",
"N",
"Omega Factor 1",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Omega Factor 2",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Composite reliability",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Factor correlation")
tfm2.srl.summary## Study N Omega Factor 1 SE Lower 95% CI Upper 95% CI
## 1 Acar 477 0.79342944 0.016278481 0.76152421 0.8253347
## 2 Acaretal 375 0.84145745 0.014069535 0.81388167 0.8690332
## 3 Auth 30 0.65564386 0.104524431 0.45077974 0.8605080
## 4 Balcia 264 0.52510048 0.039484520 0.44771224 0.6024887
## 5 Balcib 105 0.38902715 0.099196619 0.19460535 0.5834490
## 6 Barkul 599 0.87684294 0.009058061 0.85908947 0.8945964
## 7 Berman 13 0.21016567 0.195514311 -0.17303533 0.5933667
## 8 Callans 60 0.82486789 0.038333729 0.74973516 0.9000006
## 9 Carter_a 24 0.92550812 0.026368277 0.87382725 0.9771890
## 10 Carter_b 24 0.87840114 0.041440901 0.79717846 0.9596238
## 11 Chi 203 0.67031672 0.034570247 0.60256028 0.7380732
## 12 Cho_F 24 0.78113939 0.075683560 0.63280234 0.9294764
## 13 Cho_M 35 0.88259986 0.033779342 0.81639357 0.9488062
## 14 Cockcroft 36 0.51583279 0.135877965 0.24951687 0.7821487
## 15 Conway 25 0.83865184 0.055255865 0.73035233 0.9469513
## 16 Crawford 21 0.92207022 0.030206796 0.86286599 0.9812745
## 17 Digranes 17 0.91767077 0.034578505 0.84989815 0.9854434
## 18 Fishkin 116 0.67543999 0.052009003 0.57350422 0.7773758
## 19 Forsyth 45 0.94055748 0.015555705 0.91006886 0.9710461
## 20 Gao 319 0.50809497 0.047486027 0.41502407 0.6011659
## 21 Garcia 95 0.77668460 0.039474752 0.69931551 0.8540537
## 22 Gollmar 128 0.69572184 0.041101217 0.61516494 0.7762787
## 23 Hokanson 1758 0.82110347 0.007243849 0.80690579 0.8353012
## 24 Houtz 42 0.75289536 0.063984429 0.62748818 0.8783025
## 25 Humble 125 0.79567253 0.030606303 0.73568528 0.8556598
## 26 Ibrahim 99 0.75192054 0.039023666 0.67543556 0.8284055
## 27 Kim_1 500 0.88283380 0.008954717 0.86528287 0.9003847
## 28 Kim_2a 1000 0.88851515 0.006118823 0.87652248 0.9005078
## 29 Kim_2b 1000 0.82563158 0.009412889 0.80718266 0.8440805
## 30 Kim_2c 1000 0.88656108 0.006190075 0.87442875 0.8986934
## 31 Kim_3a 125 0.58458079 0.052557496 0.48156999 0.6875916
## 32 Kim_3b 137 0.23957472 0.093681148 0.05596305 0.4231864
## 33 Liu 1047 0.85566281 0.007922086 0.84013581 0.8711898
## 34 Miranda 12 0.66708273 0.162981604 0.34764466 0.9865208
## 35 Nguyen 187 0.77821059 0.026939478 0.72541019 0.8310110
## 36 Rose 12 0.68572562 0.143683681 0.40411078 0.9673405
## 37 Rose_b 19 0.73783593 0.093877348 0.55383971 0.9218322
## 38 Roskos_Y 39 0.66456878 0.089855121 0.48845598 0.8406816
## 39 Roskos_O 31 0.51431965 0.194303603 0.13349158 0.8951477
## 40 Rubenstein_a 371 0.88973463 0.009920206 0.87029138 0.9091779
## 41 Rubenstein_b 371 0.73558217 0.022951589 0.69059788 0.7805665
## 42 Samuels 51 0.64164977 0.074671111 0.49529708 0.7880025
## 43 Shore 18 0.04879865 0.194405932 -0.33222998 0.4298273
## 44 Stephens 84 0.88711473 0.021369179 0.84523191 0.9289976
## 45 Storer_a 43 0.68572584 0.075903808 0.53695711 0.8344946
## 46 Storer_b 46 0.82236445 0.044859980 0.73444050 0.9102884
## 47 Tannehill 199 0.80535849 0.023291147 0.75970868 0.8510083
## 48 Tisone 24 0.74438242 0.081760886 0.58413403 0.9046308
## 49 Trigani 107 0.85280675 0.023974059 0.80581846 0.8997950
## 50 Voss 120 0.88620094 0.017965305 0.85098959 0.9214123
## 51 Wan 95 0.43441825 0.100123344 0.23818011 0.6306564
## 52 Warne 432 0.84503235 0.012736174 0.82006991 0.8699948
## 53 Yoon 163 0.74419778 0.031567220 0.68232716 0.8060684
## 54 Zbarskaya 125 0.65602250 0.050572601 0.55690203 0.7551430
## 55 Zhang 1067 0.91182228 0.004665094 0.90267886 0.9209657
## Omega Factor 2 SE Lower 95% CI Upper 95% CI Composite reliability
## 1 0.54442539 0.04311695 0.459917724 0.6289331 0.7995981
## 2 0.71460948 0.05953533 0.597922378 0.8312966 0.8473549
## 3 0.56496187 0.25729832 0.060666434 1.0692573 0.7062665
## 4 0.48524883 0.07529410 0.337675104 0.6328226 0.5777127
## 5 0.59196819 0.13406183 0.329211825 0.8547245 0.5673408
## 6 0.76734112 0.01859382 0.730897899 0.8037843 0.9067247
## 7 0.13125388 0.63906542 -1.121291339 1.3837991 0.3792193
## 8 0.57080369 0.37412291 -0.162463739 1.3040711 0.7960050
## 9 0.77000008 0.08239435 0.608510111 0.9314900 0.9062086
## 10 0.44168307 0.22914851 -0.007439755 0.8908059 0.8502797
## 11 0.59393298 0.05735172 0.481525674 0.7063403 0.7389661
## 12 0.74362049 0.10602301 0.535819218 0.9514218 0.8521182
## 13 0.52951075 0.15915172 0.217579111 0.8414424 0.8288963
## 14 0.57158106 0.16899674 0.240353530 0.9028086 0.7019128
## 15 0.81823414 0.07401261 0.673172080 0.9632962 0.8918299
## 16 0.55176587 0.19595122 0.167708530 0.9358232 0.8802762
## 17 0.19431940 0.44525652 -0.678367348 1.0670062 0.8539556
## 18 0.55499998 0.06156126 0.434342125 0.6756578 0.6332173
## 19 0.69824220 0.09756896 0.507010559 0.8894739 0.9148728
## 20 0.40557490 0.06722093 0.273824291 0.5373255 0.6626395
## 21 0.66395267 0.06913231 0.528455821 0.7994495 0.8312741
## 22 0.61601475 0.14217430 0.337358237 0.8946713 0.5904046
## 23 0.68896747 0.01872603 0.652265130 0.7256698 0.8376745
## 24 0.37844715 0.19970361 -0.012964733 0.7698590 0.7544145
## 25 0.12418332 0.17032751 -0.209652468 0.4580191 0.7245921
## 26 0.49703327 0.12591109 0.250252063 0.7438145 0.7632648
## 27 0.64133584 0.04348908 0.556098816 0.7265729 0.8553685
## 28 0.66067493 0.02159081 0.618357717 0.7029921 0.8796064
## 29 0.46688762 0.03601213 0.396305141 0.5374701 0.7621709
## 30 0.59277500 0.02756163 0.538755195 0.6467948 0.8475895
## 31 0.72052879 0.07125025 0.580880865 0.8601767 0.7188807
## 32 0.66982318 0.05641769 0.559246536 0.7803998 0.6533906
## 33 0.82925737 0.01403242 0.801754324 0.8567604 0.8909234
## 34 0.34500000 0.22212131 -0.090349758 0.7803498 0.5295116
## 35 0.26798772 0.11342589 0.045677058 0.4902984 0.6832934
## 36 0.78499954 0.10997994 0.569442830 1.0005563 0.8199744
## 37 0.34999823 0.17644147 0.004179309 0.6958171 0.7224723
## 38 0.33423467 0.21365718 -0.084525713 0.7529950 0.7051059
## 39 0.07834996 0.33674747 -0.581662952 0.7383629 0.5443966
## 40 0.64601509 0.03877220 0.570022975 0.7220072 0.8790599
## 41 0.56722532 0.04629451 0.476489758 0.6579609 0.7846358
## 42 0.74000003 0.06263745 0.617232885 0.8627672 0.6581952
## 43 0.47991626 0.24917184 -0.008451577 0.9682841 0.5015645
## 44 0.23396191 0.18273740 -0.124196812 0.5921206 0.7925324
## 45 0.78499986 0.05809909 0.671127741 0.8988720 0.8199745
## 46 0.49852653 0.16718496 0.170850040 0.8262030 0.8118961
## 47 0.55369507 0.06374239 0.428762276 0.6786279 0.8048860
## 48 0.64999988 0.11519913 0.424213735 0.8757860 0.6900959
## 49 0.57668943 0.10559243 0.369732059 0.7836468 0.8213932
## 50 0.70101044 0.05527658 0.592670348 0.8093505 0.8902267
## 51 0.42685868 0.11811324 0.195360980 0.6583564 0.5986157
## 52 0.60592141 0.03833412 0.530787909 0.6810549 0.8427107
## 53 0.73499994 0.03558982 0.665245164 0.8047547 0.7849945
## 54 0.75199998 0.03847119 0.676597836 0.8274021 0.7536246
## 55 0.62011383 0.08280858 0.457811995 0.7824157 0.8619198
## SE Lower 95% CI Upper 95% CI Factor correlation
## 1 0.015202535 0.7698017 0.8293946 0.67112035
## 2 0.016050724 0.8158961 0.8788138 0.40766720
## 3 0.092312134 0.5253380 0.8871949 0.48204516
## 4 0.038109666 0.5030191 0.6524063 0.32801331
## 5 0.071227956 0.4277366 0.7069451 0.42054214
## 6 0.006367321 0.8942449 0.9192044 0.90362658
## 7 0.292905012 -0.1948640 0.9533026 -1.99997082
## 8 0.084711215 0.6299741 0.9620360 0.33847042
## 9 0.030774514 0.8458916 0.9665255 0.34618970
## 10 0.051826648 0.7487013 0.9518581 1.06267836
## 11 0.028113255 0.6838652 0.7940671 0.60827484
## 12 0.049000598 0.7560788 0.9481577 0.78415097
## 13 0.048676434 0.7334922 0.9243004 0.41362514
## 14 0.081491839 0.5421917 0.8616339 1.05894673
## 15 0.035242566 0.8227557 0.9609040 0.70916554
## 16 0.045274721 0.7915393 0.9690130 0.74551392
## 17 0.068403324 0.7198875 0.9880236 1.86818896
## 18 0.047527859 0.5400644 0.7263702 -0.02750476
## 19 0.022620103 0.8705382 0.9592074 0.65616223
## 20 0.030946022 0.6019864 0.7232925 1.30264630
## 21 0.028347792 0.7757135 0.8868348 0.80388201
## 22 0.084898049 0.4240075 0.7568017 -0.29942308
## 23 0.006533163 0.8248697 0.8504793 0.51334010
## 24 0.062922580 0.6310885 0.8777404 0.92952069
## 25 0.042791799 0.6407217 0.8084624 1.38626219
## 26 0.039552275 0.6857437 0.8407858 0.67185316
## 27 0.011844659 0.8321534 0.8785836 0.42054700
## 28 0.006389031 0.8670841 0.8921287 0.68298986
## 29 0.012648486 0.7373803 0.7869614 0.29332002
## 30 0.008232262 0.8314545 0.8637244 0.43867231
## 31 0.038151808 0.6441045 0.7936568 0.43386323
## 32 0.047508286 0.5602760 0.7465051 1.51035460
## 33 0.005719269 0.8797138 0.9021329 0.50836078
## 34 0.183120296 0.1706024 0.8884208 0.39013455
## 35 0.038323521 0.6081807 0.7584062 0.36799956
## 36 0.080796026 0.6616171 0.9783317 0.71164958
## 37 0.090955429 0.5442030 0.9007417 -0.56000128
## 38 0.077116973 0.5539595 0.8562524 1.11841425
## 39 0.133724762 0.2823009 0.8064923 2.07692958
## 40 0.010666716 0.8581535 0.8999663 0.70681067
## 41 0.018205624 0.7489534 0.8203182 0.79647720
## 42 0.060411229 0.5397913 0.7765990 -0.10999984
## 43 0.161694096 0.1846499 0.8184791 3.33168108
## 44 0.040502569 0.7131489 0.8719160 0.74456919
## 45 0.042682138 0.7363191 0.9036300 0.71164801
## 46 0.047358438 0.7190753 0.9047170 0.74711475
## 47 0.022718967 0.7603577 0.8494144 0.64817413
## 48 0.082728027 0.5279519 0.8522398 -0.11999999
## 49 0.030549877 0.7615165 0.8812698 0.41817045
## 50 0.016719483 0.8574572 0.9229963 0.74367211
## 51 0.067252572 0.4668031 0.7304283 0.99740453
## 52 0.012553997 0.8181054 0.8673161 0.66580678
## 53 0.024549751 0.7368779 0.8331111 0.29000060
## 54 0.033105707 0.6887387 0.8185106 0.36486522
## 55 0.017322974 0.8279674 0.8958722 0.28115578## Write out the results
write.table(tfm2.srl.summary, "TwoFactorModel2-Reliability.txt", sep='\t')
## Note: The standard errors of the derived parameters are identical
## to the asymptotic sampling errors.2.2.1.3 Model 3:
Two-factor model with a cross-loading of Res
## Model 3
## Two-factor model with a cross-loading of RES
## Model estimation
tfmc.fit.all <- lapply(1:k, FUN = tra_study_lavaan,
data = TTCT1,
para_names = para_names,
model = TwoFactorModelC,
estimator = "ML",
fixed.x = FALSE)
head(tfmc.fit.all, 5)## [[1]]
## [[1]]$coefs
## L1 L2 L3 L4 L5
## 0.8131975 0.7993139 0.4172651 0.5956043 0.6094648
##
## [[1]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0019904044 0.0002967262 0.0002107058 0.00019608301 0.00020064613
## L2 0.0002967262 0.0019938449 0.0002621118 0.00019273530 0.00019722051
## L3 0.0002107058 0.0002621118 0.0051404659 0.00071402718 0.00080502182
## L4 0.0001960830 0.0001927353 0.0007140272 0.00317679530 0.00005871318
## L5 0.0002006461 0.0001972205 0.0008050218 0.00005871318 0.00322786977
##
## [[1]]$fit
## lavaan 0.6.16 ended normally after 20 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 477
##
## Model Test User Model:
##
## Test statistic 44.785
## Degrees of freedom 3
## P-value (Chi-square) 0.000
##
##
## [[2]]
## [[2]]$coefs
## L1 L2 L3 L4 L5
## 0.8731001 0.8738977 0.5700938 0.5508761 0.8513708
##
## [[2]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0021150725 0.0006428700 0.0006173610 0.0001158732 0.0001790802
## L2 0.0006428700 0.0021140797 0.0006148720 0.0001159791 0.0001792439
## L3 0.0006173610 0.0006148720 0.0030131822 -0.0003233503 0.0008866267
## L4 0.0001158732 0.0001159791 -0.0003233503 0.0046558590 -0.0025418223
## L5 0.0001790802 0.0001792439 0.0008866267 -0.0025418223 0.0074277732
##
## [[2]]$fit
## lavaan 0.6.16 ended normally after 18 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 375
##
## Model Test User Model:
##
## Test statistic 27.222
## Degrees of freedom 3
## P-value (Chi-square) 0.000
##
##
## [[3]]
## [[3]]$coefs
## L1 L2 L3 L4 L5
## 0.5841981 0.9243441 0.2905938 0.8505294 0.3762364
##
## [[3]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0472744385 -0.020955050 0.0149970190 0.001811084 0.0008011436
## L2 -0.0209550502 0.069905399 -0.0173292962 0.002865578 0.0012676049
## L3 0.0149970190 -0.017329296 0.0603273242 0.004479300 -0.0008967692
## L4 0.0018110844 0.002865578 0.0044793005 0.187477810 -0.0567234757
## L5 0.0008011436 0.001267605 -0.0008967692 -0.056723476 0.0626024237
##
## [[3]]$fit
## lavaan 0.6.16 ended normally after 27 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 30
##
## Model Test User Model:
##
## Test statistic 3.824
## Degrees of freedom 3
## P-value (Chi-square) 0.281
##
##
## [[4]]
## [[4]]$coefs
## L1 L2 L3 L4 L5
## 0.2920002 1.0000000 0.1488891 0.6714460 0.4527535
##
## [[4]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.00361265511 0.0005509357 0.00008202829 0.00003994436 0.00002693432
## L2 0.00055093569 0.0018867655 0.00028091873 0.00013679570 0.00009224080
## L3 0.00008202829 0.0002809187 0.00463717783 0.00021385343 -0.00007048838
## L4 0.00003994436 0.0001367957 0.00021385343 0.02280211045 -0.01012120683
## L5 0.00002693432 0.0000922408 -0.00007048838 -0.01012120683 0.01242535195
##
## [[4]]$fit
## lavaan 0.6.16 ended normally after 53 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 264
##
## Model Test User Model:
##
## Test statistic 17.429
## Degrees of freedom 3
## P-value (Chi-square) 0.001
##
##
## [[5]]
## [[5]]$coefs
## L1 L2 L3 L4 L5
## 0.8041868 0.5347016 -0.1506717 0.4541420 0.7993090
##
## [[5]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0359874335 -0.0148887075 0.004788094 0.0003165004 0.0005570545
## L2 -0.0148887075 0.0211724338 -0.003191701 0.0002104401 0.0003703837
## L3 0.0047880939 -0.0031917005 0.019901383 0.0017858456 -0.0041118279
## L4 0.0003165004 0.0002104401 0.001785846 0.0230965143 -0.0203239472
## L5 0.0005570545 0.0003703837 -0.004111828 -0.0203239472 0.0517589851
##
## [[5]]$fit
## lavaan 0.6.16 ended normally after 26 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 12
## Number of inequality constraints 5
##
## Number of observations 105
##
## Model Test User Model:
##
## Test statistic 4.643
## Degrees of freedom 3
## P-value (Chi-square) 0.200## Extract relevant model parameters
## FIT INDICES
## Chi-squares and p values of the parameter-based models
tfmc_model_fit <- t(sapply(tfmc.fit.all, function(x) {
fitMeasures(x$fit, c("chisq", "pvalue", "ntotal",
"cfi", "srmr"))}))
row.names(tfmc_model_fit) <- names(TTCT1$data)
options(scipen=50)
round(tfmc_model_fit,4)## chisq pvalue ntotal cfi srmr
## Acar 44.7849 0.0000 477 0.9369 0.0470
## Acaretal 27.2219 0.0000 375 0.9632 0.0372
## Auth 3.8239 0.2811 30 0.9432 0.0678
## Balcia 17.4290 0.0006 264 0.8154 0.0568
## Balcib 4.6425 0.1999 105 0.9623 0.0417
## Barkul 105.9531 0.0000 599 0.9634 0.0207
## Berman 6.7223 0.0813 13 0.7371 0.1304
## Callans 4.3791 0.2233 60 0.9849 0.0658
## Carter_a 0.6436 0.8864 24 1.0000 0.0421
## Carter_b 2.0061 0.5711 24 1.0000 0.0328
## Chi 55.4909 0.0000 203 0.8070 0.1078
## Cho_F 4.5596 0.2070 24 0.9655 0.0536
## Cho_M 6.4872 0.0902 35 0.9475 0.0580
## Cockcroft 8.2605 0.0409 36 0.8439 0.0962
## Conway 0.7394 0.8639 25 1.0000 0.0151
## Crawford 3.8836 0.2743 21 0.9874 0.0518
## Digranes 4.1299 0.2478 17 0.9790 0.0554
## Fishkin 0.6283 0.8899 116 1.0000 0.0151
## Forsyth 4.2433 0.2364 45 0.9922 0.0310
## Gao 5.0991 0.1647 319 0.9903 0.0240
## Garcia 11.5125 0.0093 95 0.9438 0.0469
## Gollmar 33.1378 0.0000 128 0.8000 0.1100
## Hokanson 196.2533 0.0000 1758 0.9413 0.0444
## Houtz 2.9690 0.3964 42 1.0000 0.0447
## Humble 6.2968 0.0980 125 0.9834 0.0406
## Ibrahim 40.7638 0.0000 99 0.7365 0.1105
## Kim_1 81.5590 0.0000 500 0.9350 0.0973
## Kim_2a 117.9371 0.0000 1000 0.9555 0.0404
## Kim_2b 12.0904 0.0071 1000 0.9937 0.0131
## Kim_2c 107.9193 0.0000 1000 0.9559 0.0864
## Kim_3a 43.2068 0.0000 125 0.8197 0.1415
## Kim_3b 36.3656 0.0000 137 0.8633 0.1240
## Liu 36.6686 0.0000 1047 0.9913 0.0300
## Miranda 0.6569 0.8833 12 1.0000 0.0430
## Nguyen 3.0489 0.3841 187 0.9997 0.0239
## Rose 5.0376 0.1691 12 0.8985 0.0878
## Rose_b 11.2871 0.0103 19 0.6262 0.1509
## Roskos_Y 8.8309 0.0316 39 0.8198 0.0855
## Roskos_O 2.1785 0.5362 31 1.0000 0.0618
## Rubenstein_a 25.8150 0.0000 371 0.9757 0.0267
## Rubenstein_b 15.1161 0.0017 371 0.9739 0.0354
## Samuels 4.1263 0.2481 51 0.9801 0.0680
## Shore 7.5574 0.0561 18 0.7629 0.1392
## Stephens 1.8276 0.6090 84 1.0000 0.0280
## Storer_a 18.0516 0.0004 43 0.8460 0.0878
## Storer_b 9.1835 0.0269 46 0.9132 0.0768
## Tannehill 6.4306 0.0924 199 0.9909 0.0214
## Tisone 1.9791 0.5768 24 1.0000 0.0637
## Trigani 7.1323 0.0678 107 0.9789 0.0529
## Voss 2.8902 0.4089 120 1.0000 0.0183
## Wan 7.0116 0.0715 95 0.8878 0.0551
## Warne 28.0935 0.0000 432 0.9690 0.0439
## Yoon 23.1185 0.0000 163 0.9339 0.0907
## Zbarskaya 4.3840 0.2229 125 0.9895 0.0362
## Zhang 17.3737 0.0006 1067 0.9942 0.0187## Write out these indices
write.csv(round(tfmc_model_fit,4), "TwoFactorModel3-Fit.csv")
write.table(round(tfmc_model_fit,4), "TwoFactorModel3-Fit.txt", sep='\t')
head(tfmc_model_fit, 5)## chisq pvalue ntotal cfi srmr
## Acar 44.784875 0.000000001027982 477 0.9368596 0.04699874
## Acaretal 27.221879 0.000005289304654 375 0.9631597 0.03724663
## Auth 3.823867 0.281122185722077 30 0.9431856 0.06779564
## Balcia 17.429025 0.000576728751025 264 0.8153577 0.05677000
## Balcib 4.642518 0.199924901808852 105 0.9623481 0.04169383## Extract the scale reliability coefficients from the primary studies
## Point estimates
tfmc.srl1 <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[15,5]}))
tfmc.srl2 <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[16,5]}))
tfmc.srlcr <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[17,5]}))
## Standard errors
tfmc.srl1.se <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[15,6]}))
tfmc.srl2.se <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[16,6]}))
tfmc.srlcr.se <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[17,6]}))
## Lower 95% CI bound
tfmc.srl1.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[15,9]}))
tfmc.srl2.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[16,9]}))
tfmc.srlcr.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[17,9]}))
## Upper 95% CI bound
tfmc.srl1.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[15,10]}))
tfmc.srl2.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[16,10]}))
tfmc.srlcr.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[17,10]}))
## Factor correlation
tfmc.fcorr <- t(sapply(1:k, function(x) {parameterEstimates(tfmc.fit.all[[x]]$fit)[14,5]}))
## Save these statistics
tfmc.srl.summary <- data.frame(names.studies,
TTCT1$n,
t(tfmc.srl1 ),
t(tfmc.srl1.se),
t(tfmc.srl1.ci.low),
t(tfmc.srl1.ci.upp),
t(tfmc.srl2),
t(tfmc.srl2.se),
t(tfmc.srl2.ci.low),
t(tfmc.srl2.ci.upp),
t(tfmc.srlcr),
t(tfmc.srlcr.se),
t(tfmc.srlcr.ci.low),
t(tfmc.srlcr.ci.upp),
t(tfmc.fcorr))
colnames(tfmc.srl.summary) <- c("Study",
"N",
"Omega Factor 1",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Omega Factor 2",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Composite reliability",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Factor correlation")
tfmc.srl.summary## Study N Omega Factor 1 SE Lower 95% CI Upper 95% CI
## 1 Acar 477 0.768526179 0.019834695 0.7296509 0.8074015
## 2 Acaretal 375 0.835859315 0.014797985 0.8068558 0.8648628
## 3 Auth 30 0.653670574 0.107856677 0.4422754 0.8650658
## 4 Balcia 264 0.523362747 0.040893434 0.4432131 0.6035124
## 5 Balcib 105 0.410740317 0.106384992 0.2022296 0.6192511
## 6 Barkul 599 0.787772957 0.028512584 0.7318893 0.8436566
## 7 Berman 13 0.683443767 0.159833252 0.3701763 0.9967112
## 8 Callans 60 0.856329746 0.040851555 0.7762622 0.9363973
## 9 Carter_a 24 0.927154283 0.025691873 0.8767991 0.9775094
## 10 Carter_b 24 0.847019015 0.227871788 0.4003985 1.2936395
## 11 Chi 203 0.593002702 0.049954691 0.4950933 0.6909121
## 12 Cho_F 24 0.817798140 0.062998939 0.6943225 0.9412738
## 13 Cho_M 35 0.882090358 0.034132973 0.8151910 0.9489898
## 14 Cockcroft 36 0.343777202 0.472594250 -0.5824905 1.2700449
## 15 Conway 25 0.807463911 0.069678278 0.6708970 0.9440308
## 16 Crawford 21 0.933745960 0.031927839 0.8711685 0.9963234
## 17 Digranes 17 0.913968974 0.038167446 0.8391622 0.9887758
## 18 Fishkin 116 0.669786264 0.051302282 0.5692356 0.7703369
## 19 Forsyth 45 0.937871024 0.016425512 0.9056776 0.9700644
## 20 Gao 319 0.544650902 0.086227062 0.3756490 0.7136528
## 21 Garcia 95 0.735378067 0.051965699 0.6335272 0.8372290
## 22 Gollmar 128 0.766058096 0.038435615 0.6907257 0.8413905
## 23 Hokanson 1758 0.804897564 0.008142107 0.7889393 0.8208558
## 24 Houtz 42 0.458329744 1.281344374 -2.0530591 2.9697186
## 25 Humble 125 0.001127804 0.106258653 -0.2071353 0.2093909
## 26 Ibrahim 99 0.758951858 0.043049709 0.6745760 0.8433277
## 27 Kim_1 500 0.889613868 0.008285288 0.8733750 0.9058527
## 28 Kim_2a 1000 0.878281651 0.006987244 0.8645869 0.8919764
## 29 Kim_2b 1000 0.824550915 0.009605933 0.8057236 0.8433782
## 30 Kim_2c 1000 0.888960256 0.005933625 0.8773306 0.9005899
## 31 Kim_3a 125 0.564555188 0.065229490 0.4367077 0.6924026
## 32 Kim_3b 137 0.596009927 0.061019415 0.4764141 0.7156058
## 33 Liu 1047 0.881255175 0.007114774 0.8673105 0.8951999
## 34 Miranda 12 0.664389696 0.166030810 0.3389753 0.9898041
## 35 Nguyen 187 0.776273251 0.028415615 0.7205797 0.8319668
## 36 Rose 12 0.716436389 0.150983978 0.4205132 1.0123595
## 37 Rose_b 19 0.993548228 0.063572049 0.8689493 1.1181472
## 38 Roskos_Y 39 0.658265502 0.504882906 -0.3312868 1.6478178
## 39 Roskos_O 31 0.314200161 0.418032551 -0.5051286 1.1335289
## 40 Rubenstein_a 371 0.875943499 0.011889541 0.8526404 0.8992466
## 41 Rubenstein_b 371 0.663651357 0.042377095 0.5805938 0.7467089
## 42 Samuels 51 0.659250825 0.072295638 0.5175540 0.8009477
## 43 Shore 18 0.105086770 0.441498475 -0.7602343 0.9704079
## 44 Stephens 84 0.871663728 0.051241046 0.7712331 0.9720943
## 45 Storer_a 43 0.716436134 0.079760447 0.5601085 0.8727637
## 46 Storer_b 46 0.835509219 0.049907087 0.7376931 0.9333253
## 47 Tannehill 199 0.772246618 0.031778095 0.7099627 0.8345305
## 48 Tisone 24 0.750704490 0.080076475 0.5937575 0.9076515
## 49 Trigani 107 0.857182140 0.022762679 0.8125681 0.9017962
## 50 Voss 120 0.877239537 0.020103505 0.8378374 0.9166417
## 51 Wan 95 0.149108396 0.583197331 -0.9939374 1.2921542
## 52 Warne 432 0.832182178 0.014113254 0.8045207 0.8598436
## 53 Yoon 163 0.807233506 0.028921209 0.7505490 0.8639180
## 54 Zbarskaya 125 0.648665154 0.047054882 0.5564393 0.7408910
## 55 Zhang 1067 0.912546014 0.004657329 0.9034178 0.9216742
## Omega Factor 2 SE Lower 95% CI Upper 95% CI Composite reliability
## 1 0.5661430 0.037231389 0.49317081 0.6391152 0.8049032
## 2 0.6079821 0.037758518 0.53397676 0.6819874 0.8424830
## 3 0.4304457 0.228896963 -0.01818406 0.8790755 0.7080218
## 4 0.3589215 0.070339441 0.22105873 0.4967843 0.5792982
## 5 0.3911985 0.120940859 0.15415875 0.6282382 0.5543792
## 6 0.8229706 0.020476067 0.78283822 0.8631029 0.9178338
## 7 0.7640589 0.105668421 0.55695259 0.9711652 0.6825554
## 8 0.6209308 0.082703511 0.45883493 0.7830267 0.8126529
## 9 0.7684078 0.076993231 0.61750383 0.9193117 0.9125341
## 10 0.4775192 0.735797596 -0.96461763 1.9196559 0.8488549
## 11 0.7425099 0.033753708 0.67635385 0.8086659 0.8036901
## 12 0.7974172 0.072746606 0.65483647 0.9399979 0.8789835
## 13 0.5326083 0.165911937 0.20742687 0.8577897 0.8283102
## 14 0.6753116 0.195492669 0.29215303 1.0584702 0.7281700
## 15 0.8246381 0.062864877 0.70142522 0.9478510 0.9091115
## 16 0.7211523 0.107084957 0.51126967 0.9310350 0.9031175
## 17 0.1955976 0.452136611 -0.69057391 1.0817690 0.8537584
## 18 0.4145218 0.066743116 0.28370768 0.5453359 0.6674901
## 19 0.7290754 0.074780379 0.58250858 0.8756423 0.9219511
## 20 0.2629411 0.100301499 0.06635377 0.4595284 0.6621673
## 21 0.7041132 0.056422518 0.59352706 0.8146993 0.8415844
## 22 0.6126398 0.059837349 0.49536079 0.7299189 0.6615778
## 23 0.6345411 0.015417964 0.60432247 0.6647598 0.8430808
## 24 0.7879058 0.439812817 -0.07411152 1.6499230 0.8128315
## 25 0.8493042 0.072976087 0.70627367 0.9923347 0.8239503
## 26 0.5848694 0.079054069 0.42992630 0.7398126 0.7830406
## 27 0.7072821 0.020200626 0.66768961 0.7468746 0.8731967
## 28 0.6546655 0.020281307 0.61491491 0.6944162 0.8832819
## 29 0.4826084 0.029316947 0.42514820 0.5400685 0.7778928
## 30 0.6794374 0.015380890 0.64929141 0.7095834 0.8706153
## 31 0.8472008 0.023755036 0.80064182 0.8937598 0.8387767
## 32 0.8589660 0.020294884 0.81918874 0.8987432 0.8404802
## 33 0.8470108 0.008347057 0.83065084 0.8633707 0.9254371
## 34 0.1303795 0.938401194 -1.70885300 1.9696121 0.5333744
## 35 0.2079270 0.102716523 0.00660629 0.4092477 0.6841580
## 36 0.5457923 0.212095395 0.13009299 0.9614917 0.8180478
## 37 0.9882795 0.088600698 0.81462535 1.1619337 0.7823249
## 38 0.3066629 0.903013544 -1.46321117 2.0765369 0.7052351
## 39 0.6699684 0.112883969 0.44871988 0.8912169 0.6355349
## 40 0.6740912 0.031196083 0.61294802 0.7352344 0.8837767
## 41 0.6008464 0.047419291 0.50790633 0.6937865 0.7939180
## 42 0.6433924 0.073962362 0.49842884 0.7883560 0.7680445
## 43 0.9003439 0.052944286 0.79657497 1.0041128 0.8075012
## 44 0.5430463 0.396553821 -0.23418488 1.3202775 0.8198361
## 45 0.5457923 0.112043736 0.32619063 0.7653940 0.8180476
## 46 0.3700022 0.227095886 -0.07509756 0.8151020 0.8158777
## 47 0.6777027 0.044607538 0.59027352 0.7651319 0.8392724
## 48 0.6648445 0.102731710 0.46349403 0.8661949 0.8010826
## 49 0.7005870 0.044817183 0.61274692 0.7884270 0.8574797
## 50 0.7123495 0.049526931 0.61527846 0.8094205 0.8922357
## 51 0.6114699 0.352081875 -0.07859790 1.3015377 0.6171624
## 52 0.6426562 0.030686589 0.58251154 0.7028008 0.8524045
## 53 0.7939201 0.027715595 0.73959848 0.8482416 0.8556028
## 54 0.7088408 0.040217202 0.63001652 0.7876651 0.7958534
## 55 0.5389738 0.068302045 0.40510422 0.6728433 0.8581444
## SE Lower 95% CI Upper 95% CI Factor correlation
## 1 0.014734698 0.77602375 0.8337827 0.6221182
## 2 0.013847621 0.81534219 0.8696239 0.4256405
## 3 0.093818456 0.52414099 0.8919026 0.4756440
## 4 0.039274769 0.50232103 0.6562753 0.3286031
## 5 0.071671866 0.41390493 0.6948535 0.4286476
## 6 0.005606417 0.90684544 0.9288222 0.8964628
## 7 0.116559034 0.45410389 0.9110069 -0.2555810
## 8 0.039229646 0.73576425 0.8895416 0.3046393
## 9 0.028867430 0.85595499 0.9691132 0.3155412
## 10 0.052077286 0.74678534 0.9509245 1.0626498
## 11 0.021633441 0.76128929 0.8460908 0.6051602
## 12 0.039679215 0.80121365 0.9567533 0.4926070
## 13 0.049017303 0.73223802 0.9243823 0.3944023
## 14 0.073562141 0.58399081 0.8723491 0.8396080
## 15 0.029530824 0.85123213 0.9669908 0.6654713
## 16 0.035981513 0.83259500 0.9736399 0.6492101
## 17 0.068628757 0.71924852 0.9882683 1.8872193
## 18 0.044358318 0.58054934 0.7544308 0.1409424
## 19 0.021161247 0.88047580 0.9634264 0.5991126
## 20 0.030943240 0.60151963 0.7228149 1.2698894
## 21 0.026490306 0.78966433 0.8935044 0.6792078
## 22 0.046748235 0.56995292 0.7532026 -0.3256691
## 23 0.006154373 0.83101841 0.8551431 0.5311592
## 24 0.092697734 0.63114733 0.9945158 0.8470350
## 25 0.024863836 0.77521811 0.8726826 0.9188285
## 26 0.037837034 0.70888135 0.8571998 0.4052477
## 27 0.008823746 0.85590243 0.8904909 0.1960032
## 28 0.006193187 0.87114352 0.8954204 0.6547173
## 29 0.011914261 0.75454131 0.8012444 0.2602787
## 30 0.006371707 0.85812695 0.8831036 0.2200003
## 31 0.022318890 0.79503245 0.8825209 0.4653317
## 32 0.020846923 0.79962097 0.8813394 0.4123712
## 33 0.003867541 0.91785681 0.9330173 0.5317694
## 34 0.311078740 -0.07632876 1.1430775 0.3290446
## 35 0.038487946 0.60872302 0.7595930 0.3713846
## 36 0.082036171 0.65725983 0.9788357 0.7106701
## 37 0.075736496 0.63388412 0.9307657 0.9965072
## 38 0.077288864 0.55375168 0.8567185 1.1226288
## 39 0.107921399 0.42401286 0.8470570 0.5035671
## 40 0.010125234 0.86393161 0.9036218 0.6921394
## 41 0.017410296 0.75979442 0.8280415 0.7771981
## 42 0.047824176 0.67431084 0.8617782 0.2400000
## 43 0.068962607 0.67233699 0.9426654 0.8649308
## 44 0.043676233 0.73423222 0.9054399 0.7635129
## 45 0.043337322 0.73310804 0.9029872 0.7106699
## 46 0.047857357 0.72207900 0.9096764 0.7013148
## 47 0.018905308 0.80221867 0.8763261 0.6336887
## 48 0.060289851 0.68291666 0.9192485 0.2600004
## 49 0.021163632 0.81599972 0.8989596 0.2800012
## 50 0.016376975 0.86013739 0.9243339 0.6807858
## 51 0.065714285 0.48836479 0.7459601 0.8782405
## 52 0.011747424 0.82937996 0.8754290 0.5602003
## 53 0.018001022 0.82032143 0.8908841 0.3064697
## 54 0.027128833 0.74268189 0.8490250 0.3270000
## 55 0.014981635 0.82878090 0.8875078 0.2958872## Write out the results
write.table(tfmc.srl.summary, "TwoFactorModel3-Reliability.txt", sep='\t')
## Note: The standard errors of the derived parameters are identical
## to the asymptotic sampling errors.2.2.1.4 Model 4:
Two-factor model with Res assigned to
Adapt
## Model 4
## Two-factor model with two correlated traits and Res assigned to Adapt
## Model estimation
tfm.fit.all <- lapply(1:k, FUN = tra_study_lavaan,
data = TTCT1,
para_names = para_names,
model = TwoFactorModel,
estimator = "ML",
fixed.x = FALSE)
head(tfm.fit.all, 5)## [[1]]
## [[1]]$coefs
## L1 L2 L3 L4 L5
## 0.8002171 0.8122797 0.4981151 0.5156007 0.7470457
##
## [[1]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0020041928 0.0002860722 0.0002688384 0.0002782756 0.0004031892
## L2 0.0002860722 0.0020015283 0.0002728912 0.0002824706 0.0004092673
## L3 0.0002688384 0.0002728912 0.0024895394 0.0002488667 0.0002489445
## L4 0.0002782756 0.0002824706 0.0002488667 0.0024713113 0.0002535638
## L5 0.0004031892 0.0004092673 0.0002489445 0.0002535638 0.0024480658
##
## [[1]]$fit
## lavaan 0.6.16 ended normally after 16 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 477
##
## Model Test User Model:
##
## Test statistic 58.274
## Degrees of freedom 4
## P-value (Chi-square) 0.000
##
##
## [[2]]
## [[2]]$coefs
## L1 L2 L3 L4 L5
## 0.8559405 0.8914171 0.3763517 0.4981037 0.7619132
##
## [[2]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0021812266 0.0005938561 0.0002644743 0.0003500328 0.0005354197
## L2 0.0005938561 0.0021407548 0.0002754362 0.0003645409 0.0005576116
## L3 0.0002644743 0.0002754362 0.0033086306 0.0002395522 0.0002491501
## L4 0.0003500328 0.0003645409 0.0002395522 0.0031568366 0.0003054554
## L5 0.0005354197 0.0005576116 0.0002491501 0.0003054554 0.0033523641
##
## [[2]]$fit
## lavaan 0.6.16 ended normally after 21 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 375
##
## Model Test User Model:
##
## Test statistic 85.235
## Degrees of freedom 4
## P-value (Chi-square) 0.000
##
##
## [[3]]
## [[3]]$coefs
## L1 L2 L3 L4 L5
## 0.5400006 1.0000000 0.7651308 0.3827568 0.2172019
##
## [[3]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0275242359 0.0087000184 0.001834831 0.0009178736 0.0005208637
## L2 0.0087000184 0.0161111333 0.003397832 0.0016997646 0.0009645616
## L3 0.0018348309 0.0033978323 0.098934551 -0.0177614703 -0.0084516627
## L4 0.0009178736 0.0016997646 -0.017761470 0.0517805816 0.0040403316
## L5 0.0005208637 0.0009645616 -0.008451663 0.0040403316 0.0488336913
##
## [[3]]$fit
## lavaan 0.6.16 ended normally after 43 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 30
##
## Model Test User Model:
##
## Test statistic 4.964
## Degrees of freedom 4
## P-value (Chi-square) 0.291
##
##
## [[4]]
## [[4]]$coefs
## L1 L2 L3 L4 L5
## 0.2920004 1.0000000 0.6261702 0.4715271 0.1182645
##
## [[4]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.003612652359 0.00055093578 0.00004356659 0.00003280708 0.000008228402
## L2 0.000550935783 0.00188676440 0.00014920048 0.00011235290 0.000028179429
## L3 0.000043566586 0.00014920048 0.01660434996 -0.00673301463 -0.000752408588
## L4 0.000032807081 0.00011235290 -0.00673301463 0.01114541774 0.000365149631
## L5 0.000008228402 0.00002817943 -0.00075240859 0.00036514963 0.007284998235
##
## [[4]]$fit
## lavaan 0.6.16 ended normally after 72 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 264
##
## Model Test User Model:
##
## Test statistic 22.035
## Degrees of freedom 4
## P-value (Chi-square) 0.000
##
##
## [[5]]
## [[5]]$coefs
## L1 L2 L3 L4 L5
## 0.7067616 0.6084091 0.4883365 0.7340460 -0.1865381
##
## [[5]]$vcoefs
## L1 L2 L3 L4 L5
## L1 0.0270488964 -0.0117067518 0.0004066690 0.0006112869 -0.0001553422
## L2 -0.0117067518 0.0224872239 0.0003500772 0.0005262205 -0.0001337249
## L3 0.0004066690 0.0003500772 0.0201440484 -0.0112781006 -0.0017229643
## L4 0.0006112869 0.0005262205 -0.0112781006 0.0328240430 0.0026200213
## L5 -0.0001553422 -0.0001337249 -0.0017229643 0.0026200213 0.0151928792
##
## [[5]]$fit
## lavaan 0.6.16 ended normally after 22 iterations
##
## Estimator ML
## Optimization method NLMINB
## Number of model parameters 11
## Number of inequality constraints 5
##
## Number of observations 105
##
## Model Test User Model:
##
## Test statistic 5.338
## Degrees of freedom 4
## P-value (Chi-square) 0.254## Extract relevant model parameters
## FIT INDICES
## Chi-squares and p values of the parameter-based models
tfm_model_fit <- t(sapply(tfm.fit.all, function(x) {
fitMeasures(x$fit, c("chisq", "pvalue", "ntotal",
"cfi", "srmr"))}))
row.names(tfm_model_fit) <- names(TTCT1$data)
options(scipen=50)
round(tfm_model_fit,4)## chisq pvalue ntotal cfi srmr
## Acar 58.2737 0.0000 477 0.9180 0.0561
## Acaretal 85.2350 0.0000 375 0.8764 0.0887
## Auth 4.9636 0.2911 30 0.9336 0.0983
## Balcia 22.0349 0.0002 264 0.7692 0.0587
## Balcib 5.3376 0.2544 105 0.9693 0.0421
## Barkul 114.0180 0.0000 599 0.9608 0.0288
## Berman 6.7520 0.1496 13 0.8056 0.1312
## Callans 5.7929 0.2152 60 0.9804 0.0720
## Carter_a 8.5375 0.0738 24 0.9294 0.1234
## Carter_b 2.0228 0.7316 24 1.0000 0.0333
## Chi 56.2894 0.0000 203 0.8077 0.1096
## Cho_F 7.8447 0.0974 24 0.9149 0.0717
## Cho_M 10.2295 0.0367 35 0.9062 0.0962
## Cockcroft 8.3021 0.0811 36 0.8723 0.0992
## Conway 0.8890 0.9261 25 1.0000 0.0184
## Crawford 4.1617 0.3846 21 0.9977 0.0513
## Digranes 6.6504 0.1556 17 0.9507 0.0747
## Fishkin 2.0369 0.7290 116 1.0000 0.0310
## Forsyth 10.3382 0.0351 45 0.9603 0.0653
## Gao 12.8084 0.0123 319 0.9595 0.0356
## Garcia 13.5111 0.0090 95 0.9373 0.0476
## Gollmar 43.1792 0.0000 128 0.7400 0.1268
## Hokanson 283.9768 0.0000 1758 0.9149 0.0735
## Houtz 3.4475 0.4859 42 1.0000 0.0473
## Humble 8.2194 0.0839 125 0.9787 0.0538
## Ibrahim 42.7677 0.0000 99 0.7295 0.0975
## Kim_1 167.2994 0.0000 500 0.8649 0.0992
## Kim_2a 192.5836 0.0000 1000 0.9270 0.0605
## Kim_2b 77.5064 0.0000 1000 0.9491 0.0616
## Kim_2c 236.0753 0.0000 1000 0.9024 0.0848
## Kim_3a 64.7562 0.0000 125 0.7276 0.1798
## Kim_3b 51.0455 0.0000 137 0.8072 0.1601
## Liu 36.6795 0.0000 1047 0.9916 0.0299
## Miranda 1.3888 0.8461 12 1.0000 0.0638
## Nguyen 5.9355 0.2040 187 0.9898 0.0384
## Rose 5.0518 0.2820 12 0.9476 0.0882
## Rose_b 13.9349 0.0075 19 0.5519 0.1686
## Roskos_Y 8.9535 0.0623 39 0.8469 0.0889
## Roskos_O 2.1785 0.7030 31 1.0000 0.0618
## Rubenstein_a 40.5525 0.0000 371 0.9611 0.0427
## Rubenstein_b 15.3233 0.0041 371 0.9756 0.0357
## Samuels 4.9267 0.2949 51 0.9836 0.0861
## Shore 17.9608 0.0013 18 0.2736 0.2218
## Stephens 1.8280 0.7674 84 1.0000 0.0280
## Storer_a 18.1023 0.0012 43 0.8557 0.0882
## Storer_b 10.3482 0.0350 46 0.9109 0.0710
## Tannehill 7.1674 0.1273 199 0.9916 0.0201
## Tisone 3.5012 0.4777 24 1.0000 0.0752
## Trigani 21.8630 0.0002 107 0.9087 0.0915
## Voss 16.1637 0.0028 120 0.9589 0.0549
## Wan 7.2675 0.1224 95 0.9086 0.0558
## Warne 56.1459 0.0000 432 0.9356 0.0552
## Yoon 23.1916 0.0001 163 0.9369 0.0918
## Zbarskaya 4.3965 0.3550 125 0.9970 0.0372
## Zhang 129.4983 0.0000 1067 0.9493 0.0806## Write out these indices
write.csv(round(tfm_model_fit,4), "TwoFactorModel4-Fit.csv")
write.table(round(tfm_model_fit,4), "TwoFactorModel4-Fit.txt", sep='\t')
head(tfm_model_fit, 5)## chisq pvalue ntotal cfi srmr
## Acar 58.273738 0.000000000006685208 477 0.9179879 0.05611845
## Acaretal 85.235030 0.000000000000000000 375 0.8764455 0.08868591
## Auth 4.963578 0.291055121191334365 30 0.9335511 0.09825559
## Balcia 22.034942 0.000197235994077283 264 0.7692142 0.05869786
## Balcib 5.337602 0.254377491832347369 105 0.9693378 0.04205625## Extract the scale reliability coefficients from the primary studies
## Point estimates
tfm.srl1 <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[14,5]}))
tfm.srl2 <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[15,5]}))
tfm.srlcr <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[16,5]}))
## Standard errors
tfm.srl1.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[14,6]}))
tfm.srl2.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[15,6]}))
tfm.srlcr.se <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[16,6]}))
## Lower 95% CI bound
tfm.srl1.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[14,9]}))
tfm.srl2.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[15,9]}))
tfm.srlcr.ci.low <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[16,9]}))
## Upper 95% CI bound
tfm.srl1.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[14,10]}))
tfm.srl2.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[15,10]}))
tfm.srlcr.ci.upp <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[16,10]}))
## Factor correlation
tfm.fcorr <- t(sapply(1:k, function(x) {parameterEstimates(tfm.fit.all[[x]]$fit)[13,5]}))
## Save these statistics
tfm.srl.summary <- data.frame(names.studies,
TTCT1$n,
t(tfm.srl1 ),
t(tfm.srl1.se),
t(tfm.srl1.ci.low),
t(tfm.srl1.ci.upp),
t(tfm.srl2),
t(tfm.srl2.se),
t(tfm.srl2.ci.low),
t(tfm.srl2.ci.upp),
t(tfm.srlcr),
t(tfm.srlcr.se),
t(tfm.srlcr.ci.low),
t(tfm.srlcr.ci.upp),
t(tfm.fcorr))
colnames(tfm.srl.summary) <- c("Study",
"N",
"Omega Factor 1",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Omega Factor 2",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Composite reliability",
"SE",
"Lower 95% CI",
"Upper 95% CI",
"Factor correlation")
tfm.srl.summary## Study N Omega Factor 1 SE Lower 95% CI Upper 95% CI
## 1 Acar 477 0.7879230 0.019422465 0.74985562 0.8259903
## 2 Acaretal 375 0.8659270 0.013861703 0.83875860 0.8930955
## 3 Auth 30 0.7700003 0.073695681 0.62555945 0.9144412
## 4 Balcia 264 0.6460004 0.035022508 0.57735750 0.7146432
## 5 Balcib 105 0.6047810 0.079138763 0.44967187 0.7598901
## 6 Barkul 599 0.9744939 0.002080806 0.97041555 0.9785722
## 7 Berman 13 0.6949999 0.141042727 0.41856121 0.9714385
## 8 Callans 60 0.8827988 0.031949253 0.82017946 0.9454182
## 9 Carter_a 24 0.9279418 0.029174066 0.87076173 0.9851220
## 10 Carter_b 24 0.9099999 0.035050009 0.84130312 0.9786966
## 11 Chi 203 0.7165000 0.033685510 0.65047763 0.7825224
## 12 Cho_F 24 0.8050006 0.071425899 0.66500843 0.9449928
## 13 Cho_M 35 0.7476911 0.085012742 0.58106919 0.9143130
## 14 Cockcroft 36 0.5554435 0.188006684 0.18695714 0.9239298
## 15 Conway 25 0.9011436 0.039548341 0.82363032 0.9786570
## 16 Crawford 21 0.9700000 0.012895183 0.94472590 0.9952741
## 17 Digranes 17 0.8640321 0.065895715 0.73487891 0.9931854
## 18 Fishkin 116 0.7595687 0.052606411 0.65646200 0.8626753
## 19 Forsyth 45 0.9538664 0.013684640 0.92704499 0.9806878
## 20 Gao 319 0.4732834 0.058981047 0.35768271 0.5888842
## 21 Garcia 95 0.7710162 0.047072858 0.67875512 0.8632773
## 22 Gollmar 128 0.8359994 0.026507783 0.78404511 0.8879537
## 23 Hokanson 1758 0.8949419 0.005770018 0.88363288 0.9062509
## 24 Houtz 42 0.7823017 0.080033070 0.62543972 0.9391636
## 25 Humble 125 0.9132729 0.016844923 0.88025750 0.9462884
## 26 Ibrahim 99 0.7596899 0.048448655 0.66473228 0.8546475
## 27 Kim_1 500 0.9209904 0.007201180 0.90687639 0.9351045
## 28 Kim_2a 1000 0.9130688 0.005498046 0.90229281 0.9238448
## 29 Kim_2b 1000 0.8830057 0.007417871 0.86846695 0.8975445
## 30 Kim_2c 1000 0.9282173 0.004592202 0.91921677 0.9372179
## 31 Kim_3a 125 0.7300000 0.041266747 0.64911862 0.8108813
## 32 Kim_3b 137 0.7000003 0.042888383 0.61594059 0.7840600
## 33 Liu 1047 0.9692173 0.001912591 0.96546866 0.9729659
## 34 Miranda 12 0.6674744 0.201382393 0.27277217 1.0621766
## 35 Nguyen 187 0.8623294 0.021582766 0.82002798 0.9046309
## 36 Rose 12 0.8959024 0.060156839 0.77799720 1.0138077
## 37 Rose_b 19 0.8350002 0.069179887 0.69941013 0.9705903
## 38 Roskos_Y 39 0.4299804 0.188549800 0.06042962 0.7995313
## 39 Roskos_O 31 0.3140816 0.300832780 -0.27553980 0.9037030
## 40 Rubenstein_a 371 0.9059530 0.009765379 0.88681320 0.9250928
## 41 Rubenstein_b 371 0.7861099 0.023131314 0.74077339 0.8314465
## 42 Samuels 51 0.8750002 0.032745994 0.81081927 0.9391812
## 43 Shore 18 0.6299999 0.138441342 0.35865989 0.9013400
## 44 Stephens 84 0.9262282 0.016156380 0.89456231 0.9578942
## 45 Storer_a 43 0.8959024 0.031779111 0.83361644 0.9581883
## 46 Storer_b 46 0.8439369 0.046020784 0.75373784 0.9341360
## 47 Tannehill 199 0.9161898 0.012441007 0.89180584 0.9405737
## 48 Tisone 24 0.8400000 0.059866512 0.72266381 0.9573362
## 49 Trigani 107 0.8600006 0.025102314 0.81080093 0.9092002
## 50 Voss 120 0.8378753 0.029424061 0.78020522 0.8955454
## 51 Wan 95 0.4282581 0.118440347 0.19611924 0.6603969
## 52 Warne 432 0.8564004 0.013961649 0.82903610 0.8837648
## 53 Yoon 163 0.8650001 0.019668799 0.82644992 0.9035502
## 54 Zbarskaya 125 0.8070001 0.031014799 0.74621216 0.8677879
## 55 Zhang 1067 0.8821890 0.007137126 0.86820044 0.8961775
## Omega Factor 2 SE Lower 95% CI Upper 95% CI Composite reliability
## 1 0.61657373 0.029900377 0.55797007 0.6751774 0.7961029
## 2 0.56882295 0.037500185 0.49532393 0.6423220 0.8033699
## 3 0.45624463 0.174095303 0.11502411 0.7974652 0.6874343
## 4 0.38402741 0.063634609 0.25930587 0.5087490 0.5647680
## 5 0.32904400 0.104533434 0.12416223 0.5339258 0.5565458
## 6 0.77863942 0.014326462 0.75056007 0.8067188 0.9172739
## 7 0.76409053 0.105262005 0.55778079 0.9704003 0.6762776
## 8 0.65864387 0.065378406 0.53050455 0.7867832 0.8232824
## 9 0.62746975 0.110753332 0.41039721 0.8445423 0.8533497
## 10 0.60965463 0.137095073 0.34095323 0.8783560 0.8498451
## 11 0.73092237 0.032558180 0.66710951 0.7947352 0.8028454
## 12 0.78704960 0.071398319 0.64711146 0.9269877 0.8653867
## 13 0.53593998 0.122019197 0.29678675 0.7750932 0.7712868
## 14 0.64197605 0.095900723 0.45401408 0.8299380 0.7316390
## 15 0.83260782 0.056803611 0.72127479 0.9439409 0.9084899
## 16 0.75090462 0.088968879 0.57652882 0.9252804 0.9017108
## 17 0.74279966 0.102234527 0.54242367 0.9431757 0.8840397
## 18 0.13501415 0.133848305 -0.12732370 0.3973520 0.5451633
## 19 0.72290929 0.067145839 0.59130587 0.8545127 0.8942712
## 20 0.52320575 0.045809452 0.43342088 0.6129906 0.6757915
## 21 0.73147909 0.047424678 0.63852843 0.8244298 0.8370563
## 22 0.57767217 0.063700177 0.45282212 0.7025222 0.7373011
## 23 0.66848336 0.013636458 0.64175639 0.6952103 0.8356207
## 24 0.63128965 0.097167364 0.44084511 0.8217342 0.7921521
## 25 0.45739326 0.080424792 0.29976356 0.6150230 0.7720023
## 26 0.59575313 0.068000334 0.46247492 0.7290313 0.7734210
## 27 0.60299308 0.028874546 0.54640001 0.6595862 0.8330267
## 28 0.69120730 0.016601296 0.65866936 0.7237452 0.8681365
## 29 0.45982149 0.030343008 0.40035029 0.5192927 0.7583341
## 30 0.57274705 0.022273656 0.52909149 0.6164026 0.8274552
## 31 0.81414657 0.027904399 0.75945495 0.8688382 0.7397523
## 32 0.83913813 0.022590664 0.79486124 0.8834150 0.7616337
## 33 0.84682710 0.008162806 0.83082829 0.8628259 0.9254409
## 34 0.04610459 0.177849064 -0.30247317 0.3946823 0.4571518
## 35 0.22396085 0.102912245 0.02225656 0.4256651 0.6642839
## 36 0.55869990 0.177142545 0.21150689 0.9058929 0.8163590
## 37 0.59599410 0.132096970 0.33708880 0.8548994 0.7568528
## 38 0.64891319 0.093567233 0.46552478 0.8323016 0.7144250
## 39 0.66999465 0.088558872 0.49642245 0.8435668 0.6355428
## 40 0.71959418 0.024493928 0.67158696 0.7676014 0.8753199
## 41 0.61687449 0.034273202 0.54970025 0.6840487 0.7938111
## 42 0.63343611 0.075028329 0.48638329 0.7804889 0.7764478
## 43 0.68826146 0.128942389 0.43553903 0.9409839 0.5965216
## 44 0.55117737 0.081168520 0.39208999 0.7102647 0.8204643
## 45 0.55870048 0.093579106 0.37528880 0.7421122 0.8163591
## 46 0.53121564 0.117131833 0.30164147 0.7607898 0.7952131
## 47 0.65969490 0.041772638 0.57782203 0.7415678 0.8386434
## 48 0.62199784 0.127650613 0.37180724 0.8721884 0.7710646
## 49 0.62354154 0.052907188 0.51984535 0.7272377 0.8171664
## 50 0.74098227 0.039078493 0.66438983 0.8175747 0.8692816
## 51 0.49301658 0.089663433 0.31727948 0.6687537 0.6127855
## 52 0.65742730 0.027808434 0.60292377 0.7119308 0.8352745
## 53 0.79446255 0.027477785 0.74060709 0.8483180 0.8557819
## 54 0.70948778 0.039708236 0.63166107 0.7873145 0.7953206
## 55 0.38759036 0.031732931 0.32539496 0.4497858 0.7712278
## SE Lower 95% CI Upper 95% CI Factor correlation
## 1 0.015185967 0.76633894 0.8258668 0.8029838
## 2 0.016598931 0.77083661 0.8359032 0.7857477
## 3 0.089172197 0.51266000 0.8622086 0.5250131
## 4 0.039675182 0.48700609 0.6425299 0.3553694
## 5 0.067293016 0.42465389 0.6884377 0.4998183
## 6 0.005562633 0.90637138 0.9281765 0.8720216
## 7 0.114536441 0.45179028 0.9007649 -0.2900000
## 8 0.034783979 0.75510706 0.8914578 0.5302387
## 9 0.047287275 0.76066833 0.9460310 0.8234923
## 10 0.050779068 0.75031999 0.9493703 1.0336723
## 11 0.021735104 0.76024536 0.8454454 0.5716607
## 12 0.043172322 0.78077054 0.9500029 0.7090588
## 13 0.061062399 0.65160666 0.8909669 0.9957951
## 14 0.072698823 0.58915192 0.8741261 0.7845056
## 15 0.029817101 0.85004946 0.9669303 0.6827169
## 16 0.035560859 0.83201280 0.9714088 0.7643346
## 17 0.045706132 0.79445729 0.9736220 1.0398209
## 18 0.070251030 0.40747384 0.6828528 0.6704365
## 19 0.026008062 0.84329635 0.9452461 0.8915926
## 20 0.029604286 0.61776817 0.7338148 1.0997318
## 21 0.027244187 0.78365869 0.8904539 0.7791342
## 22 0.036354007 0.66604852 0.8085536 0.2946672
## 23 0.006418958 0.82303978 0.8482016 0.6169859
## 24 0.051780409 0.69066433 0.8936398 0.7257213
## 25 0.033500218 0.70634311 0.8376615 0.6891124
## 26 0.036703874 0.70148273 0.8453593 0.7498885
## 27 0.012075119 0.80935995 0.8566935 0.7292714
## 28 0.006883575 0.85464495 0.8816281 0.8568125
## 29 0.012683112 0.73347561 0.7831925 0.6521377
## 30 0.008944629 0.80992409 0.8449864 0.7642222
## 31 0.032991000 0.67509111 0.8044135 -0.2595952
## 32 0.028219152 0.70632520 0.8169422 -0.2178581
## 33 0.003867127 0.91786152 0.9330204 0.5310282
## 34 0.225939266 0.01431894 0.8999846 0.6841114
## 35 0.041725233 0.58250396 0.7460639 0.8047517
## 36 0.081647631 0.65633258 0.9763854 0.7107647
## 37 0.079579629 0.60087957 0.9128260 0.3400015
## 38 0.074331273 0.56873837 0.8601116 0.9707069
## 39 0.105000135 0.42974630 0.8413393 0.5037385
## 40 0.010612929 0.85451889 0.8961208 0.8397992
## 41 0.017417539 0.75967335 0.8279488 0.7905793
## 42 0.045035613 0.68817960 0.8647160 0.2399999
## 43 0.136849912 0.32830071 0.8647425 -0.2906585
## 44 0.032228515 0.75729762 0.8836311 0.7700793
## 45 0.043132015 0.73182188 0.9008963 0.7107627
## 46 0.049547402 0.69810200 0.8923242 0.9521721
## 47 0.018983592 0.80143623 0.8758505 0.6041018
## 48 0.072799427 0.62838038 0.9137489 0.3688393
## 49 0.026912805 0.76441824 0.8699145 0.6899985
## 50 0.019220322 0.83161048 0.9069528 0.9260633
## 51 0.064908872 0.48556644 0.7400045 0.8372834
## 52 0.012890119 0.81001034 0.8605387 0.8128623
## 53 0.017943182 0.82061395 0.8909499 0.3176184
## 54 0.026779849 0.74283304 0.8478081 0.3269999
## 55 0.012018395 0.74767218 0.7947834 1.0418061## Write out the results
write.table(tfm.srl.summary, "TwoFactorModel4-Reliability.txt", sep='\t')
## Note: The standard errors of the derived parameters are identical
## to the asymptotic sampling errors.2.3 Stage-2 Analysis: Meta-Analysis of the Reliability Coefficients
2.3.1 Data Preparation
## Create effect size and study identifiers
esid <- as.vector(seq(1:length(TTCT1$data)))
studid <- as.vector(as.numeric(as.factor(names.studid)))
## Create a meta-analytic data set with effect sizes,
## sampling variances, and moderators
srel.meta1 <- data.frame(esid, studid, names.studies,
TTCT1$n,
t(tfm.srl1), t(tfm.srl1.se),
t(tfm.srl2), t(tfm.srl2.se),
t(tfm.srlcr), t(tfm.srlcr.se),
t(tfmc.srl1), t(tfmc.srl1.se),
t(tfmc.srl2), t(tfmc.srl2.se),
t(tfmc.srlcr), t(tfmc.srlcr.se),
t(tfm.fcorr), t(tfmc.fcorr),
mod.adults, mod.forms,
mod.scores, mod.validity)
## Column names
colnames(srel.meta1) <- c("ESID", "STUDID", "Reference",
"N",
"SREL1", "SE.SREL1",
"SREL2", "SE.SREL2",
"SRELCR", "SE.SRELCR",
"SREL1C", "SE.SREL1C",
"SREL2C", "SE.SREL2C",
"SRELCRC", "SE.SRELCRC",
"Fcorr", "FcorrC",
"Adults", "Forms",
"Scores", "Validity")
## Create sampling variances
srel.meta1$SREL1.vg <- srel.meta1$SE.SREL1^2
srel.meta1$SREL2.vg <- srel.meta1$SE.SREL2^2
srel.meta1$SRELCR.vg <- srel.meta1$SE.SRELCR^2
srel.meta1$SREL1C.vg <- srel.meta1$SE.SREL1C^2
srel.meta1$SREL2C.vg <- srel.meta1$SE.SREL2C^2
srel.meta1$SRELCRC.vg <- srel.meta1$SE.SRELCRC^2
## Identify the studies with poor model fit
tfm_model_fit## chisq pvalue ntotal cfi srmr
## Acar 58.2737384 0.0000000000066852079428 477 0.9179879 0.05611845
## Acaretal 85.2350300 0.0000000000000000000000 375 0.8764455 0.08868591
## Auth 4.9635776 0.2910551211913343649940 30 0.9335511 0.09825559
## Balcia 22.0349422 0.0001972359940772827613 264 0.7692142 0.05869786
## Balcib 5.3376015 0.2543774918323473688275 105 0.9693378 0.04205625
## Barkul 114.0179567 0.0000000000000000000000 599 0.9608442 0.02876380
## Berman 6.7520142 0.1495880010679065952317 13 0.8056406 0.13120067
## Callans 5.7928546 0.2151613150094205950680 60 0.9803705 0.07195184
## Carter_a 8.5374948 0.0737588150888134341798 24 0.9293966 0.12344312
## Carter_b 2.0227827 0.7315683246857485055870 24 1.0000000 0.03325786
## Chi 56.2893634 0.0000000000174372738471 203 0.8077218 0.10964189
## Cho_F 7.8447267 0.0974345320442254747562 24 0.9149484 0.07170580
## Cho_M 10.2295462 0.0367335260865379042983 35 0.9061654 0.09617379
## Cockcroft 8.3021446 0.0811166222855811414050 36 0.8723006 0.09924377
## Conway 0.8889755 0.9261371061244323410477 25 1.0000000 0.01843786
## Crawford 4.1616946 0.3845649220312404859001 21 0.9976996 0.05125261
## Digranes 6.6504236 0.1555558127019329717200 17 0.9506538 0.07473993
## Fishkin 2.0368505 0.7289809937310370857588 116 1.0000000 0.03097716
## Forsyth 10.3381919 0.0351003685949619059770 45 0.9602724 0.06526681
## Gao 12.8083576 0.0122511651009710176297 319 0.9594559 0.03563730
## Garcia 13.5111291 0.0090304420932041384873 95 0.9372565 0.04764523
## Gollmar 43.1791907 0.0000000094987471221941 128 0.7400054 0.12680085
## Hokanson 283.9767705 0.0000000000000000000000 1758 0.9149149 0.07349493
## Houtz 3.4474977 0.4859062122440982456339 42 1.0000000 0.04730142
## Humble 8.2194499 0.0838622784140112820239 125 0.9787311 0.05375296
## Ibrahim 42.7677064 0.0000000115623308705182 99 0.7294550 0.09754886
## Kim_1 167.2993976 0.0000000000000000000000 500 0.8649442 0.09923085
## Kim_2a 192.5835800 0.0000000000000000000000 1000 0.9270304 0.06051507
## Kim_2b 77.5063974 0.0000000000000005551115 1000 0.9490579 0.06163331
## Kim_2c 236.0752688 0.0000000000000000000000 1000 0.9024048 0.08477319
## Kim_3a 64.7562446 0.0000000000002896571871 125 0.7275914 0.17983244
## Kim_3b 51.0455158 0.0000000002183865310812 137 0.8072334 0.16014229
## Liu 36.6794921 0.0000002097008866552841 1047 0.9915996 0.02992908
## Miranda 1.3887585 0.8461464699435627956348 12 1.0000000 0.06378383
## Nguyen 5.9355052 0.2040168367815755612327 187 0.9897562 0.03840829
## Rose 5.0518104 0.2820226174393694051901 12 0.9476006 0.08815172
## Rose_b 13.9348757 0.0075058326841783706840 19 0.5519365 0.16864979
## Roskos_Y 8.9535133 0.0622719885770953718307 39 0.8469183 0.08886656
## Roskos_O 2.1784895 0.7029692688101959463509 31 1.0000000 0.06181563
## Rubenstein_a 40.5525279 0.0000000332679327419427 371 0.9611431 0.04273534
## Rubenstein_b 15.3233317 0.0040755124077527238313 371 0.9756381 0.03565326
## Samuels 4.9266837 0.2949032714816962608140 51 0.9835969 0.08610035
## Shore 17.9607613 0.0012560801072869498540 18 0.2735955 0.22179542
## Stephens 1.8279847 0.7673585947114770489108 84 1.0000000 0.02798716
## Storer_a 18.1023205 0.0011785479448310764994 43 0.8557475 0.08815127
## Storer_b 10.3481802 0.0349537821149170646606 46 0.9109395 0.07096107
## Tannehill 7.1674124 0.1273013297674244759250 199 0.9915642 0.02005184
## Tisone 3.5011952 0.4776966312815904558420 24 1.0000000 0.07522753
## Trigani 21.8629651 0.0002134085734527557676 107 0.9087319 0.09152566
## Voss 16.1637294 0.0028071510715872438624 120 0.9589320 0.05493900
## Wan 7.2675425 0.1224074308794176335624 95 0.9086441 0.05579743
## Warne 56.1459071 0.0000000000186879400843 432 0.9355730 0.05516412
## Yoon 23.1916233 0.0001159388910124681260 163 0.9369490 0.09184230
## Zbarskaya 4.3964860 0.3549986083092668831540 125 0.9969846 0.03721615
## Zhang 129.4982963 0.0000000000000000000000 1067 0.9492744 0.08064859### EXCLUSIONS DUE TO POOR MODEL FIT
## Exclude samples with at least two fails on the following three criteria:
## Significant chi-square value (p>.05), CFI >= .90, SRMR <= .10
fitexclude <- c(4,7,11,22,26,27,31,32,37,43,45)
## Data exclusions due to poor model fit
srel.meta <- srel.meta1[-fitexclude,]
## ADD NEW MODERATORS (see Table 1 in the manuscript)
srel.meta$LanguageEnglish <- c(1,1,1,0,0,1,1,1,1,1,1,1,1,1,1,1,0,1,1,
1,0,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,0)
srel.meta$Culture <- c("Mixed","Western","Western","Eastern","Eastern","Western","Western","Western","Western","Western","African","Western","Western","Western","Western","Western","Eastern","Western","Western","Western","African","Western","Western","Western","Eastern","Western","Mixed","Western","Western","Western","Western","Western","Western","Western","Western","Western","Western","Western","Western","Eastern","Western","Eastern","Western","Eastern")
table(srel.meta$Culture)##
## African Eastern Mixed Western
## 2 7 2 33srel.meta$CultureWestern <- c(0,1,1,0,0,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,0,1,1,1,
0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,0)
srel.meta$PropFemale <- c(69.83,52.94,53,52,62.1,75,54.17,79.17,
100,0,27.8,60,52,47.05,38.79,NA,31.66,NA,50,69.05,54,
51.27,51.27,51.27,47.94,58.33,70.59,75,49,73,48.9,48.9,
94.12,45,50,50,50,100,46,50.59,75,39.9,100,44.7)
head(srel.meta, 5)## ESID STUDID Reference N SREL1 SE.SREL1 SREL2 SE.SREL2
## 1 1 1 Acar 477 0.7879230 0.019422465 0.6165737 0.02990038
## 2 2 2 Acaretal 375 0.8659270 0.013861703 0.5688229 0.03750018
## 3 3 3 Auth 30 0.7700003 0.073695681 0.4562446 0.17409530
## 5 5 5 Balcib 105 0.6047810 0.079138763 0.3290440 0.10453343
## 6 6 6 Barkul 599 0.9744939 0.002080806 0.7786394 0.01432646
## SRELCR SE.SRELCR SREL1C SE.SREL1C SREL2C SE.SREL2C SRELCRC
## 1 0.7961029 0.015185967 0.7685262 0.01983469 0.5661430 0.03723139 0.8049032
## 2 0.8033699 0.016598931 0.8358593 0.01479798 0.6079821 0.03775852 0.8424830
## 3 0.6874343 0.089172197 0.6536706 0.10785668 0.4304457 0.22889696 0.7080218
## 5 0.5565458 0.067293016 0.4107403 0.10638499 0.3911985 0.12094086 0.5543792
## 6 0.9172739 0.005562633 0.7877730 0.02851258 0.8229706 0.02047607 0.9178338
## SE.SRELCRC Fcorr FcorrC Adults Forms Scores Validity SREL1.vg
## 1 0.014734698 0.8029838 0.6221182 1 A 0 1 0.000377232156
## 2 0.013847621 0.7857477 0.4256405 0 A 0 1 0.000192146808
## 3 0.093818456 0.5250131 0.4756440 1 <NA> 1 0 0.005431053334
## 5 0.071671866 0.4998183 0.4286476 0 Both 0 1 0.006262943736
## 6 0.005606417 0.8720216 0.8964628 1 A 0 0 0.000004329754
## SREL2.vg SRELCR.vg SREL1C.vg SREL2C.vg SRELCRC.vg
## 1 0.0008940326 0.00023061360 0.0003934151 0.0013861763 0.00021711132
## 2 0.0014062639 0.00027552451 0.0002189803 0.0014257057 0.00019175659
## 3 0.0303091744 0.00795168077 0.0116330628 0.0523938196 0.00880190271
## 5 0.0109272389 0.00452835001 0.0113177664 0.0146266914 0.00513685636
## 6 0.0002052475 0.00003094289 0.0008129674 0.0004192693 0.00003143191
## LanguageEnglish Culture CultureWestern PropFemale
## 1 1 Mixed 0 69.83
## 2 1 Western 1 52.94
## 3 1 Western 1 53.00
## 5 0 Eastern 0 52.00
## 6 0 Eastern 0 62.102.3.2 Description of the Meta-Analytic Data
## Number of effect sizes
length(srel.meta$ESID)## [1] 44## Number of primary studies
length(unique(srel.meta$STUDID))## [1] 44## Proportion adult samples
prop.table(table(srel.meta$Adults))##
## 0 1
## 0.5681818 0.4318182## Proportion evidence against validity
prop.table(table(srel.meta$Validity))##
## 0 1
## 0.6590909 0.3409091## Distribution of test forms
prop.table(table(srel.meta$Forms))##
## A B Both
## 0.79487179 0.15384615 0.05128205## Distribution of test score types
prop.table(table(srel.meta$Scores))##
## 0 1
## 0.5348837 0.4651163## Distribution of the sample sizes
psych::describe(srel.meta$N)## vars n mean sd median trimmed mad min max range skew kurtosis
## X1 1 44 271.25 392.02 100 194.47 112.68 12 1758 1746 1.95 3.33
## se
## X1 59.1sum(unique(srel.meta$N))## [1] 9260## Distribution of English language
prop.table(table(srel.meta$LanguageEnglish))##
## 0 1
## 0.1818182 0.8181818## Distribution of cultures
prop.table(table(srel.meta$Culture))##
## African Eastern Mixed Western
## 0.04545455 0.15909091 0.04545455 0.75000000prop.table(table(srel.meta$CultureWestern))##
## 0 1
## 0.25 0.75## Distribution of the proportion of women in the samples
psych::describe(srel.meta$PropFemale)## vars n mean sd median trimmed mad min max range skew kurtosis se
## X1 1 42 57.13 19.61 51.64 56.09 9.88 0 100 100 0.23 0.99 3.03The meta-analytic data are hierarchical with multiple effect sizes derived from independent samples that are nested in primary studies.
2.3.3 Meta-Analysis of the Two-Factor Model without Cross-Loadings
2.3.3.1 Meta-Analytic Baseline Models
## Reliability of factor 1
## Three-level REM
## Model specification and estimation
MLREMCSC.tfm.srel1 <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfm.srel1, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 31.4582 -62.9164 -56.9164 -51.6328 -56.3010
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0033 0.0576 44 no STUDID
## sigma^2.2 0.0033 0.0576 44 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 43) = 1086.5664, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.8540 0.0140 60.7950 43 <.0001 0.8257 0.8823 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfm.srel1, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0033 0.0000 0.0129
## sigma.1 0.0576 0.0000 0.1136
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0033 0.0000 0.0129
## sigma.2 0.0576 0.0000 0.1136## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfm.srel1))## % of total variance I2
## Level 1 1.196814 ---
## Level 2 49.401593 49.4
## Level 3 49.401593 49.4
## Total I2: 98.8%plot(dmetar::var.comp(MLREMCSC.tfm.srel1))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfm.srel1, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.854 0.014 38.3 0.826 0.882# Profile plots
profile.rma.mv(MLREMCSC.tfm.srel1, sigma2=1)
profile.rma.mv(MLREMCSC.tfm.srel1, sigma2=2)
## Reliability of factor 2
## Three-level REM
## Model specification and estimation
MLREMCSC.tfm.srel2 <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfm.srel2, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.7004 -37.4008 -31.4008 -26.1172 -30.7854
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0079 0.0890 44 no STUDID
## sigma^2.2 0.0079 0.0890 44 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 43) = 687.4240, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.6196 0.0220 28.2249 43 <.0001 0.5753 0.6638 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfm.srel2, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0079 0.0000 0.0290
## sigma.1 0.0890 0.0000 0.1702
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0079 0.0000 0.0290
## sigma.2 0.0890 0.0000 0.1702## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfm.srel2))## % of total variance I2
## Level 1 6.462584 ---
## Level 2 46.768708 46.77
## Level 3 46.768708 46.77
## Total I2: 93.54%plot(dmetar::var.comp(MLREMCSC.tfm.srel2))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfm.srel2, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.62 0.0219 40.1 0.575 0.664# Profile plots
profile.rma.mv(MLREMCSC.tfm.srel2, sigma2=1)
profile.rma.mv(MLREMCSC.tfm.srel2, sigma2=2)
## Composite reliability
## Three-level REM
## Model specification and estimation
MLREMCSC.tfm.srelcr <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfm.srelcr, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 43.6098 -87.2195 -81.2195 -75.9359 -80.6042
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0023 0.0479 44 no STUDID
## sigma^2.2 0.0023 0.0479 44 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 43) = 733.6577, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.8078 0.0118 68.3086 43 <.0001 0.7840 0.8317 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfm.srelcr, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0023 0.0000 0.0086
## sigma.1 0.0479 0.0000 0.0926
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0023 0.0000 0.0086
## sigma.2 0.0479 0.0000 0.0926## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfm.srelcr))## % of total variance I2
## Level 1 4.696466 ---
## Level 2 47.651767 47.65
## Level 3 47.651767 47.65
## Total I2: 95.3%plot(dmetar::var.comp(MLREMCSC.tfm.srelcr))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfm.srelcr, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.808 0.0118 39.3 0.784 0.832# Profile plots
profile.rma.mv(MLREMCSC.tfm.srel1, sigma2=1)
profile.rma.mv(MLREMCSC.tfm.srel1, sigma2=2)
2.3.3.2 Moderator Analyses for the Reliability of Factor 1
2.3.3.2.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.age <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 32.1164 -64.2328 -56.2328 -49.2822 -55.1518
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0032 0.0565 44 no STUDID
## sigma^2.2 0.0032 0.0565 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 1003.7200, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 4.2267, p-val = 0.0460
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8764 0.0174 50.2584 42 <.0001 0.8413 0.9116 ***
## Adults -0.0587 0.0285 -2.0559 42 0.0460 -0.1163 -0.0011 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8764 0.0132 22.6 0.849 0.90375
## Adults -0.0587 0.0324 30.3 -0.125 0.00754## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8764 0.0174 0.8413 0.9116 0.7114 1.0415
## 2 0.8178 0.0226 0.7722 0.8634 0.6502 0.98532.3.3.2.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.validity <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 31.5384 -63.0767 -55.0767 -48.1261 -53.9957
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0029 0.0543 44 no STUDID
## sigma^2.2 0.0029 0.0543 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 911.0285, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 3.1914, p-val = 0.0812
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8705 0.0157 55.3755 42 <.0001 0.8388 0.9022 ***
## Validity -0.0531 0.0297 -1.7865 42 0.0812 -0.1132 0.0069 .
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8705 0.0164 25.5 0.837 0.9042
## Validity -0.0531 0.0273 21.3 -0.110 0.0035## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8705 0.0157 0.8388 0.9022 0.7124 1.0286
## 2 0.8174 0.0252 0.7664 0.8683 0.6543 0.98052.3.3.2.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.forms <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfm.srel1.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 39; method: REML)
##
## logLik Deviance AIC BIC AICc
## 43.9121 -87.8242 -77.8242 -69.9066 -75.8242
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0012 0.0342 39 no STUDID
## sigma^2.2 0.0012 0.0342 39 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 36) = 906.1620, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 36) = 6.4205, p-val = 0.0041
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8894 0.0099 89.4683 36 <.0001 0.8692 0.9096 ***
## factor(Forms)B -0.1360 0.0380 -3.5813 36 0.0010 -0.2130 -0.0590 **
## factor(Forms)Both -0.0149 0.0453 -0.3280 36 0.7448 -0.1068 0.0771
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8894 0.00911 27.33 0.871 0.9081
## factor(Forms)B -0.1360 0.02845 2.49 -0.238 -0.0339
## factor(Forms)Both -0.0149 0.14657 1.10 -1.511 1.4815## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8894 0.0099 0.8692 0.9096 0.7892 0.9896
## 2 0.7534 0.0366 0.6791 0.8277 0.6303 0.8766
## 3 0.8745 0.0442 0.7848 0.9642 0.7415 1.00752.3.3.2.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.scores <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 43; method: REML)
##
## logLik Deviance AIC BIC AICc
## 33.4685 -66.9371 -58.9371 -52.0828 -57.8260
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0029 0.0543 43 no STUDID
## sigma^2.2 0.0029 0.0543 43 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 41) = 998.7100, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 41) = 0.0178, p-val = 0.8945
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8578 0.0185 46.3035 41 <.0001 0.8204 0.8952 ***
## Scores 0.0036 0.0268 0.1335 41 0.8945 -0.0506 0.0578
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.85782 0.0155 19.3 0.8253 0.8903
## Scores 0.00358 0.0271 35.9 -0.0515 0.0586## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8578 0.0185 0.8204 0.8952 0.6984 1.0173
## 2 0.8614 0.0194 0.8222 0.9006 0.7015 1.02132.3.3.2.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.lang <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 30.5043 -61.0085 -53.0085 -46.0578 -51.9274
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0036 0.0597 44 no STUDID
## sigma^2.2 0.0036 0.0597 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 536.7430, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.0166, p-val = 0.3191
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8228 0.0333 24.7290 42 <.0001 0.7556 0.8899 ***
## LanguageEnglish 0.0373 0.0370 1.0083 42 0.3191 -0.0373 0.1118
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8228 0.0625 6.35 0.672 0.974
## LanguageEnglish 0.0373 0.0635 9.55 -0.105 0.180## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8228 0.0333 0.7556 0.8899 0.6396 1.0060
## 2 0.8600 0.0161 0.8276 0.8925 0.6865 1.03362.3.3.2.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel1.cult <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 30.9885 -61.9770 -53.9770 -47.0263 -52.8959
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0035 0.0590 44 no STUDID
## sigma^2.2 0.0035 0.0590 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 605.1605, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.9487, p-val = 0.1701
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8187 0.0287 28.5648 42 <.0001 0.7608 0.8765 ***
## CultureWestern 0.0462 0.0331 1.3960 42 0.1701 -0.0206 0.1130
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8187 0.0476 8.66 0.7104 0.927
## CultureWestern 0.0462 0.0488 14.89 -0.0579 0.150## Predicted subgroup values
predict(MLMEMCSC.tfm.srel1.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8187 0.0287 0.7608 0.8765 0.6406 0.9967
## 2 0.8649 0.0165 0.8315 0.8983 0.6932 1.03652.3.3.2.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfm.srel1.female <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfm.srel1.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 42; method: REML)
##
## logLik Deviance AIC BIC AICc
## 28.4303 -56.8607 -48.8607 -42.1052 -47.7178
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0035 0.0592 42 no STUDID
## sigma^2.2 0.0035 0.0592 42 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 40) = 1070.4668, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 40) = 0.5254, p-val = 0.4728
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.8154 0.0528 15.4562 40 <.0001 0.7088
## asin(sqrt(PropFemale/100)) 0.0414 0.0572 0.7248 40 0.4728 -0.0741
## ci.ub
## intrcpt 0.9221 ***
## asin(sqrt(PropFemale/100)) 0.1569
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel1.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8154 0.0538 8.95 0.6937 0.937
## asin(sqrt(PropFemale/100)) 0.0414 0.0509 6.37 -0.0814 0.1642.3.3.3 Moderator Analyses for the Reliability of Factor 2
2.3.3.3.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.age <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 17.7180 -35.4359 -27.4359 -20.4853 -26.3549
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0082 0.0904 44 no STUDID
## sigma^2.2 0.0082 0.0904 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 682.3180, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.0299, p-val = 0.8635
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6159 0.0292 21.0746 42 <.0001 0.5569 0.6749 ***
## Adults 0.0078 0.0451 0.1729 42 0.8635 -0.0831 0.0987
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.61591 0.0298 22.4 0.5542 0.6776
## Adults 0.00779 0.0446 36.0 -0.0827 0.0983## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6159 0.0292 0.5569 0.6749 0.3511 0.8807
## 2 0.6237 0.0343 0.5545 0.6929 0.3565 0.89092.3.3.3.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.validity <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.8625 -37.7249 -29.7249 -22.7742 -28.6438
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0077 0.0880 44 no STUDID
## sigma^2.2 0.0077 0.0880 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 626.6437, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 2.2510, p-val = 0.1410
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6401 0.0256 24.9914 42 <.0001 0.5884 0.6918 ***
## Validity -0.0728 0.0485 -1.5003 42 0.1410 -0.1708 0.0251
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6401 0.0253 26.9 0.588 0.6921
## Validity -0.0728 0.0491 22.7 -0.174 0.0288## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6401 0.0256 0.5884 0.6918 0.3837 0.8965
## 2 0.5673 0.0412 0.4841 0.6505 0.3027 0.83182.3.3.3.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.forms <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfm.srel2.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 39; method: REML)
##
## logLik Deviance AIC BIC AICc
## 14.3449 -28.6899 -18.6899 -10.7723 -16.6899
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0087 0.0934 39 no STUDID
## sigma^2.2 0.0087 0.0934 39 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 36) = 641.7264, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 36) = 0.3712, p-val = 0.6925
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6209 0.0268 23.1926 36 <.0001 0.5666 0.6752 ***
## factor(Forms)B 0.0373 0.0679 0.5494 36 0.5861 -0.1004 0.1750
## factor(Forms)Both -0.0698 0.1144 -0.6102 36 0.5456 -0.3019 0.1622
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6209 0.0278 28.12 0.5640 0.678
## factor(Forms)B 0.0373 0.0340 6.78 -0.0437 0.118
## factor(Forms)Both -0.0698 0.1973 1.12 -2.0265 1.887## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6209 0.0268 0.5666 0.6752 0.3476 0.8942
## 2 0.6582 0.0624 0.5317 0.7847 0.3620 0.9545
## 3 0.5511 0.1112 0.3255 0.7767 0.2009 0.90132.3.3.3.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.scores <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 43; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.2093 -36.4186 -28.4186 -21.5644 -27.3075
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0076 0.0871 43 no STUDID
## sigma^2.2 0.0076 0.0871 43 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 41) = 449.2050, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 41) = 2.3124, p-val = 0.1360
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.5912 0.0301 19.6583 41 <.0001 0.5305 0.6519 ***
## Scores 0.0664 0.0436 1.5207 41 0.1360 -0.0218 0.1545
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.5912 0.0307 19.9 0.5271 0.655
## Scores 0.0664 0.0435 37.7 -0.0216 0.154## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.5912 0.0301 0.5305 0.6519 0.3352 0.8471
## 2 0.6576 0.0316 0.5937 0.7214 0.4008 0.91432.3.3.3.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.lang <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 17.7458 -35.4916 -27.4916 -20.5409 -26.4105
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0082 0.0906 44 no STUDID
## sigma^2.2 0.0082 0.0906 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 441.6944, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.1804, p-val = 0.6732
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6005 0.0493 12.1782 42 <.0001 0.5010 0.7000 ***
## LanguageEnglish 0.0235 0.0553 0.4247 42 0.6732 -0.0881 0.1350
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6005 0.0734 6.77 0.426 0.775
## LanguageEnglish 0.0235 0.0764 10.54 -0.145 0.192## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6005 0.0493 0.5010 0.7000 0.3233 0.8776
## 2 0.6239 0.0250 0.5735 0.6743 0.3604 0.88752.3.3.3.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srel2.cult <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 18.2623 -36.5245 -28.5245 -21.5738 -27.4434
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0080 0.0895 44 no STUDID
## sigma^2.2 0.0080 0.0895 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 489.6220, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.1829, p-val = 0.2830
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.5800 0.0424 13.6814 42 <.0001 0.4945 0.6656 ***
## CultureWestern 0.0540 0.0497 1.0876 42 0.2830 -0.0462 0.1542
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.580 0.0611 9.6 0.4431 0.717
## CultureWestern 0.054 0.0644 17.3 -0.0818 0.190## Predicted subgroup values
predict(MLMEMCSC.tfm.srel2.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.5800 0.0424 0.4945 0.6656 0.3105 0.8496
## 2 0.6341 0.0258 0.5819 0.6862 0.3732 0.89492.3.3.3.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfm.srel2.female <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfm.srel2.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 42; method: REML)
##
## logLik Deviance AIC BIC AICc
## 16.9573 -33.9145 -25.9145 -19.1590 -24.7717
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0082 0.0906 42 no STUDID
## sigma^2.2 0.0082 0.0906 42 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 40) = 681.8770, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 40) = 1.4018, p-val = 0.2434
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.5243 0.0787 6.6616 40 <.0001 0.3652
## asin(sqrt(PropFemale/100)) 0.1012 0.0854 1.1840 40 0.2434 -0.0715
## ci.ub
## intrcpt 0.6833 ***
## asin(sqrt(PropFemale/100)) 0.2739
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srel2.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.524 0.0578 8.91 0.3933 0.655
## asin(sqrt(PropFemale/100)) 0.101 0.0520 6.27 -0.0248 0.2272.3.3.4 Moderator Analyses for the Composite Reliability
2.3.3.4.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.age <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 42.5517 -85.1034 -77.1034 -70.1527 -76.0223
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0023 0.0482 44 no STUDID
## sigma^2.2 0.0023 0.0482 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 732.4656, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.1021, p-val = 0.2998
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8178 0.0153 53.3680 42 <.0001 0.7869 0.8487 ***
## Adults -0.0255 0.0243 -1.0498 42 0.2998 -0.0746 0.0235
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8178 0.0140 22.6 0.7889 0.847
## Adults -0.0255 0.0253 33.5 -0.0771 0.026## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8178 0.0153 0.7869 0.8487 0.6767 0.9589
## 2 0.7923 0.0189 0.7542 0.8304 0.6494 0.93512.3.3.4.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.validity <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 42.9839 -85.9678 -77.9678 -71.0171 -76.8867
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0022 0.0472 44 no STUDID
## sigma^2.2 0.0022 0.0472 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 646.9318, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.9200, p-val = 0.1732
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8181 0.0137 59.5123 42 <.0001 0.7904 0.8458 ***
## Validity -0.0362 0.0261 -1.3856 42 0.1732 -0.0888 0.0165
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8181 0.0136 26.5 0.7902 0.8459
## Validity -0.0362 0.0265 21.8 -0.0911 0.0187## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8181 0.0137 0.7904 0.8458 0.6806 0.9556
## 2 0.7819 0.0222 0.7372 0.8267 0.6401 0.92382.3.3.4.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.forms <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 39; method: REML)
##
## logLik Deviance AIC BIC AICc
## 39.2491 -78.4982 -68.4982 -60.5806 -66.4982
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0019 0.0433 39 no STUDID
## sigma^2.2 0.0019 0.0433 39 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 36) = 633.6879, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 36) = 1.3003, p-val = 0.2849
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8255 0.0125 66.0309 36 <.0001 0.8001 0.8508 ***
## factor(Forms)B -0.0501 0.0354 -1.4138 36 0.1660 -0.1219 0.0218
## factor(Forms)Both -0.0489 0.0552 -0.8863 36 0.3813 -0.1608 0.0630
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8255 0.0115 27.64 0.802 0.8491
## factor(Forms)B -0.0501 0.0262 5.37 -0.116 0.0159
## factor(Forms)Both -0.0489 0.1613 1.11 -1.673 1.5750## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8255 0.0125 0.8001 0.8508 0.6986 0.9523
## 2 0.7754 0.0331 0.7081 0.8426 0.6341 0.9167
## 3 0.7766 0.0537 0.6676 0.8855 0.6113 0.94192.3.3.4.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.scores <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 43; method: REML)
##
## logLik Deviance AIC BIC AICc
## 42.8102 -85.6204 -77.6204 -70.7661 -76.5093
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0021 0.0462 43 no STUDID
## sigma^2.2 0.0021 0.0462 43 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 41) = 583.3724, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 41) = 0.1278, p-val = 0.7225
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8076 0.0160 50.5906 41 <.0001 0.7754 0.8398 ***
## Scores 0.0083 0.0232 0.3575 41 0.7225 -0.0385 0.0551
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.80759 0.0154 19.4 0.7755 0.8397
## Scores 0.00829 0.0232 36.9 -0.0388 0.0554## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8076 0.0160 0.7754 0.8398 0.6718 0.9434
## 2 0.8159 0.0168 0.7819 0.8499 0.6796 0.95212.3.3.4.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.lang <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 42.3185 -84.6369 -76.6369 -69.6863 -75.5559
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0024 0.0489 44 no STUDID
## sigma^2.2 0.0024 0.0489 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 452.7653, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.7257, p-val = 0.3991
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7870 0.0269 29.2563 42 <.0001 0.7327 0.8412 ***
## LanguageEnglish 0.0256 0.0301 0.8519 42 0.3991 -0.0351 0.0863
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7870 0.0434 6.62 0.6831 0.891
## LanguageEnglish 0.0256 0.0447 10.20 -0.0738 0.125## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7870 0.0269 0.7327 0.8412 0.6373 0.9366
## 2 0.8126 0.0134 0.7854 0.8397 0.6705 0.95462.3.3.4.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfm.srelcr.cult <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 43.3295 -86.6589 -78.6589 -71.7083 -77.5779
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0023 0.0475 44 no STUDID
## sigma^2.2 0.0023 0.0475 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 515.1130, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 2.7482, p-val = 0.1048
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7756 0.0228 34.0287 42 <.0001 0.7296 0.8216 ***
## CultureWestern 0.0441 0.0266 1.6578 42 0.1048 -0.0096 0.0978
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7756 0.0345 9.34 0.698 0.853
## CultureWestern 0.0441 0.0360 16.62 -0.032 0.120## Predicted subgroup values
predict(MLMEMCSC.tfm.srelcr.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7756 0.0228 0.7296 0.8216 0.6325 0.9187
## 2 0.8197 0.0137 0.7920 0.8473 0.6814 0.95802.3.3.4.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfm.srelcr.female <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 42; method: REML)
##
## logLik Deviance AIC BIC AICc
## 39.7329 -79.4658 -71.4658 -64.7102 -70.3229
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0025 0.0496 42 no STUDID
## sigma^2.2 0.0025 0.0496 42 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 40) = 725.7394, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 40) = 0.5529, p-val = 0.4615
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.7733 0.0431 17.9546 40 <.0001 0.6863
## asin(sqrt(PropFemale/100)) 0.0348 0.0469 0.7436 40 0.4615 -0.0599
## ci.ub
## intrcpt 0.8603 ***
## asin(sqrt(PropFemale/100)) 0.1295
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfm.srelcr.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7733 0.0321 8.51 0.7001 0.847
## asin(sqrt(PropFemale/100)) 0.0348 0.0282 6.16 -0.0337 0.1032.3.3.5 Publication Bias and Infuential Effect Sizes
## Reliability of factor 1
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfm.srel1.pet <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SREL1.vg))
## Summarize the results
summary(MLMEMCSC.tfm.srel1.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 50.5279 -101.0558 -93.0558 -86.1052 -91.9748
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0011 0.0333 44 no STUDID
## sigma^2.2 0.0011 0.0333 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 512.9322, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 51.3170, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.9319 0.0125 74.4776 42 <.0001 0.9067 0.9572 ***
## sqrt(SREL1.vg) -2.4858 0.3470 -7.1636 42 <.0001 -3.1860 -1.7855 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel1.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.932 0.0104 89.61 23.9 <0.001 ***
## sqrt(SREL1.vg) -2.486 0.3750 -6.63 18.3 <0.001 ***## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfm.srel1.fpt <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 31.3081 -62.6162 -54.6162 -47.6655 -53.5351
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0033 0.0572 44 no STUDID
## sigma^2.2 0.0033 0.0572 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 1085.2507, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 2.7879, p-val = 0.1024
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8357 0.0178 46.8873 42 <.0001 0.7997 0.8717 ***
## N 0.0001 0.0000 1.6697 42 0.1024 -0.0000 0.0001
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel1.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.8356840 0.0176658 47.31 31.08 <0.001 ***
## N 0.0000554 0.0000228 2.43 5.88 0.0523 .## Contour-enhanced funnel plot
funnel(MLREMCSC.tfm.srel1,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfm.srel1.peese <- rma.mv(SREL1,
SREL1.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SREL1.vg)
## Summarize the results
summary(MLMEMCSC.tfm.srel1.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 42.2879 -84.5758 -76.5758 -69.6252 -75.4948
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0019 0.0441 44 no STUDID
## sigma^2.2 0.0019 0.0441 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 947.5408, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 25.3902, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8797 0.0118 74.7350 42 <.0001 0.8559 0.9034 ***
## SREL1.vg -11.0495 2.1929 -5.0389 42 <.0001 -15.4749 -6.6242 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel1.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.88 0.0114 76.84 33.79 <0.001 ***
## SREL1.vg -11.05 3.0788 -3.59 3.58 0.0276 *## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfm.srel1 <- influence.rma.uni(rma(SREL1,
SREL1.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfm.srel1)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfm.srel1)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 -0.7973 -0.1319 0.0175 1.0321 0.0067 1016.2702 0.0281 2.8099 -0.1319
## 2 0.1520 0.0799 0.0067 1.0800 0.0070 1049.1006 0.0289 2.8859 0.0801
## 3 -0.7696 -0.0971 0.0094 1.0185 0.0067 1080.5682 0.0163 1.6344 -0.0971
## 4 -2.3093 -0.4237 0.1670 0.8961 0.0057 1067.4820 0.0153 1.5291 -0.4348
## 5 1.5258 0.2235 0.0478 0.9925 0.0064 890.2013 0.0297 2.9674 0.2228
## 6 0.3342 0.1021 0.0109 1.0706 0.0070 1082.0745 0.0257 2.5742 0.1021
## 7 0.8593 0.1690 0.0293 1.0523 0.0068 1085.9693 0.0263 2.6324 0.1690
## 8 0.6349 0.1403 0.0204 1.0610 0.0069 1085.2319 0.0251 2.5063 0.1403
## 9 -0.4506 -0.0401 0.0016 1.0342 0.0068 1082.4175 0.0168 1.6802 -0.0400
## 10 -0.9110 -0.1161 0.0134 1.0084 0.0066 1080.8760 0.0142 1.4228 -0.1162
## 11 -1.4639 -0.1197 0.0143 0.9871 0.0065 1082.1516 0.0047 0.4701 -0.1205
## 12 0.5244 0.1235 0.0158 1.0623 0.0069 1085.0096 0.0240 2.4039 0.1234
## 13 1.4428 0.2187 0.0464 1.0028 0.0064 1084.2524 0.0290 2.8969 0.2183
## 14 0.1005 0.0489 0.0024 1.0502 0.0069 1084.8453 0.0180 1.7963 0.0487
## 15 -0.9908 -0.1600 0.0253 1.0080 0.0065 1073.3917 0.0210 2.0967 -0.1602
## 16 1.2279 0.2074 0.0428 1.0258 0.0066 1086.5042 0.0289 2.8880 0.2073
## 17 -4.7213 -1.3508 1.2490 0.5875 0.0035 1021.0850 0.0195 1.9494 -1.4702 *
## 18 -0.8943 -0.1416 0.0199 1.0169 0.0066 1072.0246 0.0223 2.2271 -0.1416
## 19 0.5046 0.1364 0.0196 1.0759 0.0070 989.9328 0.0295 2.9545 0.1369
## 20 -0.6295 -0.0691 0.0048 1.0234 0.0067 1082.1504 0.0151 1.5124 -0.0690
## 21 0.7156 0.1602 0.0267 1.0644 0.0069 1081.6672 0.0285 2.8478 0.1605
## 22 0.7264 0.1646 0.0282 1.0659 0.0069 1038.0706 0.0296 2.9559 0.1651
## 23 0.3595 0.1150 0.0140 1.0796 0.0070 1001.7169 0.0294 2.9450 0.1154
## 24 0.9154 0.1848 0.0350 1.0536 0.0068 1061.4372 0.0296 2.9600 0.1851
## 25 1.4529 0.2212 0.0473 1.0013 0.0064 931.5610 0.0297 2.9677 0.2206
## 26 -0.8607 -0.0594 0.0035 1.0009 0.0066 1084.5916 0.0042 0.4183 -0.0595
## 27 0.1068 0.0699 0.0051 1.0768 0.0070 1069.8402 0.0277 2.7749 0.0700
## 28 0.4179 0.0953 0.0093 1.0526 0.0069 1085.7434 0.0192 1.9228 0.0949
## 29 -2.0785 -0.1814 0.0327 0.9681 0.0064 1078.9459 0.0047 0.4678 -0.1841
## 30 -1.7356 -0.0933 0.0087 0.9883 0.0065 1082.0914 0.0020 0.2032 -0.0939
## 31 0.6360 0.1530 0.0244 1.0700 0.0070 1065.4814 0.0293 2.9274 0.1534
## 32 -0.8107 -0.1342 0.0180 1.0301 0.0067 1035.9488 0.0275 2.7482 -0.1342
## 33 0.2451 0.0881 0.0081 1.0711 0.0070 1081.2484 0.0256 2.5570 0.0881
## 34 0.8746 0.1776 0.0324 1.0549 0.0068 1084.3037 0.0286 2.8572 0.1779
## 35 -0.1007 0.0246 0.0006 1.0596 0.0069 1081.2045 0.0225 2.2520 0.0246
## 36 0.7579 0.1666 0.0288 1.0631 0.0069 1078.9070 0.0290 2.9018 0.1670
## 37 -0.1332 0.0149 0.0002 1.0503 0.0069 1083.1603 0.0193 1.9294 0.0149
## 38 0.0784 0.0637 0.0043 1.0749 0.0070 1073.5498 0.0271 2.7123 0.0638
## 39 -0.1773 0.0147 0.0002 1.0673 0.0070 1071.9008 0.0263 2.6272 0.0147
## 40 -3.0652 -0.4428 0.1865 0.8769 0.0056 1067.1248 0.0095 0.9548 -0.4632
## 41 0.0381 0.0595 0.0037 1.0791 0.0070 1040.8474 0.0288 2.8848 0.0597
## 42 0.1389 0.0761 0.0061 1.0778 0.0070 1067.6170 0.0281 2.8061 0.0762
## 43 -0.5354 -0.0624 0.0040 1.0492 0.0068 1065.1380 0.0259 2.5941 -0.0624
## 44 0.3497 0.1136 0.0136 1.0799 0.0070 992.4820 0.0295 2.9468 0.1140## Reliability of factor 2
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfm.srel2.pet <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SREL2.vg))
## Summarize the results
summary(MLMEMCSC.tfm.srel2.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 23.2131 -46.4262 -38.4262 -31.4755 -37.3451
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0063 0.0796 44 no STUDID
## sigma^2.2 0.0063 0.0796 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 475.8382, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 11.5036, p-val = 0.0015
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7254 0.0364 19.9125 42 <.0001 0.6518 0.7989 ***
## sqrt(SREL2.vg) -1.7269 0.5092 -3.3917 42 0.0015 -2.7545 -0.6994 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel2.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.725 0.0368 19.69 20.6 < 0.001 ***
## sqrt(SREL2.vg) -1.727 0.5463 -3.16 22.3 0.00448 **## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfm.srel2.fpt <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 17.6227 -35.2454 -27.2454 -20.2947 -26.1643
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0083 0.0909 44 no STUDID
## sigma^2.2 0.0083 0.0909 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 679.3818, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.0083, p-val = 0.9280
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6175 0.0285 21.6700 42 <.0001 0.5600 0.6750 ***
## N 0.0000 0.0001 0.0910 42 0.9280 -0.0001 0.0001
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel2.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.61746242 0.0268281 23.0155 33.97 <0.001 ***
## N 0.00000483 0.0000546 0.0884 5.82 0.932## Contour-enhanced funnel plot
funnel(MLREMCSC.tfm.srel2,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfm.srel2.peese <- rma.mv(SREL2,
SREL2.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SREL2.vg)
## Summarize the results
summary(MLMEMCSC.tfm.srel2.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 23.7085 -47.4170 -39.4170 -32.4664 -38.3359
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0062 0.0790 44 no STUDID
## sigma^2.2 0.0062 0.0790 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 578.7327, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 12.3539, p-val = 0.0011
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6785 0.0256 26.5318 42 <.0001 0.6269 0.7301 ***
## SREL2.vg -10.9840 3.1251 -3.5148 42 0.0011 -17.2906 -4.6774 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srel2.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.679 0.0259 26.16 25.9 < 0.001 ***
## SREL2.vg -10.984 3.5211 -3.12 10.9 0.00982 **## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfm.srel2 <- influence.rma.uni(rma(SREL2,
SREL2.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfm.srel2)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfm.srel2)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 -0.0192 0.0195 0.0004 1.0651 0.0165 674.3450 0.0288 2.8811 0.0195
## 2 -0.3834 -0.0474 0.0023 1.0554 0.0163 670.1693 0.0280 2.7954 -0.0475
## 3 -0.7639 -0.0770 0.0059 1.0129 0.0159 685.0678 0.0104 1.0444 -0.0770
## 4 -1.8358 -0.2856 0.0782 0.9611 0.0147 673.1647 0.0180 1.8008 -0.2878
## 5 1.2851 0.2169 0.0462 1.0162 0.0155 670.7878 0.0300 3.0048 0.2165
## 6 0.2780 0.0626 0.0040 1.0538 0.0164 686.4384 0.0240 2.3967 0.0625
## 7 0.0490 0.0207 0.0004 1.0390 0.0163 686.6727 0.0172 1.7150 0.0206
## 8 -0.0521 0.0050 0.0000 1.0311 0.0162 686.7350 0.0139 1.3917 0.0050
## 9 1.1725 0.1779 0.0316 1.0205 0.0158 686.6255 0.0230 2.3024 0.1779
## 10 -0.4780 -0.0523 0.0028 1.0285 0.0161 685.0607 0.0157 1.5686 -0.0522
## 11 0.1439 0.0361 0.0013 1.0439 0.0163 686.7017 0.0193 1.9253 0.0361
## 12 1.5873 0.2317 0.0518 0.9912 0.0152 683.7018 0.0253 2.5284 0.2316
## 13 0.8580 0.1315 0.0175 1.0324 0.0161 687.3281 0.0203 2.0292 0.1314
## 14 0.7645 0.1139 0.0131 1.0324 0.0161 687.3879 0.0183 1.8334 0.1137
## 15 -2.7516 -0.3964 0.1475 0.9066 0.0137 668.0774 0.0143 1.4279 -0.4061
## 16 0.7288 0.1256 0.0160 1.0425 0.0162 687.4240 0.0237 2.3691 0.1256
## 17 -0.7243 -0.1133 0.0130 1.0381 0.0160 668.1287 0.0269 2.6875 -0.1133
## 18 0.8384 0.1485 0.0224 1.0425 0.0161 687.3946 0.0267 2.6651 0.1486
## 19 0.3883 0.0898 0.0084 1.0651 0.0165 669.0688 0.0301 3.0084 0.0901
## 20 0.0759 0.0264 0.0007 1.0435 0.0163 686.5236 0.0191 1.9067 0.0263
## 21 -1.1051 -0.1723 0.0294 1.0099 0.0156 676.4486 0.0216 2.1609 -0.1724
## 22 0.5666 0.1175 0.0143 1.0596 0.0164 683.3453 0.0299 2.9917 0.1178
## 23 -1.2792 -0.2430 0.0569 0.9987 0.0152 610.1505 0.0288 2.8765 -0.2423
## 24 -0.3625 -0.0442 0.0020 1.0593 0.0164 639.5891 0.0295 2.9513 -0.0443
## 25 1.9153 0.2866 0.0742 0.9519 0.0143 348.0298 0.0303 3.0310 0.2836
## 26 -2.6932 -0.3158 0.0966 0.9337 0.0143 672.9118 0.0102 1.0153 -0.3232
## 27 -2.5825 -0.4318 0.1702 0.8990 0.0135 663.8248 0.0182 1.8238 -0.4390
## 28 -0.2803 -0.0216 0.0005 1.0208 0.0160 686.5591 0.0102 1.0207 -0.0216
## 29 0.1894 0.0430 0.0019 1.0446 0.0163 686.7889 0.0196 1.9600 0.0429
## 30 0.3301 0.0638 0.0041 1.0453 0.0163 687.0595 0.0204 2.0355 0.0636
## 31 0.7853 0.1484 0.0225 1.0492 0.0162 687.3992 0.0293 2.9326 0.1487
## 32 -0.0168 0.0196 0.0004 1.0641 0.0165 677.5841 0.0283 2.8335 0.0196
## 33 0.0973 0.0331 0.0011 1.0512 0.0164 685.9811 0.0225 2.2454 0.0331
## 34 -0.4564 -0.0558 0.0032 1.0402 0.0162 682.9079 0.0215 2.1493 -0.0557
## 35 -0.5152 -0.0585 0.0035 1.0287 0.0161 684.7281 0.0163 1.6307 -0.0584
## 36 0.3049 0.0721 0.0054 1.0610 0.0165 685.0700 0.0274 2.7416 0.0723
## 37 0.0150 0.0142 0.0002 1.0339 0.0162 686.7923 0.0150 1.5000 0.0142
## 38 0.0323 0.0261 0.0007 1.0588 0.0165 683.8348 0.0259 2.5865 0.0261
## 39 0.9301 0.1640 0.0272 1.0392 0.0160 687.2183 0.0278 2.7760 0.1641
## 40 -0.8262 -0.1166 0.0136 1.0237 0.0159 680.8050 0.0202 2.0187 -0.1166
## 41 0.2960 0.0731 0.0056 1.0646 0.0165 681.6355 0.0290 2.9020 0.0734
## 42 1.3945 0.2262 0.0498 1.0066 0.0154 680.5347 0.0291 2.9052 0.2257
## 43 0.6850 0.1299 0.0173 1.0510 0.0163 687.2998 0.0277 2.7681 0.1301
## 44 -1.9446 -0.4190 0.1549 0.9269 0.0139 572.9768 0.0286 2.8617 -0.4150## Reliability of factor 1
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfm.srelcr.pet <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SRELCR.vg))
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 52.1067 -104.2135 -96.2135 -89.2628 -95.1324
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0015 0.0382 44 no STUDID
## sigma^2.2 0.0015 0.0382 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 468.1269, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 21.6672, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8718 0.0163 53.6439 42 <.0001 0.8390 0.9046 ***
## sqrt(SRELCR.vg) -2.0332 0.4368 -4.6548 42 <.0001 -2.9147 -1.1517 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srelcr.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.872 0.014 62.42 20.8 <0.001 ***
## sqrt(SRELCR.vg) -2.033 0.402 -5.05 18.4 <0.001 ***## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfm.srelcr.fpt <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 42.5214 -85.0428 -77.0428 -70.0921 -75.9617
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0023 0.0484 44 no STUDID
## sigma^2.2 0.0023 0.0484 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 727.4392, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.2323, p-val = 0.2733
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7970 0.0153 52.1098 42 <.0001 0.7661 0.8279 ***
## N 0.0000 0.0000 1.1101 42 0.2733 -0.0000 0.0001
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srelcr.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.7969885 0.0151634 52.56 33.02 <0.001 ***
## N 0.0000314 0.0000232 1.35 5.86 0.226## Contour-enhanced funnel plot
funnel(MLREMCSC.tfm.srelcr,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfm.srelcr.peese <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SRELCR.vg)
## Summarize the results
summary(MLMEMCSC.tfm.srelcr.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 49.8151 -99.6302 -91.6302 -84.6796 -90.5492
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0016 0.0401 44 no STUDID
## sigma^2.2 0.0016 0.0401 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 627.4626, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 15.2622, p-val = 0.0003
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8310 0.0115 72.4628 42 <.0001 0.8078 0.8541 ***
## SRELCR.vg -14.0584 3.5986 -3.9067 42 0.0003 -21.3206 -6.7963 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfm.srelcr.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.831 0.0131 63.50 26.36 <0.001 ***
## SRELCR.vg -14.058 6.7475 -2.08 2.59 0.143## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfm.srelcr <- influence.rma.uni(rma(SRELCR,
SRELCR.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfm.srelcr)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfm.srelcr)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 -0.1620 0.0037 0.0000 1.0671 0.0048 708.7936 0.0290 2.9044 0.0037
## 2 -0.0574 0.0231 0.0006 1.0686 0.0048 716.7401 0.0288 2.8776 0.0232
## 3 -1.0840 -0.1227 0.0150 1.0022 0.0045 729.4139 0.0112 1.1157 -0.1230
## 4 -2.7828 -0.4647 0.1984 0.8808 0.0038 711.7884 0.0153 1.5348 -0.4781 *
## 5 1.6836 0.2492 0.0581 0.9763 0.0043 652.9455 0.0303 3.0300 0.2477
## 6 0.2072 0.0617 0.0039 1.0588 0.0048 731.7622 0.0241 2.4136 0.0617
## 7 0.5549 0.1019 0.0106 1.0456 0.0047 733.5168 0.0205 2.0505 0.1017
## 8 0.5000 0.0923 0.0087 1.0447 0.0047 733.4825 0.0195 1.9525 0.0921
## 9 0.7213 0.1258 0.0161 1.0432 0.0047 733.6403 0.0217 2.1689 0.1257
## 10 -0.3998 -0.0382 0.0015 1.0336 0.0047 730.9835 0.0168 1.6824 -0.0381
## 11 -0.7713 -0.0903 0.0082 1.0169 0.0046 729.9755 0.0142 1.4171 -0.0903
## 12 1.3886 0.2087 0.0428 1.0082 0.0045 732.0757 0.0256 2.5551 0.2087
## 13 1.2454 0.1897 0.0358 1.0187 0.0046 732.9132 0.0239 2.3911 0.1897
## 14 0.9401 0.1482 0.0222 1.0333 0.0047 733.5772 0.0210 2.0957 0.1480
## 15 -2.8386 -0.4622 0.1971 0.8805 0.0038 712.1145 0.0147 1.4692 -0.4764 *
## 16 1.2105 0.1960 0.0382 1.0226 0.0046 732.8574 0.0266 2.6583 0.1959
## 17 -1.9297 -0.4187 0.1570 0.9216 0.0040 689.9162 0.0256 2.5610 -0.4181
## 18 0.4040 0.0955 0.0094 1.0614 0.0048 732.0884 0.0263 2.6255 0.0956
## 19 0.4118 0.1049 0.0115 1.0699 0.0048 699.3885 0.0302 3.0233 0.1053
## 20 -0.1807 -0.0043 0.0000 1.0440 0.0047 731.3302 0.0192 1.9249 -0.0043
## 21 -0.4718 -0.0560 0.0032 1.0468 0.0047 724.8751 0.0245 2.4507 -0.0560
## 22 0.8936 0.1722 0.0303 1.0485 0.0047 733.4553 0.0302 3.0193 0.1725
## 23 -0.7224 -0.1158 0.0136 1.0416 0.0046 652.3538 0.0295 2.9471 -0.1159
## 24 0.2914 0.0852 0.0076 1.0717 0.0048 708.4478 0.0300 2.9982 0.0856
## 25 1.8316 0.2590 0.0612 0.9583 0.0042 450.8212 0.0304 3.0406 0.2568
## 26 -1.4895 -0.0817 0.0067 0.9942 0.0045 730.3010 0.0025 0.2514 -0.0820
## 27 -1.9124 -0.3717 0.1275 0.9333 0.0041 709.0301 0.0221 2.2110 -0.3737
## 28 0.0826 0.0250 0.0006 1.0304 0.0047 733.2082 0.0124 1.2432 0.0249
## 29 -0.9366 -0.1143 0.0130 1.0100 0.0046 729.2123 0.0138 1.3835 -0.1144
## 30 -1.3909 -0.1473 0.0216 0.9908 0.0045 728.6236 0.0090 0.8960 -0.1482
## 31 0.9956 0.1832 0.0340 1.0412 0.0046 733.4916 0.0298 2.9774 0.1834
## 32 -0.1938 -0.0026 0.0000 1.0653 0.0048 713.6792 0.0286 2.8612 -0.0026
## 33 -0.3835 -0.0378 0.0015 1.0432 0.0047 729.2323 0.0211 2.1150 -0.0378
## 34 0.1729 0.0576 0.0034 1.0607 0.0048 731.1802 0.0249 2.4871 0.0576
## 35 -0.1468 0.0013 0.0000 1.0461 0.0047 731.3087 0.0199 1.9868 0.0013
## 36 0.4415 0.1054 0.0116 1.0650 0.0048 730.7008 0.0283 2.8282 0.1057
## 37 -0.3691 -0.0319 0.0010 1.0287 0.0047 731.7686 0.0142 1.4150 -0.0318
## 38 0.1330 0.0534 0.0030 1.0643 0.0048 729.6191 0.0263 2.6344 0.0535
## 39 0.8801 0.1648 0.0277 1.0466 0.0047 733.6488 0.0282 2.8231 0.1650
## 40 -2.1645 -0.3493 0.1154 0.9301 0.0041 717.8053 0.0159 1.5898 -0.3550
## 41 0.4015 0.1017 0.0108 1.0684 0.0048 725.7256 0.0294 2.9438 0.1021
## 42 0.6884 0.1410 0.0205 1.0568 0.0047 732.9203 0.0285 2.8504 0.1413
## 43 -0.1660 0.0016 0.0000 1.0609 0.0048 725.6035 0.0264 2.6379 0.0016
## 44 -0.5291 -0.0718 0.0053 1.0538 0.0047 662.4034 0.0296 2.9573 -0.07202.3.3.6 Forest Plots
## Reliability of factor 1
#pdf(file = "ForestPlots-TwoFactorModel4.pdf", onefile = TRUE)
forest.rma(MLREMCSC.tfm.srel1,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfm.srel1$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Reliability Factor 1 (Model 4)")
## Reliability of factor 2
forest.rma(MLREMCSC.tfm.srel2,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfm.srel2$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Reliability Factor 2 (Model 4)")
## Composite reliability
forest.rma(MLREMCSC.tfm.srelcr,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfm.srelcr$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Composite Reliability (Model 4)")
#dev.off()2.3.4 Meta-Analysis of the Two-Factor Model with Cross-Loadings
2.3.4.1 Meta-Analytic Baseline Models
## Reliability of factor 1
## Three-level REM
## Model specification and estimation
MLREMCSC.tfmc.srel1 <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfmc.srel1, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 13.1856 -26.3712 -20.3712 -15.0876 -19.7558
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0078 0.0884 44 no STUDID
## sigma^2.2 0.0078 0.0884 44 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 43) = 475.4746, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.7884 0.0217 36.3306 43 <.0001 0.7446 0.8322 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfmc.srel1, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0078 0.0000 0.0302
## sigma.1 0.0884 0.0000 0.1739
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0078 0.0000 0.0302
## sigma.2 0.0884 0.0000 0.1739## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfmc.srel1))## % of total variance I2
## Level 1 1.714521 ---
## Level 2 49.142740 49.14
## Level 3 49.142740 49.14
## Total I2: 98.29%plot(dmetar::var.comp(MLREMCSC.tfmc.srel1))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfmc.srel1, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.788 0.0216 36.7 0.745 0.832# Profile plots
profile.rma.mv(MLREMCSC.tfmc.srel1, sigma2=1)
profile.rma.mv(MLREMCSC.tfmc.srel1, sigma2=2)
## Reliability of factor 2
## Three-level REM
## Model specification and estimation
MLREMCSC.tfmc.srel2 <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfmc.srel2, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 13.8935 -27.7869 -21.7869 -16.5033 -21.1715
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0071 0.0845 44 no STUDID
## sigma^2.2 0.0071 0.0845 44 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 43) = 511.0945, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.6468 0.0229 28.2471 43 <.0001 0.6006 0.6930 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfmc.srel2, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0071 0.0000 0.0277
## sigma.1 0.0845 0.0000 0.1663
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0071 0.0000 0.0277
## sigma.2 0.0845 0.0000 0.1663## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfmc.srel2))## % of total variance I2
## Level 1 8.586 ---
## Level 2 45.707 45.71
## Level 3 45.707 45.71
## Total I2: 91.41%plot(dmetar::var.comp(MLREMCSC.tfmc.srel2))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfmc.srel2, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.647 0.0229 33.5 0.6 0.693# Profile plots
profile.rma.mv(MLREMCSC.tfmc.srel2, sigma2=1)
profile.rma.mv(MLREMCSC.tfmc.srel2, sigma2=2)
## Composite reliability
## Three-level REM
## Model specification and estimation
MLREMCSC.tfmc.srelcr <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t")
## Summarize the results
summary(MLREMCSC.tfmc.srelcr, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 44.8311 -89.6623 -83.6623 -78.3787 -83.0469
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0022 0.0473 44 no STUDID
## sigma^2.2 0.0022 0.0473 44 no STUDID/ESID
##
## Test for Heterogeneity:
## Q(df = 43) = 563.1827, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## 0.8244 0.0117 70.4730 43 <.0001 0.8008 0.8480 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## 95% confidence intervals
confint(MLREMCSC.tfmc.srelcr, digits=4)##
## estimate ci.lb ci.ub
## sigma^2.1 0.0022 0.0000 0.0083
## sigma.1 0.0473 0.0000 0.0909
##
## estimate ci.lb ci.ub
## sigma^2.2 0.0022 0.0000 0.0083
## sigma.2 0.0473 0.0000 0.0909## Variance distribution
summary(dmetar::var.comp(MLREMCSC.tfmc.srelcr))## % of total variance I2
## Level 1 4.281214 ---
## Level 2 47.859393 47.86
## Level 3 47.859393 47.86
## Total I2: 95.72%plot(dmetar::var.comp(MLREMCSC.tfmc.srelcr))
## RVE standard errors
clubSandwich::conf_int(MLREMCSC.tfmc.srelcr, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.824 0.0117 38.9 0.801 0.848# Profile plots
profile.rma.mv(MLREMCSC.tfmc.srel1, sigma2=1)
profile.rma.mv(MLREMCSC.tfmc.srel1, sigma2=2)
2.3.4.2 Moderator Analyses for the Reliability of Factor 1
2.3.4.2.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.age <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 12.3126 -24.6252 -16.6252 -9.6746 -15.5442
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0081 0.0900 44 no STUDID
## sigma^2.2 0.0081 0.0900 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 445.6209, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.2335, p-val = 0.6315
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7962 0.0280 28.4061 42 <.0001 0.7396 0.8528 ***
## Adults -0.0219 0.0454 -0.4832 42 0.6315 -0.1135 0.0697
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7962 0.0314 21.9 0.731 0.8613
## Adults -0.0219 0.0411 29.2 -0.106 0.0622## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7962 0.0280 0.7396 0.8528 0.5333 1.0591
## 2 0.7743 0.0357 0.7022 0.8463 0.5076 1.04092.3.4.2.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.validity <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 12.3414 -24.6828 -16.6828 -9.7321 -15.6017
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0081 0.0899 44 no STUDID
## sigma^2.2 0.0081 0.0899 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 444.3848, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.2481, p-val = 0.6210
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7949 0.0262 30.3505 42 <.0001 0.7421 0.8478 ***
## Validity -0.0241 0.0484 -0.4981 42 0.6210 -0.1219 0.0736
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7949 0.0269 24.9 0.74 0.8503
## Validity -0.0241 0.0460 20.2 -0.12 0.0717## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7949 0.0262 0.7421 0.8478 0.5329 1.0569
## 2 0.7708 0.0407 0.6886 0.8530 0.5014 1.04022.3.4.2.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.forms <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 39; method: REML)
##
## logLik Deviance AIC BIC AICc
## 11.6184 -23.2369 -13.2369 -5.3193 -11.2369
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0080 0.0895 39 no STUDID
## sigma^2.2 0.0080 0.0895 39 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 36) = 411.8678, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 36) = 1.2318, p-val = 0.3038
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8057 0.0245 32.8943 36 <.0001 0.7560 0.8554 ***
## factor(Forms)B -0.1378 0.0933 -1.4769 36 0.1484 -0.3271 0.0514
## factor(Forms)Both -0.0646 0.1039 -0.6217 36 0.5380 -0.2754 0.1462
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8057 0.0229 28.46 0.759 0.853
## factor(Forms)B -0.1378 0.0490 1.75 -0.380 0.104
## factor(Forms)Both -0.0646 0.2560 1.12 -2.595 2.465## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8057 0.0245 0.7560 0.8554 0.5443 1.0671
## 2 0.6679 0.0900 0.4853 0.8505 0.3529 0.9828
## 3 0.7411 0.1010 0.5363 0.9459 0.4128 1.06942.3.4.2.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.scores <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 43; method: REML)
##
## logLik Deviance AIC BIC AICc
## 13.5092 -27.0184 -19.0184 -12.1641 -17.9073
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0080 0.0895 43 no STUDID
## sigma^2.2 0.0080 0.0895 43 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 41) = 472.8586, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 41) = 0.7608, p-val = 0.3882
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8069 0.0301 26.7806 41 <.0001 0.7460 0.8677 ***
## Scores -0.0384 0.0440 -0.8722 41 0.3882 -0.1272 0.0505
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8069 0.0225 18.7 0.76 0.8540
## Scores -0.0384 0.0451 35.2 -0.13 0.0531## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8069 0.0301 0.7460 0.8677 0.5442 1.0696
## 2 0.7685 0.0321 0.7038 0.8332 0.5049 1.03212.3.4.2.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.lang <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 14.8971 -29.7942 -21.7942 -14.8435 -20.7131
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0075 0.0869 44 no STUDID
## sigma^2.2 0.0075 0.0869 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 403.6745, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 5.4680, p-val = 0.0242
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6800 0.0512 13.2710 42 <.0001 0.5766 0.7834 ***
## LanguageEnglish 0.1318 0.0564 2.3384 42 0.0242 0.0181 0.2456 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.680 0.103 5.76 0.427 0.934
## LanguageEnglish 0.132 0.104 8.37 -0.105 0.369## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6800 0.0512 0.5766 0.7834 0.4114 0.9486
## 2 0.8119 0.0235 0.7644 0.8593 0.5595 1.06432.3.4.2.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel1.cult <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 14.8013 -29.6027 -21.6027 -14.6520 -20.5216
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0074 0.0862 44 no STUDID
## sigma^2.2 0.0074 0.0862 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 432.4309, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 5.2923, p-val = 0.0265
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7009 0.0439 15.9780 42 <.0001 0.6123 0.7894 ***
## CultureWestern 0.1153 0.0501 2.3005 42 0.0265 0.0142 0.2165 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.701 0.0746 7.86 0.5284 0.873
## CultureWestern 0.115 0.0763 13.05 -0.0495 0.280## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel1.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7009 0.0439 0.6123 0.7894 0.4395 0.9623
## 2 0.8162 0.0243 0.7672 0.8651 0.5654 1.06692.3.4.2.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfmc.srel1.female <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 42; method: REML)
##
## logLik Deviance AIC BIC AICc
## 10.8646 -21.7292 -13.7292 -6.9737 -12.5863
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0083 0.0910 42 no STUDID
## sigma^2.2 0.0083 0.0910 42 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 40) = 405.4372, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 40) = 0.3121, p-val = 0.5795
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.8234 0.0738 11.1603 40 <.0001 0.6743
## asin(sqrt(PropFemale/100)) -0.0449 0.0804 -0.5586 40 0.5795 -0.2075
## ci.ub
## intrcpt 0.9725 ***
## asin(sqrt(PropFemale/100)) 0.1176
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel1.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8234 0.0599 6.20 0.678 0.969
## asin(sqrt(PropFemale/100)) -0.0449 0.0595 5.01 -0.198 0.1082.3.4.3 Moderator Analyses for the Reliability of Factor 2
2.3.4.3.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.age <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 12.9805 -25.9610 -17.9610 -11.0103 -16.8799
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0075 0.0864 44 no STUDID
## sigma^2.2 0.0075 0.0864 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 506.5490, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.0598, p-val = 0.8080
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6509 0.0299 21.7658 42 <.0001 0.5905 0.7112 ***
## Adults -0.0117 0.0477 -0.2445 42 0.8080 -0.1080 0.0846
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6509 0.0265 19.4 0.596 0.7062
## Adults -0.0117 0.0505 28.0 -0.115 0.0917## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6509 0.0299 0.5905 0.7112 0.3970 0.9047
## 2 0.6392 0.0372 0.5641 0.7142 0.3814 0.89702.3.4.3.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.validity <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 13.7675 -27.5349 -19.5349 -12.5842 -18.4538
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0068 0.0827 44 no STUDID
## sigma^2.2 0.0068 0.0827 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 456.5902, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.5625, p-val = 0.2182
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6638 0.0261 25.4404 42 <.0001 0.6111 0.7164 ***
## Validity -0.0645 0.0516 -1.2500 42 0.2182 -0.1688 0.0397
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6638 0.0269 23.5 0.608 0.7192
## Validity -0.0645 0.0481 15.5 -0.167 0.0376## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6638 0.0261 0.6111 0.7164 0.4219 0.9056
## 2 0.5992 0.0446 0.5093 0.6891 0.3466 0.85182.3.4.3.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.forms <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 39; method: REML)
##
## logLik Deviance AIC BIC AICc
## 13.3436 -26.6872 -16.6872 -8.7696 -14.6872
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0064 0.0800 39 no STUDID
## sigma^2.2 0.0064 0.0800 39 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 36) = 475.2289, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 36) = 0.1855, p-val = 0.8315
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6590 0.0246 26.7514 36 <.0001 0.6090 0.7089 ***
## factor(Forms)B -0.0004 0.0759 -0.0057 36 0.9955 -0.1544 0.1536
## factor(Forms)Both -0.0655 0.1078 -0.6077 36 0.5472 -0.2841 0.1531
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.658950 0.0250 24.52 0.607 0.711
## factor(Forms)B -0.000432 0.0396 3.35 -0.119 0.118
## factor(Forms)Both -0.065498 0.1675 1.11 -1.745 1.614## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6590 0.0246 0.6090 0.7089 0.4241 0.8938
## 2 0.6585 0.0718 0.5129 0.8042 0.3868 0.9303
## 3 0.5935 0.1049 0.3807 0.8062 0.2805 0.90642.3.4.3.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.scores <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 43; method: REML)
##
## logLik Deviance AIC BIC AICc
## 13.4893 -26.9786 -18.9786 -12.1244 -17.8675
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0068 0.0826 43 no STUDID
## sigma^2.2 0.0068 0.0826 43 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 41) = 277.9042, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 41) = 1.3530, p-val = 0.2515
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6229 0.0308 20.1980 41 <.0001 0.5607 0.6852 ***
## Scores 0.0525 0.0452 1.1632 41 0.2515 -0.0387 0.1437
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6229 0.0264 16.5 0.567 0.679
## Scores 0.0525 0.0459 31.8 -0.041 0.146## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6229 0.0308 0.5607 0.6852 0.3789 0.8670
## 2 0.6755 0.0330 0.6088 0.7421 0.4303 0.92062.3.4.3.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.lang <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 13.2730 -26.5460 -18.5460 -11.5953 -17.4649
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0069 0.0830 44 no STUDID
## sigma^2.2 0.0069 0.0830 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 172.5519, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.8218, p-val = 0.3698
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6875 0.0499 13.7895 42 <.0001 0.5869 0.7881 ***
## LanguageEnglish -0.0507 0.0559 -0.9066 42 0.3698 -0.1635 0.0622
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6875 0.0773 5.86 0.497 0.878
## LanguageEnglish -0.0507 0.0800 9.14 -0.231 0.130## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6875 0.0499 0.5869 0.7881 0.4301 0.9449
## 2 0.6368 0.0253 0.5857 0.6879 0.3944 0.87922.3.4.3.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srel2.cult <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 12.9792 -25.9583 -17.9583 -11.0076 -16.8772
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0076 0.0869 44 no STUDID
## sigma^2.2 0.0076 0.0869 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 234.5904, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.1356, p-val = 0.7145
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6320 0.0451 14.0155 42 <.0001 0.5410 0.7230 ***
## CultureWestern 0.0194 0.0528 0.3683 42 0.7145 -0.0870 0.1259
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.6320 0.0732 8.41 0.465 0.799
## CultureWestern 0.0194 0.0754 15.05 -0.141 0.180## Predicted subgroup values
predict(MLMEMCSC.tfmc.srel2.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.6320 0.0451 0.5410 0.7230 0.3678 0.8961
## 2 0.6514 0.0274 0.5961 0.7067 0.3973 0.90552.3.4.3.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfmc.srel2.female <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 42; method: REML)
##
## logLik Deviance AIC BIC AICc
## 11.9099 -23.8198 -15.8198 -9.0643 -14.6770
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0077 0.0876 42 no STUDID
## sigma^2.2 0.0077 0.0876 42 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 40) = 502.6352, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 40) = 1.6089, p-val = 0.2120
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.5417 0.0825 6.5632 40 <.0001 0.3749
## asin(sqrt(PropFemale/100)) 0.1115 0.0879 1.2684 40 0.2120 -0.0661
## ci.ub
## intrcpt 0.7085 ***
## asin(sqrt(PropFemale/100)) 0.2891
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srel2.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.542 0.0639 9.45 0.3982 0.685
## asin(sqrt(PropFemale/100)) 0.111 0.0551 5.96 -0.0236 0.2462.3.4.4 Moderator Analyses for the Composite Reliability
2.3.4.4.1 MODERATOR: Age groups
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.age <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Adults)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.age, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 44.0422 -88.0844 -80.0844 -73.1337 -79.0033
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0022 0.0473 44 no STUDID
## sigma^2.2 0.0022 0.0473 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 563.1695, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.6661, p-val = 0.2038
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8365 0.0150 55.8865 42 <.0001 0.8063 0.8667 ***
## Adults -0.0310 0.0240 -1.2908 42 0.2038 -0.0794 0.0174
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.age, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.836 0.0128 22.6 0.8100 0.8630
## Adults -0.031 0.0258 32.4 -0.0834 0.0215## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.age, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8365 0.0150 0.8063 0.8667 0.6982 0.9748
## 2 0.8055 0.0187 0.7677 0.8433 0.6654 0.94572.3.4.4.2 MODERATOR: Validity evidence
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.validity <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Validity)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.validity, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 43.8864 -87.7727 -79.7727 -72.8220 -78.6916
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0022 0.0469 44 no STUDID
## sigma^2.2 0.0022 0.0469 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 491.5387, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.2937, p-val = 0.2618
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8328 0.0137 60.9188 42 <.0001 0.8052 0.8604 ***
## Validity -0.0294 0.0259 -1.1374 42 0.2618 -0.0817 0.0228
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.validity, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8328 0.0136 26.3 0.8049 0.8607
## Validity -0.0294 0.0260 21.3 -0.0835 0.0247## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.validity, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8328 0.0137 0.8052 0.8604 0.6962 0.9693
## 2 0.8033 0.0220 0.7590 0.8477 0.6624 0.94432.3.4.4.3 MODERATOR: Test forms
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.forms <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ factor(Forms))
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.forms, digits=4)##
## Multivariate Meta-Analysis Model (k = 39; method: REML)
##
## logLik Deviance AIC BIC AICc
## 41.9885 -83.9771 -73.9771 -66.0595 -71.9771
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0017 0.0408 39 no STUDID
## sigma^2.2 0.0017 0.0408 39 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 36) = 455.6054, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## F(df1 = 2, df2 = 36) = 1.8254, p-val = 0.1758
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8432 0.0118 71.4886 36 <.0001 0.8192 0.8671 ***
## factor(Forms)B -0.0651 0.0354 -1.8380 36 0.0743 -0.1369 0.0067 .
## factor(Forms)Both -0.0345 0.0524 -0.6586 36 0.5144 -0.1408 0.0718
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.forms, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.8432 0.0105 27.47 0.822 0.8647
## factor(Forms)B -0.0651 0.0290 4.65 -0.141 0.0111
## factor(Forms)Both -0.0345 0.1701 1.11 -1.751 1.6824## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.forms, newmods = cbind(c(0,1,0),
c(0,0,1)))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8432 0.0118 0.8192 0.8671 0.7238 0.9625
## 2 0.7781 0.0334 0.7104 0.8458 0.6430 0.9132
## 3 0.8086 0.0511 0.7051 0.9122 0.6525 0.96482.3.4.4.4 MODERATOR: Test scores
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.scores <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ Scores)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.scores, digits=4)##
## Multivariate Meta-Analysis Model (k = 43; method: REML)
##
## logLik Deviance AIC BIC AICc
## 44.2699 -88.5399 -80.5399 -73.6856 -79.4288
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0021 0.0455 43 no STUDID
## sigma^2.2 0.0021 0.0455 43 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 41) = 471.8816, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 41) = 0.0428, p-val = 0.8372
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8305 0.0157 52.8363 41 <.0001 0.7988 0.8623 ***
## Scores -0.0047 0.0229 -0.2068 41 0.8372 -0.0509 0.0415
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.scores, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.83052 0.0145 19.2 0.8003 0.861
## Scores -0.00473 0.0230 36.5 -0.0514 0.042## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.scores, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.8305 0.0157 0.7988 0.8623 0.6967 0.9643
## 2 0.8258 0.0166 0.7922 0.8594 0.6916 0.96002.3.4.4.5 MODERATOR: Language English
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.lang <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ LanguageEnglish)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.lang, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 42.3185 -84.6369 -76.6369 -69.6863 -75.5559
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0024 0.0489 44 no STUDID
## sigma^2.2 0.0024 0.0489 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 452.7653, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.7257, p-val = 0.3991
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7870 0.0269 29.2563 42 <.0001 0.7327 0.8412 ***
## LanguageEnglish 0.0256 0.0301 0.8519 42 0.3991 -0.0351 0.0863
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.lang, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7870 0.0434 6.62 0.6831 0.891
## LanguageEnglish 0.0256 0.0447 10.20 -0.0738 0.125## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.lang, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7870 0.0269 0.7327 0.8412 0.6373 0.9366
## 2 0.8126 0.0134 0.7854 0.8397 0.6705 0.95462.3.4.4.6 MODERATOR: Western Culture
## Three-level MEM
## Model specification and estimation
MLMEMCSC.tfmc.srelcr.cult <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ CultureWestern)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.cult, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 43.3295 -86.6589 -78.6589 -71.7083 -77.5779
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0023 0.0475 44 no STUDID
## sigma^2.2 0.0023 0.0475 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 515.1130, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 2.7482, p-val = 0.1048
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7756 0.0228 34.0287 42 <.0001 0.7296 0.8216 ***
## CultureWestern 0.0441 0.0266 1.6578 42 0.1048 -0.0096 0.0978
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.cult, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7756 0.0345 9.34 0.698 0.853
## CultureWestern 0.0441 0.0360 16.62 -0.032 0.120## Predicted subgroup values
predict(MLMEMCSC.tfmc.srelcr.cult, newmods = c(0,1))##
## pred se ci.lb ci.ub pi.lb pi.ub
## 1 0.7756 0.0228 0.7296 0.8216 0.6325 0.9187
## 2 0.8197 0.0137 0.7920 0.8473 0.6814 0.95802.3.4.4.7 MODERATOR: Proportion of women in the sample
## Three-level MEM
## Model specification and estimation
## Note: The proportion is arcsine-square-root transformed to stabilize the distribution
## and approximate normality.
## Source: Schwarzer et al. (2019), https://doi.org/10.1002/jrsm.1348
MLMEMCSC.tfmc.srelcr.female <- rma.mv(SRELCR,
SRELCR.vg,
random = list(~ 1 | STUDID/ESID),
data = srel.meta,
method = "REML",
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ asin(sqrt(PropFemale/100)))
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.female, digits=4)##
## Multivariate Meta-Analysis Model (k = 42; method: REML)
##
## logLik Deviance AIC BIC AICc
## 39.7329 -79.4658 -71.4658 -64.7102 -70.3229
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0025 0.0496 42 no STUDID
## sigma^2.2 0.0025 0.0496 42 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 40) = 725.7394, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 40) = 0.5529, p-val = 0.4615
##
## Model Results:
##
## estimate se tval df pval ci.lb
## intrcpt 0.7733 0.0431 17.9546 40 <.0001 0.6863
## asin(sqrt(PropFemale/100)) 0.0348 0.0469 0.7436 40 0.4615 -0.0599
## ci.ub
## intrcpt 0.8603 ***
## asin(sqrt(PropFemale/100)) 0.1295
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
clubSandwich::conf_int(MLMEMCSC.tfmc.srelcr.female, vcov = "CR2")## Coef. Estimate SE d.f. Lower 95% CI Upper 95% CI
## intrcpt 0.7733 0.0321 8.51 0.7001 0.847
## asin(sqrt(PropFemale/100)) 0.0348 0.0282 6.16 -0.0337 0.1032.3.4.5 Publication Bias and Infuential Effect Sizes
## Reliability of factor 1
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfmc.srel1.pet <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SREL1C.vg))
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 22.1634 -44.3268 -36.3268 -29.3761 -35.2457
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0033 0.0577 44 no STUDID
## sigma^2.2 0.0033 0.0577 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 332.6110, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 23.5636, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8641 0.0197 43.8576 42 <.0001 0.8243 0.9039 ***
## sqrt(SREL1C.vg) -1.6585 0.3417 -4.8542 42 <.0001 -2.3480 -0.9690 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel1.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.864 0.0153 56.39 27.6 < 0.001 ***
## sqrt(SREL1C.vg) -1.659 0.4536 -3.66 14.0 0.00259 **## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfmc.srel1.fpt <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 12.8164 -25.6328 -17.6328 -10.6821 -16.5517
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0079 0.0891 44 no STUDID
## sigma^2.2 0.0079 0.0891 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 474.8846, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.4593, p-val = 0.2338
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7670 0.0280 27.4155 42 <.0001 0.7106 0.8235 ***
## N 0.0001 0.0001 1.2080 42 0.2338 -0.0000 0.0002
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel1.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.7670400 0.0304026 25.23 29.11 <0.001 ***
## N 0.0000626 0.0000384 1.63 5.87 0.155## Contour-enhanced funnel plot
funnel(MLREMCSC.tfmc.srel1,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfmc.srel1.peese <- rma.mv(SREL1C,
SREL1C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SREL1C.vg)
## Summarize the results
summary(MLMEMCSC.tfmc.srel1.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 15.4951 -30.9902 -22.9902 -16.0395 -21.9091
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0073 0.0852 44 no STUDID
## sigma^2.2 0.0073 0.0852 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 461.4327, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 2.7641, p-val = 0.1038
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.7946 0.0212 37.4139 42 <.0001 0.7517 0.8374 ***
## SREL1C.vg -1.0292 0.6190 -1.6626 42 0.1038 -2.2784 0.2201
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel1.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.795 0.0211 37.7 35.97 <0.001 ***
## SREL1C.vg -1.029 0.8577 -1.2 2.04 0.351## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfmc.srel1 <- influence.rma.uni(rma(SREL1C,
SREL1C.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfmc.srel1)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfmc.srel1)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 -0.1491 0.0105 0.0001 1.0786 0.0165 448.1998 0.0294 2.9408 0.0105
## 2 0.3794 0.1031 0.0112 1.0800 0.0165 469.5624 0.0297 2.9732 0.1034
## 3 -0.8226 -0.1077 0.0116 1.0195 0.0156 471.3981 0.0173 1.7278 -0.1076
## 4 -2.4604 -0.4498 0.1828 0.8663 0.0130 456.7168 0.0175 1.7480 -0.4631
## 5 0.0017 0.0380 0.0015 1.0798 0.0165 466.8167 0.0287 2.8657 0.0381
## 6 0.5186 0.1180 0.0145 1.0693 0.0163 475.3383 0.0272 2.7238 0.1180
## 7 1.1023 0.1927 0.0373 1.0329 0.0157 470.7227 0.0289 2.8926 0.1927
## 8 0.2268 0.0295 0.0009 1.0200 0.0158 475.4631 0.0070 0.6970 0.0294
## 9 0.2139 0.0663 0.0046 1.0676 0.0164 474.7501 0.0240 2.4039 0.0662
## 10 0.7260 0.1477 0.0225 1.0610 0.0162 475.3757 0.0281 2.8056 0.1478
## 11 -0.9106 -0.0422 0.0018 0.9998 0.0156 474.2282 0.0020 0.1970 -0.0422
## 12 0.1374 0.0527 0.0029 1.0650 0.0163 474.6321 0.0230 2.2999 0.0526
## 13 1.1435 0.1947 0.0379 1.0285 0.0156 471.6388 0.0283 2.8302 0.1947
## 14 0.9693 0.1747 0.0310 1.0432 0.0159 474.2250 0.0276 2.7577 0.1748
## 15 -0.8904 -0.1461 0.0213 1.0250 0.0156 460.0003 0.0258 2.5801 -0.1461
## 16 1.2081 0.2049 0.0416 1.0226 0.0155 458.7477 0.0296 2.9637 0.2049
## 17 -1.6799 -0.3040 0.0871 0.9442 0.0143 461.1057 0.0204 2.0424 -0.3068
## 18 -0.3866 -0.0371 0.0014 1.0600 0.0162 468.6107 0.0257 2.5704 -0.0371
## 19 0.1370 0.0637 0.0043 1.0842 0.0165 402.7377 0.0300 3.0022 0.0639
## 20 -0.2564 -0.0038 0.0000 1.0009 0.0156 475.3707 0.0003 0.0284 -0.0038
## 21 -6.0591 -1.4053 1.3019 0.4751 0.0064 408.3660 0.0175 1.7497 -1.6658 *
## 22 0.7204 0.1524 0.0241 1.0652 0.0162 474.3758 0.0301 3.0055 0.1527
## 23 0.2917 0.0896 0.0085 1.0826 0.0165 450.1920 0.0300 2.9972 0.0899
## 24 0.8075 0.1635 0.0276 1.0594 0.0161 465.0686 0.0301 3.0081 0.1638
## 25 0.7444 0.1555 0.0250 1.0637 0.0162 473.3140 0.0301 3.0052 0.1558
## 26 -0.5985 -0.0570 0.0033 1.0187 0.0157 473.9199 0.0109 1.0903 -0.0569
## 27 -0.0873 0.0217 0.0005 1.0783 0.0165 464.1942 0.0287 2.8667 0.0217
## 28 -0.3668 -0.0289 0.0008 1.0280 0.0159 474.4211 0.0123 1.2256 -0.0288
## 29 -0.2503 -0.0088 0.0001 1.0038 0.0157 475.2964 0.0017 0.1740 -0.0088
## 30 -1.0886 -0.0576 0.0033 0.9980 0.0155 473.6980 0.0025 0.2473 -0.0577
## 31 0.6994 0.1492 0.0231 1.0662 0.0162 475.3215 0.0299 2.9879 0.1495
## 32 -0.9637 -0.1676 0.0279 1.0191 0.0155 451.3708 0.0270 2.7040 -0.1676
## 33 -0.9063 -0.1400 0.0196 1.0202 0.0156 466.8557 0.0226 2.2588 -0.1401
## 34 0.6188 0.1278 0.0169 1.0618 0.0162 475.4745 0.0258 2.5809 0.1278
## 35 0.3530 0.0913 0.0087 1.0710 0.0164 474.9567 0.0260 2.6002 0.0912
## 36 -0.1180 0.0156 0.0003 1.0766 0.0164 465.6887 0.0283 2.8318 0.0157
## 37 -0.2498 -0.0121 0.0002 1.0544 0.0162 473.2014 0.0214 2.1373 -0.0121
## 38 0.5430 0.1261 0.0166 1.0728 0.0164 475.0811 0.0292 2.9181 0.1264
## 39 0.7041 0.1485 0.0229 1.0649 0.0162 475.3886 0.0294 2.9389 0.1488
## 40 -1.0727 -0.0414 0.0017 0.9983 0.0156 473.9407 0.0013 0.1323 -0.0415
## 41 0.3507 0.0987 0.0103 1.0808 0.0165 467.5392 0.0298 2.9769 0.0989
## 42 0.1517 0.0642 0.0043 1.0804 0.0165 470.5219 0.0286 2.8616 0.0643
## 43 -1.0736 -0.1929 0.0365 1.0072 0.0153 453.0157 0.0264 2.6405 -0.1930
## 44 1.0030 0.1861 0.0351 1.0434 0.0158 370.6931 0.0301 3.0107 0.1862## Reliability of factor 2
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfmc.srel2.pet <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SREL2C.vg))
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 16.5280 -33.0561 -25.0561 -18.1054 -23.9750
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0062 0.0788 44 no STUDID
## sigma^2.2 0.0062 0.0788 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 403.2151, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 5.0536, p-val = 0.0299
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6958 0.0303 22.9903 42 <.0001 0.6348 0.7569 ***
## sqrt(SREL2C.vg) -0.7488 0.3331 -2.2480 42 0.0299 -1.4211 -0.0766 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel2.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.696 0.0249 28.00 26.3 < 0.001 ***
## sqrt(SREL2C.vg) -0.749 0.2227 -3.36 15.8 0.00402 **## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfmc.srel2.fpt <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 12.8278 -25.6556 -17.6556 -10.7049 -16.5745
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0075 0.0864 44 no STUDID
## sigma^2.2 0.0075 0.0864 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 509.9073, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 0.0253, p-val = 0.8744
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6495 0.0309 21.0331 42 <.0001 0.5872 0.7118 ***
## N -0.0000 0.0001 -0.1590 42 0.8744 -0.0001 0.0001
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel2.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.64950296 0.0287488 22.592 27.36 <0.001 ***
## N -0.00000847 0.0000437 -0.194 5.66 0.853## Contour-enhanced funnel plot
funnel(MLREMCSC.tfmc.srel2,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfmc.srel2.peese <- rma.mv(SREL2C,
SREL2C.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SREL2C.vg)
## Summarize the results
summary(MLMEMCSC.tfmc.srel2.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 15.3971 -30.7942 -22.7942 -15.8435 -21.7131
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0069 0.0831 44 no STUDID
## sigma^2.2 0.0069 0.0831 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 499.5432, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.7581, p-val = 0.1920
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.6539 0.0232 28.2347 42 <.0001 0.6072 0.7007 ***
## SREL2C.vg -0.7801 0.5883 -1.3259 42 0.1920 -1.9673 0.4072
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srel2.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.654 0.0229 28.54 32.3 <0.001 ***
## SREL2C.vg -0.780 0.2355 -3.31 4.0 0.0297 *## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfmc.srel2 <- influence.rma.uni(rma(SREL2C,
SREL2C.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfmc.srel2)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfmc.srel2)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 -0.6465 -0.1090 0.0121 1.0530 0.0146 490.2440 0.0335 3.3459 -0.1091
## 2 -0.3051 -0.0358 0.0013 1.0696 0.0149 499.6492 0.0334 3.3375 -0.0359
## 3 -0.8412 -0.0750 0.0056 1.0078 0.0143 509.3284 0.0079 0.7864 -0.0750
## 4 -1.5373 -0.2294 0.0514 0.9815 0.0136 503.0197 0.0181 1.8136 -0.2310
## 5 1.5136 0.2667 0.0675 0.9944 0.0135 491.2307 0.0357 3.5660 0.2653
## 6 -0.1753 -0.0109 0.0001 1.0545 0.0148 509.1996 0.0248 2.4821 -0.0108
## 7 0.8628 0.1498 0.0227 1.0413 0.0145 510.9004 0.0259 2.5941 0.1497
## 8 -0.2272 -0.0065 0.0000 1.0017 0.0143 510.9724 0.0009 0.0944 -0.0065
## 9 1.0905 0.1821 0.0332 1.0295 0.0143 510.3442 0.0268 2.6784 0.1821
## 10 -0.5604 -0.0585 0.0034 1.0208 0.0144 509.6114 0.0125 1.2540 -0.0584
## 11 0.1256 0.0205 0.0004 1.0229 0.0145 511.0025 0.0100 0.9987 0.0205
## 12 1.3460 0.2205 0.0478 1.0119 0.0140 509.0279 0.0288 2.8752 0.2206
## 13 0.4666 0.0816 0.0068 1.0439 0.0147 511.0787 0.0204 2.0361 0.0813
## 14 -0.9660 -0.0484 0.0023 1.0007 0.0143 509.6733 0.0024 0.2397 -0.0484
## 15 -1.8019 -0.3605 0.1198 0.9448 0.0128 487.9676 0.0280 2.7981 -0.3612
## 16 0.5874 0.1123 0.0129 1.0531 0.0147 511.0890 0.0264 2.6379 0.1122
## 17 -2.6419 -0.4780 0.2046 0.8762 0.0117 488.9286 0.0215 2.1538 -0.4883 *
## 18 0.4366 0.0966 0.0096 1.0647 0.0149 510.8009 0.0300 3.0017 0.0966
## 19 -0.0960 0.0069 0.0001 1.0810 0.0151 463.8898 0.0361 3.6106 0.0069
## 20 0.3101 0.0178 0.0003 1.0060 0.0143 511.0797 0.0025 0.2524 0.0178
## 21 1.4809 0.2281 0.0508 1.0007 0.0138 508.6102 0.0267 2.6738 0.2284
## 22 0.0694 0.0392 0.0016 1.0817 0.0151 494.5378 0.0357 3.5679 0.0394
## 23 -1.4077 -0.3034 0.0865 0.9834 0.0134 434.9514 0.0346 3.4623 -0.3018
## 24 0.2738 0.0778 0.0064 1.0812 0.0151 496.6787 0.0361 3.6108 0.0782
## 25 1.7801 0.3018 0.0825 0.9601 0.0129 222.9964 0.0365 3.6528 0.2987
## 26 -0.5461 -0.0133 0.0002 1.0005 0.0143 510.6799 0.0006 0.0586 -0.0133
## 27 -3.0315 -0.5529 0.2666 0.8388 0.0111 484.7432 0.0211 2.1112 -0.5695 *
## 28 -0.4158 -0.0349 0.0012 1.0164 0.0144 510.3019 0.0088 0.8847 -0.0348
## 29 -0.3735 -0.0092 0.0001 1.0009 0.0143 510.8699 0.0006 0.0632 -0.0092
## 30 0.1433 0.0352 0.0013 1.0447 0.0147 510.7664 0.0194 1.9400 0.0351
## 31 0.2242 0.0667 0.0047 1.0781 0.0150 507.2348 0.0344 3.4365 0.0670
## 32 -0.3538 -0.0452 0.0021 1.0644 0.0148 503.0507 0.0317 3.1714 -0.0453
## 33 -0.0209 0.0162 0.0003 1.0605 0.0149 509.5678 0.0265 2.6542 0.0161
## 34 -0.2507 -0.0121 0.0001 1.0060 0.0143 510.8612 0.0031 0.3057 -0.0121
## 35 -1.0840 -0.1001 0.0100 1.0026 0.0142 508.5161 0.0080 0.7962 -0.1003
## 36 0.2454 0.0678 0.0048 1.0731 0.0150 509.4484 0.0322 3.2218 0.0680
## 37 0.1171 0.0335 0.0011 1.0487 0.0148 510.6328 0.0211 2.1110 0.0333
## 38 0.4241 0.0983 0.0100 1.0696 0.0149 510.5123 0.0322 3.2181 0.0985
## 39 0.5097 0.1108 0.0127 1.0653 0.0149 510.8912 0.0313 3.1327 0.1109
## 40 -0.0950 -0.0032 0.0000 1.0082 0.0144 510.9722 0.0038 0.3793 -0.0031
## 41 -0.0286 0.0194 0.0004 1.0783 0.0150 501.8721 0.0344 3.4436 0.0195
## 42 1.2277 0.2266 0.0506 1.0245 0.0141 506.3483 0.0348 3.4833 0.2263
## 43 0.4947 0.1114 0.0129 1.0690 0.0149 510.6788 0.0330 3.2973 0.1117
## 44 -0.7911 -0.1293 0.0168 1.0353 0.0144 502.8488 0.0277 2.7670 -0.1293## Reliability of factor 1
## Egger's regression test (PET)
## Details: https://wviechtb.github.io/metafor/reference/regtest.html
## Three-level MEM
MLMEMCSC.tfmc.srelcr.pet <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ sqrt(SRELCRC.vg))
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.pet, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 52.6645 -105.3289 -97.3289 -90.3782 -96.2478
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0014 0.0368 44 no STUDID
## sigma^2.2 0.0014 0.0368 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 365.8065, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 20.5494, p-val < .0001
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8817 0.0151 58.4299 42 <.0001 0.8513 0.9122 ***
## sqrt(SRELCRC.vg) -1.8804 0.4148 -4.5331 42 <.0001 -2.7176 -1.0433 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srelcr.pet, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.882 0.0123 71.90 21.8 <0.001 ***
## sqrt(SRELCRC.vg) -1.880 0.3783 -4.97 17.8 <0.001 ***## Result: The standard errors are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Funnel plot test
## Three-level MEM
MLMEMCSC.tfmc.srelcr.fpt <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ N)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.fpt, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 44.0901 -88.1802 -80.1802 -73.2295 -79.0991
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0022 0.0472 44 no STUDID
## sigma^2.2 0.0022 0.0472 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 558.1666, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 1.9655, p-val = 0.1683
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8112 0.0150 54.0336 42 <.0001 0.7809 0.8415 ***
## N 0.0000 0.0000 1.4020 42 0.1683 -0.0000 0.0001
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srelcr.fpt, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.8112186 0.0153491 52.85 32.49 <0.001 ***
## N 0.0000388 0.0000227 1.71 5.84 0.14## Contour-enhanced funnel plot
funnel(MLREMCSC.tfmc.srelcr,
main="Standard Error",
level = c(90, 95, 99),
shade = c("white", "gray55", "gray75"),
legend = TRUE)
## Result: No evidence for publication or selection bias.
## Precision-Effect Estimate with SE (PEESE)
## Three-level MEM
MLMEMCSC.tfmc.srelcr.peese <- rma.mv(SRELCRC,
SRELCRC.vg,
random = list(~ 1 | STUDID/ESID),
method = "REML",
data = srel.meta,
slab = paste(Reference),
tdist = TRUE,
test = "t",
mods =~ SRELCRC.vg)
## Summarize the results
summary(MLMEMCSC.tfmc.srelcr.peese, digits=4)##
## Multivariate Meta-Analysis Model (k = 44; method: REML)
##
## logLik Deviance AIC BIC AICc
## 48.1415 -96.2830 -88.2830 -81.3323 -87.2019
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0018 0.0423 44 no STUDID
## sigma^2.2 0.0018 0.0423 44 no STUDID/ESID
##
## Test for Residual Heterogeneity:
## QE(df = 42) = 500.1417, p-val < .0001
##
## Test of Moderators (coefficient 2):
## F(df1 = 1, df2 = 42) = 7.9483, p-val = 0.0073
##
## Model Results:
##
## estimate se tval df pval ci.lb ci.ub
## intrcpt 0.8378 0.0114 73.2762 42 <.0001 0.8148 0.8609 ***
## SRELCRC.vg -7.9524 2.8207 -2.8193 42 0.0073 -13.6449 -2.2599 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1## RVE standard errors
coef_test(MLMEMCSC.tfmc.srelcr.peese, vcov = "CR2")## Coef. Estimate SE t-stat d.f. (Satt) p-val (Satt) Sig.
## intrcpt 0.838 0.0137 61.28 30.09 <0.001 ***
## SRELCRC.vg -7.952 6.8146 -1.17 1.81 0.374## Result: The sampling variances are significantly moderating the effect size.
## This can be considered evidence for publication or selection bias.
## Influential effect sizes
## Analyses based on the REM with variation between effect sizes
## Plots indicating potential influential effect sizes
inf.tfmc.srelcr <- influence.rma.uni(rma(SRELCRC,
SRELCRC.vg,
data = srel.meta,
method = "REML"))
plot.infl.rma.uni(inf.tfmc.srelcr)
# Increase max print options to show all effect sizes
options(max.print = 1000000)
print(inf.tfmc.srelcr)##
## rstudent dffits cook.d cov.r tau2.del QE.del hat weight dfbs inf
## 1 -0.2790 -0.0209 0.0005 1.0626 0.0047 536.3614 0.0292 2.9159 -0.0209
## 2 0.2683 0.0783 0.0064 1.0688 0.0047 555.4840 0.0293 2.9318 0.0786
## 3 -1.0170 -0.1087 0.0118 1.0045 0.0044 559.8015 0.0103 1.0307 -0.1089
## 4 -2.8945 -0.4568 0.1936 0.8835 0.0038 542.4625 0.0142 1.4237 -0.4713
## 5 1.4316 0.2310 0.0517 1.0048 0.0043 512.2562 0.0304 3.0360 0.2303
## 6 -0.1478 0.0013 0.0000 1.0516 0.0046 560.1840 0.0228 2.2752 0.0013
## 7 1.2278 0.1950 0.0378 1.0210 0.0044 561.9450 0.0258 2.5775 0.1950
## 8 0.2914 0.0626 0.0040 1.0451 0.0046 562.8130 0.0190 1.9039 0.0624
## 9 0.7058 0.1256 0.0161 1.0444 0.0046 563.1813 0.0226 2.2619 0.1255
## 10 0.0503 0.0300 0.0009 1.0473 0.0046 562.0473 0.0199 1.9895 0.0299
## 11 -0.9766 -0.1201 0.0144 1.0087 0.0044 558.8916 0.0138 1.3842 -0.1203
## 12 1.1740 0.1889 0.0356 1.0247 0.0045 562.2394 0.0256 2.5589 0.1889
## 13 1.0463 0.1688 0.0287 1.0308 0.0045 562.7863 0.0237 2.3715 0.1687
## 14 0.3089 0.0556 0.0031 1.0354 0.0046 563.0308 0.0149 1.4899 0.0554
## 15 -2.0752 -0.3966 0.1445 0.9228 0.0040 540.0763 0.0212 2.1238 -0.3999
## 16 1.4240 0.2211 0.0477 1.0059 0.0044 559.3101 0.0278 2.7793 0.2208
## 17 -2.4604 -0.5706 0.2735 0.8611 0.0036 513.1865 0.0252 2.5187 -0.5708 *
## 18 0.2425 0.0693 0.0050 1.0625 0.0047 561.0124 0.0264 2.6431 0.0694
## 19 0.2816 0.0822 0.0071 1.0708 0.0047 521.7865 0.0303 3.0317 0.0825
## 20 -0.1004 0.0011 0.0000 1.0238 0.0045 562.6497 0.0105 1.0472 0.0011
## 21 -0.0015 0.0294 0.0009 1.0632 0.0047 557.9764 0.0269 2.6864 0.0295
## 22 0.8834 0.1703 0.0296 1.0485 0.0046 562.9566 0.0303 3.0314 0.1706
## 23 -0.6876 -0.1086 0.0120 1.0438 0.0045 486.8362 0.0296 2.9634 -0.1087
## 24 0.6911 0.1446 0.0216 1.0587 0.0046 560.5106 0.0303 3.0299 0.1450
## 25 1.5623 0.2429 0.0562 0.9915 0.0043 377.8937 0.0305 3.0472 0.2418
## 26 -0.9154 -0.0349 0.0012 0.9999 0.0045 561.9375 0.0014 0.1352 -0.0349
## 27 -1.9307 -0.3812 0.1335 0.9326 0.0040 537.0880 0.0230 2.2972 -0.3828
## 28 -0.0587 0.0071 0.0001 1.0283 0.0046 562.6029 0.0122 1.2213 0.0071
## 29 -1.1790 -0.1471 0.0215 0.9995 0.0044 558.0372 0.0131 1.3097 -0.1476
## 30 -1.5006 -0.1553 0.0240 0.9880 0.0044 558.0289 0.0085 0.8489 -0.1564
## 31 0.8845 0.1693 0.0292 1.0479 0.0046 563.0735 0.0299 2.9889 0.1695
## 32 -0.4373 -0.0529 0.0029 1.0555 0.0046 538.1180 0.0286 2.8635 -0.0530
## 33 -0.6891 -0.0920 0.0085 1.0290 0.0045 557.6440 0.0202 2.0235 -0.0919
## 34 -0.0537 0.0157 0.0003 1.0500 0.0046 561.2497 0.0214 2.1438 0.0157
## 35 -0.1006 0.0074 0.0001 1.0466 0.0046 561.3564 0.0202 2.0226 0.0074
## 36 0.2178 0.0684 0.0049 1.0669 0.0047 558.3733 0.0283 2.8313 0.0686
## 37 -0.2575 -0.0176 0.0003 1.0363 0.0046 561.4458 0.0169 1.6873 -0.0176
## 38 0.4743 0.1075 0.0120 1.0616 0.0047 561.9891 0.0278 2.7793 0.1077
## 39 0.9951 0.1798 0.0327 1.0396 0.0045 562.6611 0.0288 2.8845 0.1799
## 40 -2.3024 -0.3669 0.1271 0.9237 0.0040 547.1095 0.0156 1.5562 -0.3737
## 41 0.4151 0.1024 0.0109 1.0670 0.0047 557.2909 0.0297 2.9659 0.1027
## 42 0.4532 0.1059 0.0116 1.0636 0.0047 561.2460 0.0285 2.8510 0.1061
## 43 -0.3921 -0.0427 0.0019 1.0526 0.0046 553.3941 0.0263 2.6257 -0.0428
## 44 0.4949 0.1134 0.0134 1.0638 0.0047 560.9157 0.0291 2.9114 0.11372.3.4.6 Forest Plots
## Reliability of factor 1
#pdf(file = "ForestPlots-TwoFactorModel3.pdf", onefile = TRUE)
forest.rma(MLREMCSC.tfmc.srel1,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfmc.srel1$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Reliability Factor 1 (Model 3)")
## Reliability of factor 2
forest.rma(MLREMCSC.tfmc.srel2,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfmc.srel2$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Reliability Factor 2 (Model 3)")
## Composite reliability
forest.rma(MLREMCSC.tfmc.srelcr,
addfit = TRUE,
order = "obs",
header = "Reference",
refline = 0,
rows = 1:MLREMCSC.tfmc.srelcr$k,
digits = 3,
level = 95,
cex = .9,
xlab = "Omega Coefficient",
main = "Composite Reliability (Model 3)")
#dev.off()3 R session info
sessionInfo()## R version 4.3.1 (2023-06-16)
## Platform: aarch64-apple-darwin20 (64-bit)
## Running under: macOS Ventura 13.4.1
##
## Matrix products: default
## BLAS: /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libRblas.0.dylib
## LAPACK: /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libRlapack.dylib; LAPACK version 3.11.0
##
## locale:
## [1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
##
## time zone: America/Chicago
## tzcode source: internal
##
## attached base packages:
## [1] stats graphics grDevices utils datasets methods base
##
## other attached packages:
## [1] dmetar_0.1.0 devtools_2.4.5 usethis_2.2.2
## [4] dplyr_1.1.2 lavaan_0.6-16 clubSandwich_0.5.10
## [7] robumeta_2.1 metafor_4.4-0 numDeriv_2016.8-1.1
## [10] metadat_1.2-0 Matrix_1.6-1 metaSEM_1.3.1
## [13] OpenMx_2.21.8 reshape2_1.4.4 psych_2.3.6
## [16] pacman_0.5.1
##
## loaded via a namespace (and not attached):
## [1] mathjaxr_1.6-0 rstudioapi_0.15.0 jsonlite_1.8.7
## [4] magrittr_2.0.3 modeltools_0.2-23 farver_2.1.1
## [7] nloptr_2.0.3 rmarkdown_2.24 fs_1.6.3
## [10] vctrs_0.6.4 memoise_2.0.1 minqa_1.2.6
## [13] CompQuadForm_1.4.3 htmltools_0.5.6 sass_0.4.7
## [16] bslib_0.5.1 htmlwidgets_1.6.2 plyr_1.8.8
## [19] poibin_1.5 sandwich_3.0-2 zoo_1.8-12
## [22] cachem_1.0.8 mime_0.12 lifecycle_1.0.3
## [25] pkgconfig_2.0.3 R6_2.5.1 fastmap_1.1.1
## [28] shiny_1.7.5 magic_1.6-1 digest_0.6.33
## [31] colorspace_2.1-0 ps_1.7.5 pkgload_1.3.3
## [34] ellipse_0.5.0 labeling_0.4.3 fansi_1.0.5
## [37] abind_1.4-5 compiler_4.3.1 remotes_2.4.2.1
## [40] withr_2.5.1 meta_6.5-0 highr_0.10
## [43] pkgbuild_1.4.2 MASS_7.3-60 sessioninfo_1.2.2
## [46] tools_4.3.1 pbivnorm_0.6.0 prabclus_2.3-3
## [49] httpuv_1.6.11 nnet_7.3-19 glue_1.6.2
## [52] quadprog_1.5-8 callr_3.7.3 nlme_3.1-162
## [55] promises_1.2.1 grid_4.3.1 cluster_2.1.4
## [58] generics_0.1.3 gtable_0.3.4 class_7.3-22
## [61] xml2_1.3.5 utf8_1.2.4 flexmix_2.3-19
## [64] ggrepel_0.9.4 pillar_1.9.0 stringr_1.5.0
## [67] later_1.3.1 robustbase_0.99-0 splines_4.3.1
## [70] lattice_0.21-8 tidyselect_1.2.0 miniUI_0.1.1.1
## [73] pbapply_1.7-2 knitr_1.43 gridExtra_2.3
## [76] stats4_4.3.1 xfun_0.40 diptest_0.76-0
## [79] DEoptimR_1.1-3 MuMIn_1.47.5 netmeta_2.8-2
## [82] stringi_1.7.12 yaml_2.3.7 boot_1.3-28.1
## [85] evaluate_0.22 codetools_0.2-19 kernlab_0.9-32
## [88] tibble_3.2.1 cli_3.6.1 RcppParallel_5.1.7
## [91] xtable_1.8-4 munsell_0.5.0 processx_3.8.2
## [94] jquerylib_0.1.4 Rcpp_1.0.11 parallel_4.3.1
## [97] ellipsis_0.3.2 ggplot2_3.4.4 prettyunits_1.2.0
## [100] mclust_6.0.0 profvis_0.3.8 urlchecker_1.0.1
## [103] lme4_1.1-34 mvtnorm_1.2-3 scales_1.2.1
## [106] purrr_1.0.2 crayon_1.5.2 fpc_2.2-10
## [109] rlang_1.1.1 mnormt_2.1.1