Confidence model testing
Standard PA and NA-predicting models
Formulas
df.PA.standard <- lmer(bc_pa ~ PA_outcome + PA_PE + ( 1 + PA_outcome + PA_PE | cohort / id / exam_num), data = df, REML = FALSE)
df.NA.standard <- lmer(bc_na ~ NA_outcome + NA_PE + ( 1 + NA_outcome + NA_PE | cohort / id / exam_num), data = df, REML = FALSE)Summaries
summary(df.PA.standard)## Linear mixed model fit by maximum likelihood . t-tests use
## Satterthwaite's method [lmerModLmerTest]
## Formula:
## bc_pa ~ PA_outcome + PA_PE + (1 + PA_outcome + PA_PE | cohort/id/exam_num)
## Data: df
##
## AIC BIC logLik deviance df.resid
## 16273.8 16397.8 -8114.9 16229.8 2054
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -5.4160 -0.4504 0.0348 0.4830 4.8595
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## exam_num:(id:cohort) (Intercept) 274.5074 16.5683
## PA_outcome 2.4774 1.5740 -0.82
## PA_PE 11.4498 3.3838 0.69 -0.98
## id:cohort (Intercept) 220.2371 14.8404
## PA_outcome 3.4846 1.8667 -0.97
## PA_PE 2.2237 1.4912 0.99 -0.93
## cohort (Intercept) 11.1589 3.3405
## PA_outcome 0.2371 0.4869 -1.00
## PA_PE 0.1785 0.4225 1.00 -1.00
## Residual 101.5956 10.0795
## Number of obs: 2076, groups:
## exam_num:(id:cohort), 331; id:cohort, 117; cohort, 2
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) -6.5787 3.5736 2.2976 -1.841 0.1903
## PA_outcome 0.7470 0.4802 1.9381 1.556 0.2639
## PA_PE 2.6194 0.5738 3.6789 4.565 0.0125 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) PA_tcm
## PA_outcome -0.975
## PA_PE 0.728 -0.751summary(df.NA.standard)## Linear mixed model fit by maximum likelihood . t-tests use
## Satterthwaite's method [lmerModLmerTest]
## Formula:
## bc_na ~ NA_outcome + NA_PE + (1 + NA_outcome + NA_PE | cohort/id/exam_num)
## Data: df
##
## AIC BIC logLik deviance df.resid
## 16669.2 16793.2 -8312.6 16625.2 2054
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -3.7272 -0.4977 -0.0581 0.4194 4.8726
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## exam_num:(id:cohort) (Intercept) 5.294e+02 23.00781
## NA_outcome 7.384e+00 2.71744 -0.86
## NA_PE 1.092e+01 3.30515 0.69 -0.96
## id:cohort (Intercept) 1.180e+02 10.86172
## NA_outcome 8.862e-01 0.94140 -0.98
## NA_PE 5.584e+00 2.36306 0.90 -0.78
## cohort (Intercept) 3.209e-01 0.56651
## NA_outcome 6.693e-03 0.08181 1.00
## NA_PE 8.377e-03 0.09153 1.00 1.00
## Residual 1.146e+02 10.70615
## Number of obs: 2076, groups:
## exam_num:(id:cohort), 331; id:cohort, 117; cohort, 2
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 7.9061 2.2587 23.2548 3.500 0.0019 **
## NA_outcome -0.8096 0.2820 17.2281 -2.871 0.0105 *
## NA_PE -2.5963 0.5433 36.4875 -4.779 2.86e-05 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) NA_tcm
## NA_outcome -0.844
## NA_PE 0.555 -0.553AICs These are the winning AICs for all models tested
AIC(df.PA.standard)## [1] 16273.76AIC(df.NA.standard)## [1] 16669.2Models with confidence parameter
df.PA.conf <- lmer(bc_pa ~ PA_outcome + PA_PE*PA_confidence + ( 1 + PA_outcome + PA_PE + PA_confidence | cohort / id / exam_num), data = df, REML = FALSE)
df.NA.conf <- lmer(bc_na ~ NA_outcome + NA_PE*NA_confidence + ( 1 + NA_outcome + NA_PE + NA_confidence | cohort / id / exam_num), data = df, REML = FALSE)Summaries
summary(df.PA.conf)## Linear mixed model fit by maximum likelihood . t-tests use
## Satterthwaite's method [lmerModLmerTest]
## Formula: bc_pa ~ PA_outcome + PA_PE * PA_confidence + (1 + PA_outcome +
## PA_PE + PA_confidence | cohort/id/exam_num)
## Data: df
##
## AIC BIC logLik deviance df.resid
## 16429.8 16632.7 -8178.9 16357.8 2040
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -5.5080 -0.4466 0.0239 0.4816 4.9006
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## exam_num:(id:cohort) (Intercept) 171.58275 13.0990
## PA_outcome 5.26333 2.2942 -0.61
## PA_PE 35.88468 5.9904 0.46 -0.62
## PA_confidence 0.01663 0.1289 0.11 -0.80 0.17
## id:cohort (Intercept) 146.11845 12.0879
## PA_outcome 1.74336 1.3204 -0.45
## PA_PE 78.49881 8.8600 0.18 -0.40
## PA_confidence 0.04750 0.2179 -0.34 -0.52 -0.31
## cohort (Intercept) 58.72064 7.6629
## PA_outcome 27.84530 5.2769 0.21
## PA_PE 153.63133 12.3948 0.04 0.63
## PA_confidence 124.45168 11.1558 -0.18 0.14 -0.02
## Residual 95.06572 9.7502
## Number of obs: 2076, groups:
## exam_num:(id:cohort), 331; id:cohort, 117; cohort, 2
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) -4.86127 5.99846 0.21563 -0.810 0.76315
## PA_outcome 1.01244 3.76366 6.63136 0.269 0.79610
## PA_PE -1.07102 9.00913 0.12062 -0.119 0.96261
## PA_confidence -0.07012 7.88851 2.64648 -0.009 0.99353
## PA_PE:PA_confidence 0.06478 0.02458 266.90592 2.636 0.00888 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) PA_tcm PA_PE PA_cnf
## PA_outcome 0.152
## PA_PE 0.057 0.597
## PA_confidnc -0.159 0.135 -0.022
## PA_PE:PA_cn -0.046 0.045 -0.196 -0.002
## convergence code: 1
## unable to evaluate scaled gradient
## Model failed to converge: degenerate Hessian with 7 negative eigenvaluessummary(df.NA.conf)## Linear mixed model fit by maximum likelihood . t-tests use
## Satterthwaite's method [lmerModLmerTest]
## Formula: bc_na ~ NA_outcome + NA_PE * NA_confidence + (1 + NA_outcome +
## NA_PE + NA_confidence | cohort/id/exam_num)
## Data: df
##
## AIC BIC logLik deviance df.resid
## 16702.3 16905.3 -8315.1 16630.3 2040
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -3.9210 -0.4799 -0.0581 0.4146 4.8097
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## exam_num:(id:cohort) (Intercept) 50.51306 7.1073
## NA_outcome 3.69665 1.9227 0.99
## NA_PE 5.92702 2.4345 -0.55 -0.59
## NA_confidence 0.10402 0.3225 -0.85 -0.86 0.17
## id:cohort (Intercept) 199.60051 14.1280
## NA_outcome 0.99881 0.9994 -0.88
## NA_PE 11.74417 3.4270 0.54 -0.48
## NA_confidence 0.01161 0.1077 -0.87 0.52 -0.43
## cohort (Intercept) 245.17546 15.6581
## NA_outcome 143.87682 11.9949 0.29
## NA_PE 247.63777 15.7365 0.05 0.40
## NA_confidence 346.11054 18.6040 0.34 0.02 -0.14
## Residual 108.50228 10.4164
## Number of obs: 2076, groups:
## exam_num:(id:cohort), 331; id:cohort, 117; cohort, 2
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 12.32991 11.29431 23.21898 1.092 0.286
## NA_outcome -2.23785 8.49309 0.82950 -0.263 0.843
## NA_PE 3.56345 11.19132 0.02894 0.318 0.961
## NA_confidence 0.11561 13.15511 0.16081 0.009 0.997
## NA_PE:NA_confidence -0.07650 0.01535 475.19388 -4.982 8.81e-07 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) NA_tcm NA_PE NA_cnf
## NA_outcome 0.275
## NA_PE 0.053 0.393
## NA_confidnc 0.334 0.018 -0.140
## NA_PE:NA_cn -0.026 0.011 -0.089 -0.001
## convergence code: 1
## unable to evaluate scaled gradient
## Model failed to converge: degenerate Hessian with 6 negative eigenvaluesAICs These are the winning AICs for all confidence models
AIC(df.PA.conf)## [1] 16429.77AIC(df.NA.conf)## [1] 16702.3Ilustrating PE * Confidence Interaction for a sample subject with negative and positive PEs, and high and low confidence
plot 1: raw PE and Confidence variables
plot 2: raw PE * Confidence interaction
plot 3: fit values for PE * Confidence interaction (values from plot 2, multiplied by Betas from PA and NA models)
Alternate models tested (using prediction, inverse confidence, etc.)
df.PA.conf1 <- lmer(bc_pa ~ PA_outcome*PA_confidence + PA_PE + ( 1 + PA_outcome + PA_PE + PA_confidence | cohort / id / exam_num), data = df, REML = FALSE)
df.NA.conf1 <- lmer(bc_na ~ NA_outcome*NA_confidence + NA_PE + ( 1 + NA_outcome + NA_PE + NA_confidence | cohort / id / exam_num), data = df, REML = FALSE)AIC(df.PA.conf1)## [1] 16357.8AIC(df.NA.conf1)## [1] 16704.19df.PA.conf2 <- lmer(bc_pa ~ PA_outcome*PA_inverse_conf + PA_PE + ( 1 + PA_outcome + PA_PE + PA_inverse_conf | cohort / id / exam_num), data = df, REML = FALSE)
df.NA.conf2 <- lmer(bc_na ~ NA_outcome*NA_inverse_conf + NA_PE + ( 1 + NA_outcome + NA_PE + NA_inverse_conf | cohort / id / exam_num), data = df, REML = FALSE)AIC(df.PA.conf2)## [1] 16345.76AIC(df.NA.conf2)## [1] 16748.58df.PA.conf3 <- lmer(bc_pa ~ PA_outcome + PA_PE*PA_inverse_conf + ( 1 + PA_outcome + PA_PE + PA_inverse_conf | cohort / id / exam_num), data = df, REML = FALSE)
df.NA.conf3 <- lmer(bc_na ~ NA_outcome + NA_PE*NA_inverse_conf + ( 1 + NA_outcome + NA_PE + NA_inverse_conf | cohort / id / exam_num), data = df, REML = FALSE)AIC(df.PA.conf3)## [1] 16350.68AIC(df.NA.conf3)## [1] 16740.11df.PA.conf4 <- lmer(bc_pa ~ PA_outcome + PA_prediction*PA_confidence + ( 1 + PA_outcome + PA_prediction + PA_confidence | cohort / id / exam_num), data = df, REML = FALSE)
df.NA.conf4 <- lmer(bc_na ~ NA_outcome + NA_prediction*NA_confidence + ( 1 + NA_outcome + NA_prediction + NA_confidence | cohort / id / exam_num), data = df, REML = FALSE)AIC(df.PA.conf4)## [1] 16408.67AIC(df.NA.conf4)## [1] 16718.73df.PA.conf5 <- lmer(bc_pa ~ PA_PE + PA_prediction*PA_confidence + ( 1 + PA_PE + PA_prediction + PA_confidence | cohort / id / exam_num), data = df, REML = FALSE)
df.NA.conf5 <- lmer(bc_na ~ NA_PE + NA_prediction*NA_confidence + ( 1 + NA_PE + NA_prediction + NA_confidence | cohort / id / exam_num), data = df, REML = FALSE)AIC(df.PA.conf5)## [1] 16337.22AIC(df.NA.conf5)## [1] 16702.71Models with centered confidence parameter (range: -50 to +50)
… In Progress …