Dunning-Kruger: A Reanalysis
Setup
library(pacman); p_load(psych, dplyr, umx, lavaan, ggplot2, plyr, stringr, reshape, ggthemes, skedastic, magrittr, janitor, feather, MASS)
describe(logic); describe(grammar)## Warning in FUN(newX[, i], ...): no non-missing arguments to min; returning Inf
## Warning in FUN(newX[, i], ...): no non-missing arguments to min; returning Inf
## Warning in FUN(newX[, i], ...): no non-missing arguments to min; returning Inf## Warning in FUN(newX[, i], ...): no non-missing arguments to max; returning -Inf
## Warning in FUN(newX[, i], ...): no non-missing arguments to max; returning -Inf
## Warning in FUN(newX[, i], ...): no non-missing arguments to max; returning -Inf## Warning in FUN(newX[, i], ...): no non-missing arguments to min; returning Inf## Warning in FUN(newX[, i], ...): no non-missing arguments to min; returning Inf
## Warning in FUN(newX[, i], ...): no non-missing arguments to min; returning Inf## Warning in FUN(newX[, i], ...): no non-missing arguments to max; returning -Inf
## Warning in FUN(newX[, i], ...): no non-missing arguments to max; returning -Inf
## Warning in FUN(newX[, i], ...): no non-missing arguments to max; returning -InfAnalysis
Data from Jansen, Rafferty & Griffiths (2021).
LS <- lm(score ~ relAssess0_1, logic); summary(LS); cor(logic$score, logic$relAssess0_1); cor(logic[c(12:16, 37:41, 57)])##
## Call:
## lm(formula = score ~ relAssess0_1, data = logic)
##
## Residuals:
## Min 1Q Median 3Q Max
## -9.8405 -2.5482 0.3544 2.6250 9.6792
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 8.77955 0.20524 42.776 < 2e-16 ***
## relAssess0_1 0.01083 0.00315 3.437 0.000596 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 3.585 on 3541 degrees of freedom
## Multiple R-squared: 0.003324, Adjusted R-squared: 0.003043
## F-statistic: 11.81 on 1 and 3541 DF, p-value: 0.0005957## [1] 0.05765698## logicAssess0_1 absAssess0 relAssess0_1 diffSelf0_1 diffOther0_1
## logicAssess0_1 1.00000000 0.66184882 0.89407792 -0.4616638 0.06487064
## absAssess0 0.66184882 1.00000000 0.69264069 -0.5072459 -0.04661492
## relAssess0_1 0.89407792 0.69264069 1.00000000 -0.4720026 0.06038468
## diffSelf0_1 -0.46166385 -0.50724590 -0.47200257 1.0000000 0.43532448
## diffOther0_1 0.06487064 -0.04661492 0.06038468 0.4353245 1.00000000
## logicAssess1_1 0.72513394 0.53613889 0.72540617 -0.3495324 0.08432488
## absAssess1 0.48616767 0.63489101 0.51877544 -0.3397155 0.02338109
## relAssess1_1 0.66644198 0.50552651 0.70731841 -0.3239740 0.09722020
## diffSelf1_1 -0.27708746 -0.29365484 -0.27810229 0.4939021 0.22700360
## diffOther1_1 0.18015166 0.10633202 0.18094175 0.1340641 0.48892290
## score 0.04510516 0.08573495 0.05765698 -0.1443893 0.04223303
## logicAssess1_1 absAssess1 relAssess1_1 diffSelf1_1 diffOther1_1
## logicAssess0_1 0.72513394 0.48616767 0.66644198 -0.2770875 0.18015166
## absAssess0 0.53613889 0.63489101 0.50552651 -0.2936548 0.10633202
## relAssess0_1 0.72540617 0.51877544 0.70731841 -0.2781023 0.18094175
## diffSelf0_1 -0.34953235 -0.33971552 -0.32397405 0.4939021 0.13406408
## diffOther0_1 0.08432488 0.02338109 0.09722020 0.2270036 0.48892290
## logicAssess1_1 1.00000000 0.74270920 0.90329000 -0.4974259 0.01970997
## absAssess1 0.74270920 1.00000000 0.76059218 -0.5624752 -0.07861767
## relAssess1_1 0.90329000 0.76059218 1.00000000 -0.5020107 0.02775961
## diffSelf1_1 -0.49742591 -0.56247522 -0.50201068 1.0000000 0.53051517
## diffOther1_1 0.01970997 -0.07861767 0.02775961 0.5305152 1.00000000
## score 0.15372857 0.20472804 0.14592156 -0.1750649 -0.02711642
## score
## logicAssess0_1 0.04510516
## absAssess0 0.08573495
## relAssess0_1 0.05765698
## diffSelf0_1 -0.14438925
## diffOther0_1 0.04223303
## logicAssess1_1 0.15372857
## absAssess1 0.20472804
## relAssess1_1 0.14592156
## diffSelf1_1 -0.17506490
## diffOther1_1 -0.02711642
## score 1.00000000GS <- lm(score ~ relAssess0_1, grammar); summary(GS); cor(grammar$score, grammar$relAssess0_1); cor(grammar[c(12:16, 37:41, 58)])##
## Call:
## lm(formula = score ~ relAssess0_1, data = grammar)
##
## Residuals:
## Min 1Q Median 3Q Max
## -10.7765 -2.2207 0.2235 2.4088 10.1128
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 7.627150 0.214464 35.56 <2e-16 ***
## relAssess0_1 0.037051 0.003014 12.29 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 3.332 on 3513 degrees of freedom
## Multiple R-squared: 0.04124, Adjusted R-squared: 0.04096
## F-statistic: 151.1 on 1 and 3513 DF, p-value: < 2.2e-16## [1] 0.20307## grammarAssess0_1 absAssess0 relAssess0_1 diffSelf0_1
## grammarAssess0_1 1.00000000 0.707838923 0.89325312 -0.5738266
## absAssess0 0.70783892 1.000000000 0.70600105 -0.5956940
## relAssess0_1 0.89325312 0.706001050 1.00000000 -0.5592435
## diffSelf0_1 -0.57382663 -0.595693950 -0.55924351 1.0000000
## diffOther0_1 -0.05312201 -0.116015893 -0.03228031 0.4095503
## grammarAssess1_1 0.72996255 0.562355366 0.71985123 -0.4392896
## absAssess1 0.52569070 0.657140373 0.52364554 -0.4262133
## relAssess1_1 0.67636358 0.525618095 0.69949293 -0.3807506
## diffSelf1_1 -0.38240749 -0.396371361 -0.37164213 0.5661951
## diffOther1_1 0.08271416 0.002293475 0.09750054 0.1962045
## score 0.20047202 0.232606723 0.20306999 -0.3096976
## diffOther0_1 grammarAssess1_1 absAssess1 relAssess1_1
## grammarAssess0_1 -0.05312201 0.72996255 0.52569070 0.67636358
## absAssess0 -0.11601589 0.56235537 0.65714037 0.52561810
## relAssess0_1 -0.03228031 0.71985123 0.52364554 0.69949293
## diffSelf0_1 0.40955030 -0.43928962 -0.42621327 -0.38075062
## diffOther0_1 1.00000000 -0.01362124 -0.06690332 0.01426119
## grammarAssess1_1 -0.01362124 1.00000000 0.76127737 0.90561209
## absAssess1 -0.06690332 0.76127737 1.00000000 0.76286155
## relAssess1_1 0.01426119 0.90561209 0.76286155 1.00000000
## diffSelf1_1 0.24426638 -0.59323589 -0.63760209 -0.58402601
## diffOther1_1 0.53964215 -0.07442395 -0.17811591 -0.05562248
## score -0.03477355 0.25066054 0.28353478 0.23127442
## diffSelf1_1 diffOther1_1 score
## grammarAssess0_1 -0.3824075 0.082714163 0.200472023
## absAssess0 -0.3963714 0.002293475 0.232606723
## relAssess0_1 -0.3716421 0.097500539 0.203069986
## diffSelf0_1 0.5661951 0.196204488 -0.309697618
## diffOther0_1 0.2442664 0.539642153 -0.034773549
## grammarAssess1_1 -0.5932359 -0.074423954 0.250660536
## absAssess1 -0.6376021 -0.178115911 0.283534783
## relAssess1_1 -0.5840260 -0.055622483 0.231274421
## diffSelf1_1 1.0000000 0.499096508 -0.258594589
## diffOther1_1 0.4990965 1.000000000 -0.002041966
## score -0.2585946 -0.002041966 1.000000000ggplot(logic, aes(x = score, y = relAssess0_1)) + geom_point() + geom_smooth(method = loess, color = "steelblue4", formula = 'y ~ x') + geom_smooth(method = lm, color = "orangered2", formula = 'y ~ x') + labs(x = "Logic Score", y = "Relative Score self-Estimate") + theme_minimal() + theme(legend.position = "none", text = element_text(size = 12, family = "serif"), plot.title = element_text(hjust = 0.5))
ggplot(grammar, aes(x = score, y = relAssess0_1)) + geom_point() + geom_smooth(method = loess, color = "steelblue4", formula = 'y ~ x') + geom_smooth(method = lm, color = "orangered2", formula = 'y ~ x') + labs(x = "Grammar Score", y = "Relative Score self-Estimate") + theme_minimal() + theme(legend.position = "none", text = element_text(size = 12, family = "serif"), plot.title = element_text(hjust = 0.5))
plot(LS, c(1, 3)); plot(GS, c(1, 3))
ggplot(logic, aes(x = score)) +
geom_histogram(aes(y = ..density..), alpha = 0.5, position ="identity", bins = 20) +
geom_density(alpha=.4) + theme_bw() + ylab("") + xlab("Logic Score") + theme(legend.position = "none", panel.grid.major = element_blank(), panel.grid.minor = element_blank(), panel.background = element_blank(), panel.border = element_blank(), axis.ticks.y = element_blank(), axis.text.y = element_blank())
ggplot(grammar, aes(x = score)) +
geom_histogram(aes(y = ..density..), alpha = 0.5, position ="identity", bins = 21) +
geom_density(alpha=.4) + theme_bw() + ylab("") + xlab("Logic Score") + theme(legend.position = "none", panel.grid.major = element_blank(), panel.grid.minor = element_blank(), panel.background = element_blank(), panel.border = element_blank(), axis.ticks.y = element_blank(), axis.text.y = element_blank())
glejser(LS); glejser(GS)The original form of the Dunning-Kruger hypothesis seems to be clearly false. The only form that remains is the unfalsifiable one. I will reproduce how this looks below:
set.seed(1); n = 10000
df <- tibble(
true = rnorm(n),
unbiased = 0.5*true + 0.5*rnorm(n),
biased = 0.5*true + 0.5*rnorm(n) + -0.25*(true-4))
ggplot(df) + geom_smooth(mapping = aes(x = true, y = unbiased), color = "orangered") + geom_smooth(mapping = aes(x = true, y = biased)) + labs(x = "True", y = "Estimated") + theme_bw()## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'
ggplot(df) + geom_smooth(mapping = aes(x = true, y = biased - true), color = "purple") + labs(x = "True", y = "Estimated minus True") + theme_bw()## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'
References
Jansen, R. A., Rafferty, A. N., & Griffiths, T. L. (2021). A rational model of the Dunning–Kruger effect supports insensitivity to evidence in low performers. Nature Human Behaviour, 1–8. https://doi.org/10.1038/s41562-021-01057-0