Evaluating Tweets Reduces the Influence of Inaccurate Content, But Does “Liking” a Tweet Reflect Evaluation? - Experiment 2
Load data and function
data <- readr::read_csv("twitter-exp2-long.csv", col_names = TRUE)## Rows: 30240 Columns: 28
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (15): RecordedDate, ID, Read1, why_judge, gender, race, social_media_use...
## dbl (12): Progress, Duration (in seconds), Read2, age, Item, Judgment_Time, ...
## lgl (1): Validity
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.stdCoef.merMod <- function(object) {
sdy <- sd(getME(object,"y"))
sdx <- apply(getME(object,"X"), 2, sd)
sc <- fixef(object)*sdx/sdy
se.fixef <- coef(summary(object))[,"Std. Error"]
se <- se.fixef*sdx/sdy
return(data.frame(stdcoef=sc, stdse=se))
}Incorrect lure and correct answer descriptive stats for table summaries
summary <- data %>%
group_by(ID,Validity,Difficulty,Judgment) %>%
summarise(
mean_error = mean(Error,na.rm=TRUE),
mean_correct = mean(Correct, na.rm=TRUE)
) %>%
group_by(Judgment, Validity, Difficulty) %>%
summarise(
mean_err = mean(mean_error),
sd_err = sd(mean_error),
se_error = sd(mean_error)/sqrt(n()),
mean_corr = mean(mean_correct),
sd_corr = sd(mean_correct),
se_corr = sd(mean_correct)/sqrt(n())
)## `summarise()` has grouped output by 'ID', 'Validity', 'Difficulty'. You can
## override using the `.groups` argument.
## `summarise()` has grouped output by 'Judgment', 'Validity'. You can override
## using the `.groups` argument.print(summary)## # A tibble: 12 × 9
## # Groups: Judgment, Validity [6]
## Judgment Validity Difficulty mean_err sd_err se_error mean_corr sd_corr
## <chr> <lgl> <chr> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 Accuracy FALSE Easy 0.0643 0.0766 0.00648 0.780 0.197
## 2 Accuracy FALSE Hard 0.121 0.104 0.00881 0.252 0.229
## 3 Accuracy TRUE Easy 0.0115 0.0271 0.00229 0.877 0.203
## 4 Accuracy TRUE Hard 0.0270 0.0349 0.00295 0.395 0.274
## 5 Interest FALSE Easy 0.105 0.111 0.00939 0.746 0.223
## 6 Interest FALSE Hard 0.138 0.110 0.00933 0.306 0.256
## 7 Interest TRUE Easy 0.0183 0.0439 0.00371 0.866 0.180
## 8 Interest TRUE Hard 0.0290 0.0336 0.00284 0.417 0.288
## 9 Like FALSE Easy 0.0948 0.117 0.00991 0.747 0.203
## 10 Like FALSE Hard 0.125 0.117 0.00991 0.275 0.278
## 11 Like TRUE Easy 0.0135 0.0289 0.00245 0.884 0.145
## 12 Like TRUE Hard 0.0266 0.0336 0.00284 0.396 0.291
## # … with 1 more variable: se_corr <dbl>data %>%
filter(Validity == "FALSE") %>%
group_by(ID,Validity,Difficulty,Judgment) %>%
summarise(
mean_error = mean(Error,na.rm=TRUE),
mean_correct = mean(Correct, na.rm=TRUE)
) %>%
group_by(Judgment,Difficulty) %>%
summarise(
mean_err = mean(mean_error),
sd_err = sd(mean_error),
se_error = sd(mean_error)/sqrt(n()),
mean_corr = mean(mean_correct),
sd_corr = sd(mean_correct),
se_corr = sd(mean_correct)/sqrt(n())
)## `summarise()` has grouped output by 'ID', 'Validity', 'Difficulty'. You can
## override using the `.groups` argument.
## `summarise()` has grouped output by 'Judgment'. You can override using the
## `.groups` argument.## # A tibble: 6 × 8
## # Groups: Judgment [3]
## Judgment Difficulty mean_err sd_err se_error mean_corr sd_corr se_corr
## <chr> <chr> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 Accuracy Easy 0.0643 0.0766 0.00648 0.780 0.197 0.0167
## 2 Accuracy Hard 0.121 0.104 0.00881 0.252 0.229 0.0194
## 3 Interest Easy 0.105 0.111 0.00939 0.746 0.223 0.0188
## 4 Interest Hard 0.138 0.110 0.00933 0.306 0.256 0.0216
## 5 Like Easy 0.0948 0.117 0.00991 0.747 0.203 0.0172
## 6 Like Hard 0.125 0.117 0.00991 0.275 0.278 0.0235# supplemental material, response count
data %>%
count(Difficulty,Response) %>%
summarise(
Difficulty = Difficulty,
Response = Response,
n = n,
prop = n/15120
)## # A tibble: 8 × 4
## Difficulty Response n prop
## <chr> <chr> <int> <dbl>
## 1 Easy Correct 12348 0.817
## 2 Easy Incorrect-Lure 775 0.0513
## 3 Easy Incorrect-Other 484 0.0320
## 4 Easy Unsure 1513 0.100
## 5 Hard Correct 5143 0.340
## 6 Hard Incorrect-Lure 1175 0.0777
## 7 Hard Incorrect-Other 2038 0.135
## 8 Hard Unsure 6764 0.447Models to analyze incorrect lure responses
# differences by Tweet accuracy
modelerror1 <- data %>%
mutate(Validity = ifelse(Validity == "TRUE",-.5,.5)) %>%
mutate(Difficulty = ifelse(Difficulty == "Easy",-.5,.5)) %>%
glmer(Error ~ Difficulty*Validity + (1 + Difficulty + Validity | ID) + (1 | Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))## boundary (singular) fit: see help('isSingular')summary(modelerror1)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: Error ~ Difficulty * Validity + (1 + Difficulty + Validity |
## ID) + (1 | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 11000.8 11092.3 -5489.4 10978.8 30229
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.4443 -0.2282 -0.1318 -0.0701 16.0884
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 0.4695 0.6852
## Difficulty 0.5689 0.7543 -0.17
## Validity 0.3561 0.5968 0.89 0.30
## Item (Intercept) 1.5847 1.2588
## Number of obs: 30240, groups: ID, 420; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -3.89406 0.16048 -24.266 <2e-16 ***
## Difficulty 0.40238 0.31531 1.276 0.202
## Validity 1.85659 0.08236 22.543 <2e-16 ***
## Difficulty:Validity -0.24633 0.15303 -1.610 0.107
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) Dffclt Valdty
## Difficulty -0.019
## Validity -0.088 0.078
## Dffclty:Vld 0.073 -0.175 -0.305
## optimizer (bobyqa) convergence code: 0 (OK)
## boundary (singular) fit: see help('isSingular')emmeans(modelerror1, ~ Difficulty*Validity) %>%
pairs(., simple = "Validity", reverse = TRUE)## Difficulty = -0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) 1.98 0.1284 Inf 15.419 <.0001
##
## Difficulty = 0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) 1.73 0.0938 Inf 18.490 <.0001
##
## Results are given on the log odds ratio (not the response) scale.# contrast accuracy < interest + like
modelerror2.1 <- data %>%
filter(Validity == "FALSE") %>%
mutate(acc_v_intlik = case_when(
Judgment == "Interest" ~ 1,
Judgment == "Like" ~ 1,
Judgment == "Accuracy" ~ -2
)) %>%
mutate(int_vs_like = case_when(
Judgment == "Interest" ~ 1,
Judgment == "Like" ~ -1,
Judgment == "Accuracy" ~ 0
)) %>%
mutate(Difficulty = ifelse(Difficulty == "Easy",-.5,.5)) %>%
glmer(Error ~ Difficulty*acc_v_intlik + Difficulty*int_vs_like + (1 + Difficulty | ID) + (1 | Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(modelerror2.1)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: Error ~ Difficulty * acc_v_intlik + Difficulty * int_vs_like +
## (1 + Difficulty | ID) + (1 | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 8724.8 8801.0 -4352.4 8704.8 15110
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.7843 -0.3287 -0.2078 -0.1308 8.7444
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 0.7829 0.8848
## Difficulty 0.4599 0.6782 -0.03
## Item (Intercept) 1.3139 1.1462
## Number of obs: 15120, groups: ID, 420; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -2.86650 0.14882 -19.261 < 2e-16 ***
## Difficulty 0.30872 0.28576 1.080 0.27999
## acc_v_intlik 0.10902 0.03806 2.864 0.00418 **
## int_vs_like 0.11606 0.06451 1.799 0.07200 .
## Difficulty:acc_v_intlik -0.14965 0.04956 -3.019 0.00253 **
## Difficulty:int_vs_like -0.04831 0.08167 -0.591 0.55420
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) Dffclt acc_v_ int_v_ Dffclty:c__
## Difficulty -0.002
## acc_v_intlk -0.017 0.014
## int_vs_like -0.013 0.005 -0.010
## Dffclty:c__ 0.020 -0.024 -0.117 0.003
## Dffclty:n__ 0.009 -0.019 0.003 -0.067 -0.020# test of simple effects
emmeans(modelerror2.1, ~ Difficulty*acc_v_intlik) %>%
pairs(., simple = "acc_v_intlik", reverse = FALSE)## Difficulty = -0.5:
## contrast estimate SE df z.ratio p.value
## (-2) - 1 -0.552 0.143 Inf -3.847 0.0001
##
## Difficulty = 0.5:
## contrast estimate SE df z.ratio p.value
## (-2) - 1 -0.103 0.129 Inf -0.797 0.4257
##
## Results are given on the log odds ratio (not the response) scale.# standardized beta and se
stdCoef.merMod(modelerror1)## stdcoef stdse
## (Intercept) 0.0000000 0.0000000
## Difficulty 0.8191377 0.6418852
## Validity 3.7794960 0.1676564
## Difficulty:Validity -0.2507245 0.1557621stdCoef.merMod(modelerror2.1)## stdcoef stdse
## (Intercept) 0.00000000 0.0000000
## Difficulty 0.49732640 0.4603394
## acc_v_intlik 0.49671881 0.1734242
## int_vs_like 0.30531467 0.1697018
## Difficulty:acc_v_intlik -0.34093391 0.1129149
## Difficulty:int_vs_like -0.06353602 0.1074191Models to analyze correct answer responses
# differences by Tweet accuracy
modelcorrect1 <- data %>%
mutate(Validity = ifelse(Validity == "TRUE",.5,-.5)) %>%
mutate(Difficulty = ifelse(Difficulty == "Easy",.5,-.5)) %>%
glmer(Correct ~ Difficulty*Validity + (1 + Difficulty + Validity | ID) + (1 | Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(modelcorrect1)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: Correct ~ Difficulty * Validity + (1 + Difficulty + Validity |
## ID) + (1 | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 26318.6 26410.1 -13148.3 26296.6 30229
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -8.4349 -0.4371 0.1731 0.4475 8.6800
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 1.7827 1.3352
## Difficulty 1.2811 1.1319 -0.28
## Validity 0.4494 0.6703 0.26 0.33
## Item (Intercept) 0.4996 0.7069
## Number of obs: 30240, groups: ID, 420; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.54223 0.10778 5.031 4.88e-07 ***
## Difficulty 3.16250 0.18028 17.542 < 2e-16 ***
## Validity 1.04469 0.04825 21.651 < 2e-16 ***
## Difficulty:Validity 0.37708 0.07617 4.951 7.40e-07 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) Dffclt Valdty
## Difficulty -0.041
## Validity 0.130 0.116
## Dffclty:Vld 0.057 0.038 0.157# contrast accuracy > interest + like
modelcorrect2.1 <- data %>%
filter(Validity == "FALSE") %>%
mutate(acc_v_intlik = case_when(
Judgment == "Interest" ~ -2,
Judgment == "Like" ~ -1,
Judgment == "Accuracy" ~ -1
)) %>%
mutate(int_vs_like = case_when(
Judgment == "Interest" ~ 0,
Judgment == "Like" ~ -1,
Judgment == "Accuracy" ~ 1
)) %>%
mutate(Difficulty = ifelse(Difficulty == "Easy",.5,-.5)) %>%
glmer(Correct ~ Difficulty*acc_v_intlik + Difficulty*int_vs_like + (1 + Difficulty | ID) + (1 | Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(modelcorrect2.1)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: Correct ~ Difficulty * acc_v_intlik + Difficulty * int_vs_like +
## (1 + Difficulty | ID) + (1 | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 14187.5 14263.7 -7083.7 14167.5 15110
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -6.6378 -0.4600 0.1762 0.4903 6.0783
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 1.7716 1.3310
## Difficulty 1.3939 1.1806 -0.43
## Item (Intercept) 0.5052 0.7108
## Number of obs: 15120, groups: ID, 420; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -0.07469 0.22477 -0.332 0.73965
## Difficulty 3.69041 0.28420 12.985 < 2e-16 ***
## acc_v_intlik -0.05381 0.14671 -0.367 0.71376
## int_vs_like 0.03231 0.08496 0.380 0.70375
## Difficulty:acc_v_intlik 0.48061 0.15952 3.013 0.00259 **
## Difficulty:int_vs_like 0.10743 0.09357 1.148 0.25093
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) Dffclt acc_v_ int_v_ Dffclty:c__
## Difficulty -0.243
## acc_v_intlk 0.873 -0.245
## int_vs_like -0.002 0.007 -0.001
## Dffclty:c__ -0.285 0.758 -0.325 0.005
## Dffclty:n__ 0.007 -0.007 0.005 -0.328 -0.005# test of simple effects
emmeans(modelcorrect1, ~ Difficulty * Validity) %>%
pairs(., simple = "Validity", reverse = TRUE)## Difficulty = -0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) 0.856 0.0566 Inf 15.130 <.0001
##
## Difficulty = 0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) 1.233 0.0660 Inf 18.687 <.0001
##
## Results are given on the log odds ratio (not the response) scale.emmeans(modelcorrect2.1, ~ Difficulty * acc_v_intlik) %>%
pairs(., simple = "acc_v_intlik", reverse = TRUE)## Difficulty = -0.5:
## contrast estimate SE df z.ratio p.value
## (-1) - (-2) -0.294 0.188 Inf -1.561 0.1185
##
## Difficulty = 0.5:
## contrast estimate SE df z.ratio p.value
## (-1) - (-2) 0.186 0.142 Inf 1.310 0.1903
##
## Results are given on the log odds ratio (not the response) scale.Graph of incorrect lure responses by tweet accuracy
Twitter_Exp2_Figure1 <- summary %>%
filter(Validity == "FALSE") %>%
group_by(Difficulty, Judgment) %>%
ggplot(.) + aes(x = reorder(Difficulty, mean_err), y = mean_err, fill = reorder(Judgment, -mean_err)) +
geom_bar(stat = "summary", fun.y = "mean", position = "dodge") +
xlab("Tweet Difficulty") + ylab("Proportion Lure Reported") + labs(fill = "Judgment Type") +
theme_classic() +
geom_errorbar(aes(ymin= mean_err - se_error, ymax= mean_err + se_error), position=position_dodge(width=0.9), width=.1) + geom_text(aes(label=round(mean_err,digits=2)), position=position_dodge(width=.9), vjust=4) + scale_fill_manual(values=c("#f0f0f0","#bdbdbd","#636363"))## Warning: Ignoring unknown parameters: fun.yprint(Twitter_Exp2_Figure1)## No summary function supplied, defaulting to `mean_se()`
ggsave(Twitter_Exp2_Figure1, file="Twitter-Exp2-Error.jpeg", width=8, height=4)## No summary function supplied, defaulting to `mean_se()`Graph of correct answer responses by tweet accuracy
Twitter_Exp2_Figure2 <- summary %>%
filter(Validity == "FALSE") %>%
group_by(Difficulty, Judgment) %>%
ggplot(.) + aes(x = reorder(Difficulty, -mean_corr), y = mean_corr, fill = reorder(Judgment, -mean_err)) +
geom_bar(stat = "summary", fun.y = "mean", position = "dodge") +
xlab("Tweet Difficulty") + ylab("Proportion Correct Answer Reported") + labs(fill = "Judgment Type") + theme_classic() +
geom_errorbar(aes(ymin= mean_corr - se_corr, ymax= mean_corr + se_corr), position=position_dodge(width=0.9), width=.1) + geom_text(aes(label=round(mean_corr,digits=2)), position=position_dodge(width=.9), vjust=3) + scale_fill_manual(values=c("#f0f0f0","#bdbdbd","#636363"))## Warning: Ignoring unknown parameters: fun.yprint(Twitter_Exp2_Figure2)## No summary function supplied, defaulting to `mean_se()`
ggsave(Twitter_Exp2_Figure2, file="Twitter-Exp2-Correct.jpeg", width=8, height=4)## No summary function supplied, defaulting to `mean_se()`Exploratory analysis: Does accuracy, interest, or like judgment at exposure predict reproduction of that information at test?
# accuracy mindset
accuracy_reproduce <- data %>%
mutate(reproduce_info = case_when(
Validity == "TRUE" & Response == "Correct" ~ 1,
Validity == "FALSE" & Response == "Error" ~ 1,
TRUE ~ 0
)) %>%
filter(Judgment == "Accuracy") %>%
glmer(reproduce_info ~ Judgment_Response + (1 + Judgment_Response | ID) + (1 + Judgment_Response| Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5))) ## boundary (singular) fit: see help('isSingular')summary(accuracy_reproduce)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: reproduce_info ~ Judgment_Response + (1 + Judgment_Response |
## ID) + (1 + Judgment_Response | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 9432.2 9490.0 -4708.1 9416.2 10072
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -4.2959 -0.4997 -0.2914 0.4600 8.9787
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 0.230581 0.48019
## Judgment_Response 0.001963 0.04431 1.00
## Item (Intercept) 6.758001 2.59962
## Judgment_Response 0.607352 0.77933 -0.95
## Number of obs: 10080, groups: ID, 140; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -1.86644 0.31879 -5.855 4.78e-09 ***
## Judgment_Response 0.23016 0.09424 2.442 0.0146 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## Jdgmnt_Rspn -0.935
## optimizer (bobyqa) convergence code: 0 (OK)
## boundary (singular) fit: see help('isSingular')stdCoef.merMod(accuracy_reproduce)## stdcoef stdse
## (Intercept) 0.00000 0.000000
## Judgment_Response 0.82257 0.336801# interest mindset
interest_reproduce <- data %>%
mutate(reproduce_info = case_when(
Validity == "TRUE" & Response == "Correct" ~ 1,
Validity == "FALSE" & Response == "Error" ~ 1,
TRUE ~ 0
)) %>%
filter(Judgment == "Interest") %>%
glmer(reproduce_info ~ Judgment_Response + (1 + Judgment_Response| ID) + (1 + Judgment_Response| Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(interest_reproduce)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: reproduce_info ~ Judgment_Response + (1 + Judgment_Response |
## ID) + (1 + Judgment_Response | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 11894.6 11952.3 -5939.3 11878.6 10072
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.6328 -0.6739 -0.4650 1.0168 3.8314
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 0.139284 0.37321
## Judgment_Response 0.005982 0.07734 0.28
## Item (Intercept) 0.555082 0.74504
## Judgment_Response 0.024200 0.15556 -0.54
## Number of obs: 10080, groups: ID, 140; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -0.99056 0.11204 -8.841 <2e-16 ***
## Judgment_Response 0.04557 0.02833 1.609 0.108
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## Jdgmnt_Rspn -0.626# social media mindset
like_reproduce <- data %>%
mutate(reproduce_info = case_when(
Validity == "TRUE" & Response == "Correct" ~ 1,
Validity == "FALSE" & Response == "Error" ~ 1,
TRUE ~ 0
)) %>%
filter(Judgment == "Like") %>%
glmer(reproduce_info ~ Judgment_Response + (1 + Judgment_Response| ID) + (1 + Judgment_Response| Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(like_reproduce)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: reproduce_info ~ Judgment_Response + (1 + Judgment_Response |
## ID) + (1 + Judgment_Response | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 11666.2 11723.9 -5825.1 11650.2 10072
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.6448 -0.6581 -0.4424 0.9183 5.3068
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 0.14684 0.3832
## Judgment_Response 0.02117 0.1455 -0.36
## Item (Intercept) 0.84442 0.9189
## Judgment_Response 0.09458 0.3075 -0.71
## Number of obs: 10080, groups: ID, 140; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -1.03745 0.12632 -8.213 <2e-16 ***
## Judgment_Response 0.05957 0.04322 1.378 0.168
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr)
## Jdgmnt_Rspn -0.720Exploratory analysis: Are there differences in time taken to judge tweets between judgment groups?
# table of descriptive stats of response time
time_summary <- data %>%
group_by(Judgment) %>%
summarise(
mean_judge_time = mean(Judgment_Time),
sd_judge_time = sd(Judgment_Time),
mean_test_time = mean(Test_Time),
sd_test_time = sd(Test_Time)
)
print(time_summary)## # A tibble: 3 × 5
## Judgment mean_judge_time sd_judge_time mean_test_time sd_test_time
## <chr> <dbl> <dbl> <dbl> <dbl>
## 1 Accuracy 11.6 22.8 9.06 19.1
## 2 Interest 9.07 20.6 10.3 19.5
## 3 Like 9.90 26.5 9.54 16.0# exposure judgment time
exposure_time_model <- data %>%
lmer(Judgment_Time ~ Judgment + (1 | ID) + (1 + Judgment | Item),., control = lmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))## boundary (singular) fit: see help('isSingular')## Warning: Model failed to converge with 1 negative eigenvalue: -2.4e-01summary(exposure_time_model)## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: Judgment_Time ~ Judgment + (1 | ID) + (1 + Judgment | Item)
## Data: .
## Control: lmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## REML criterion at convergence: 275656.2
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.802 -0.183 -0.087 0.000 62.067
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 26.3503 5.1333
## Item (Intercept) 3.6208 1.9028
## JudgmentInterest 0.4054 0.6367 -1.00
## JudgmentLike 2.7322 1.6530 -0.64 0.64
## Residual 519.8289 22.7998
## Number of obs: 30240, groups: ID, 420; Item, 72
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 11.5845 0.5386 296.1099 21.509 < 2e-16 ***
## JudgmentInterest -2.5108 0.6966 404.5787 -3.605 0.000352 ***
## JudgmentLike -1.6856 0.7194 314.5378 -2.343 0.019748 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) JdgmnI
## JdgmntIntrs -0.684
## JudgmentLik -0.691 0.497
## optimizer (bobyqa) convergence code: 0 (OK)
## boundary (singular) fit: see help('isSingular')stdCoef.merMod(exposure_time_model)## stdcoef stdse
## (Intercept) 0.00000000 0.00000000
## JudgmentInterest -0.05049117 0.01400776
## JudgmentLike -0.03389673 0.01446673# test answer time
test_time_model <- data %>%
lmer(Test_Time ~ Judgment + (1 | ID) + (1 + Judgment| Item),., control = lmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))## boundary (singular) fit: see help('isSingular')summary(test_time_model)## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: Test_Time ~ Judgment + (1 | ID) + (1 + Judgment | Item)
## Data: .
## Control: lmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## REML criterion at convergence: 260028.9
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.391 -0.216 -0.104 0.036 77.251
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 23.5548 4.8533
## Item (Intercept) 1.6397 1.2805
## JudgmentInterest 1.1208 1.0587 0.52
## JudgmentLike 0.0233 0.1526 -0.93 -0.16
## Residual 308.4212 17.5619
## Number of obs: 30240, groups: ID, 420; Item, 72
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 9.0605 0.4708 373.6583 19.246 <2e-16 ***
## JudgmentInterest 1.2101 0.6429 374.3136 1.882 0.0606 .
## JudgmentLike 0.4776 0.6309 415.9126 0.757 0.4494
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) JdgmnI
## JdgmntIntrs -0.625
## JudgmentLik -0.678 0.489
## optimizer (bobyqa) convergence code: 0 (OK)
## boundary (singular) fit: see help('isSingular')Coding of accuracy focus mentioned in open-response text of judgment strategies
# proportion of individuals who mention taking accuracy into account in their judgments (by judgment condition)
Twitter_Exp2_Figure3 <- data %>%
group_by(Judgment) %>%
summarise(
prop = sum(mentions_accuracy == 1)/n()
) %>%
ggplot(.) + aes(x = reorder(Judgment, -prop), y = prop, fill = reorder(Judgment, -prop)) +
geom_bar(stat = "summary", fun.y = "mean", position = "dodge") + ylab("Proportion Mention Accuracy as Judgment Strategy") + labs(fill = "Requested Judgment at Exposure") + theme_classic() + theme(axis.title.x=element_blank(), axis.text.x=element_blank(), axis.ticks.x=element_blank(), legend.position = "bottom") + geom_text(aes(label=round(prop,digits=2)), position=position_dodge(width=.9), vjust=3) + scale_fill_manual(values=c("#636363","#bdbdbd","#f0f0f0"))## Warning: Ignoring unknown parameters: fun.yggsave(Twitter_Exp2_Figure3, file="Twitter-Exp3-OR.jpeg", width=8, height=4)## No summary function supplied, defaulting to `mean_se()`# prop incorrect and correct answers based on mentioning they took accuracy into account in their judgments (by judgment condition)
data %>%
filter(Validity == "FALSE") %>%
group_by(ID,Judgment,Difficulty,mentions_accuracy) %>%
summarise(
mean_error = mean(Error,na.rm=TRUE),
mean_correct = mean(Correct, na.rm=TRUE)
) %>%
group_by(Judgment,Difficulty, mentions_accuracy) %>%
summarise(
mean_err = mean(mean_error),
sd_err = sd(mean_error),
se_error = sd(mean_error)/sqrt(n()),
mean_corr = mean(mean_correct),
sd_corr = sd(mean_correct),
se_corr = sd(mean_correct)/sqrt(n())
) ## `summarise()` has grouped output by 'ID', 'Judgment', 'Difficulty'. You can
## override using the `.groups` argument.## `summarise()` has grouped output by 'Judgment', 'Difficulty'. You can override
## using the `.groups` argument.## # A tibble: 12 × 9
## # Groups: Judgment, Difficulty [6]
## Judgment Difficulty mentions_accuracy mean_err sd_err se_error mean_corr
## <chr> <chr> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 Accuracy Easy 0 0.0660 0.0866 0.0216 0.601
## 2 Accuracy Easy 1 0.0641 0.0757 0.00679 0.803
## 3 Accuracy Hard 0 0.0694 0.0772 0.0193 0.240
## 4 Accuracy Hard 1 0.128 0.106 0.00948 0.254
## 5 Interest Easy 0 0.125 0.123 0.0127 0.696
## 6 Interest Easy 1 0.0652 0.0666 0.00981 0.848
## 7 Interest Hard 0 0.129 0.101 0.0104 0.268
## 8 Interest Hard 1 0.156 0.126 0.0186 0.382
## 9 Like Easy 0 0.126 0.133 0.0145 0.667
## 10 Like Easy 1 0.0486 0.0671 0.00896 0.867
## 11 Like Hard 0 0.124 0.126 0.0137 0.203
## 12 Like Hard 1 0.126 0.105 0.0140 0.383
## # … with 2 more variables: sd_corr <dbl>, se_corr <dbl>qualitative_error <- data %>%
filter(Validity == "FALSE") %>%
group_by(ID,Judgment,Difficulty,mentions_accuracy) %>%
summarise(
mean_error = mean(Error,na.rm=TRUE),
mean_correct = mean(Correct, na.rm=TRUE)
) %>%
group_by(Difficulty, mentions_accuracy) %>%
summarise(
mean_err = mean(mean_error),
sd_err = sd(mean_error),
se_error = sd(mean_error)/sqrt(n()),
mean_corr = mean(mean_correct),
sd_corr = sd(mean_correct),
se_corr = sd(mean_correct)/sqrt(n())
) %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 1, "Yes","No")) %>%
ggplot(.) + aes(x = Difficulty, y = mean_err, fill = mentions_accuracy) +
geom_bar(stat = "summary", fun.y = "mean", position = "dodge") +
xlab("Tweet Difficulty") + ylab("Proportion Lure Reported") + labs(fill = "Self-Reported Conisderation of Accuracy") + theme_classic() + theme(legend.position = "bottom") + geom_errorbar(aes(ymin= mean_err - se_error, ymax= mean_err + se_error), position=position_dodge(width=0.9), width=.1) + geom_text(aes(label=round(mean_err,digits=2)), position=position_dodge(width=.9), vjust=4) + scale_fill_manual(values=c("#f0f0f0","#636363"))## `summarise()` has grouped output by 'ID', 'Judgment', 'Difficulty'. You can
## override using the `.groups` argument.
## `summarise()` has grouped output by 'Difficulty'. You can override using the
## `.groups` argument.## Warning: Ignoring unknown parameters: fun.yggsave(qualitative_error, file="Twitter-Exp3-Qual-Error.jpeg", width=8, height=4)## No summary function supplied, defaulting to `mean_se()`qualitative_correct <- data %>%
filter(Validity == "FALSE") %>%
group_by(ID,Judgment,Difficulty,mentions_accuracy) %>%
summarise(
mean_error = mean(Error,na.rm=TRUE),
mean_correct = mean(Correct, na.rm=TRUE)
) %>%
group_by(Difficulty, mentions_accuracy) %>%
summarise(
mean_err = mean(mean_error),
sd_err = sd(mean_error),
se_error = sd(mean_error)/sqrt(n()),
mean_corr = mean(mean_correct),
sd_corr = sd(mean_correct),
se_corr = sd(mean_correct)/sqrt(n())
) %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 1, "Yes","No")) %>%
ggplot(.) + aes(x = Difficulty, y = mean_corr, fill = mentions_accuracy) +
geom_bar(stat = "summary", fun.y = "mean", position = "dodge") +
xlab("Tweet Difficulty") + ylab("Proportion Lure Reported") + labs(fill = "Self-Reported Conisderation of Accuracy") + theme_classic() + theme(legend.position = "bottom") + geom_errorbar(aes(ymin= mean_corr - se_corr, ymax= mean_corr + se_corr), position=position_dodge(width=0.9), width=.1) + geom_text(aes(label=round(mean_corr,digits=2)), position=position_dodge(width=.9), vjust=4) + scale_fill_manual(values=c("#f0f0f0","#636363"))## `summarise()` has grouped output by 'ID', 'Judgment', 'Difficulty'. You can
## override using the `.groups` argument.
## `summarise()` has grouped output by 'Difficulty'. You can override using the
## `.groups` argument.## Warning: Ignoring unknown parameters: fun.yggsave(qualitative_correct, file="Twitter-Exp3-Qual-Correct.jpeg", width=8, height=4)## No summary function supplied, defaulting to `mean_se()`# graph after non-evaluative judgments only
qualitative_error2 <- data %>%
filter(Validity == "FALSE") %>%
filter(Judgment != "Accuracy") %>%
group_by(ID,Judgment,Difficulty,mentions_accuracy) %>%
summarise(
mean_error = mean(Error,na.rm=TRUE),
mean_correct = mean(Correct, na.rm=TRUE)
) %>%
group_by(Judgment,Difficulty, mentions_accuracy) %>%
summarise(
mean_err = mean(mean_error),
sd_err = sd(mean_error),
se_error = sd(mean_error)/sqrt(n()),
mean_corr = mean(mean_correct),
sd_corr = sd(mean_correct),
se_corr = sd(mean_correct)/sqrt(n())
) %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 1, "Yes","No")) %>%
ggplot(.) + aes(x = Judgment, y = mean_err, fill = mentions_accuracy) +
geom_bar(stat = "summary", fun.y = "mean", position = "dodge") +
xlab("Judgment Type") + ylab("Proportion Lure Reported") + labs(fill = "Self-Reported Consideration of Accuracy") +
theme_classic() + theme(legend.position = "bottom") + geom_errorbar(aes(ymin= mean_err - se_error, ymax= mean_err + se_error), position=position_dodge(width=0.9), width=.1) + geom_text(aes(label=round(mean_err,digits=2)), position=position_dodge(width=.9), vjust=4) + facet_wrap(vars(Difficulty)) + scale_fill_manual(values=c("#f0f0f0","#636363"))## `summarise()` has grouped output by 'ID', 'Judgment', 'Difficulty'. You can
## override using the `.groups` argument.
## `summarise()` has grouped output by 'Judgment', 'Difficulty'. You can override
## using the `.groups` argument.## Warning: Ignoring unknown parameters: fun.yggsave(qualitative_error2, file="Twitter-Exp3-Qual-Error2.jpeg", width=8, height=4)## No summary function supplied, defaulting to `mean_se()`
## No summary function supplied, defaulting to `mean_se()`qualitative_correct2 <- data %>%
filter(Validity == "FALSE") %>%
filter(Judgment != "Accuracy") %>%
group_by(ID,Judgment,Difficulty,mentions_accuracy) %>%
summarise(
mean_error = mean(Error,na.rm=TRUE),
mean_correct = mean(Correct, na.rm=TRUE)
) %>%
group_by(Judgment,Difficulty, mentions_accuracy) %>%
summarise(
mean_err = mean(mean_error),
sd_err = sd(mean_error),
se_error = sd(mean_error)/sqrt(n()),
mean_corr = mean(mean_correct),
sd_corr = sd(mean_correct),
se_corr = sd(mean_correct)/sqrt(n())
) %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 1, "Yes","No")) %>%
ggplot(.) + aes(x = Judgment, y = mean_corr, fill = mentions_accuracy) +
geom_bar(stat = "summary", fun.y = "mean", position = "dodge") +
xlab("Self-Reported Consideration of Accuracy") + ylab("Proportion Correct Answer Reported") + labs(fill = "Judgment Type") + theme_classic() + theme(legend.position = "bottom") + geom_errorbar(aes(ymin= mean_corr - se_corr, ymax= mean_corr + se_corr), position=position_dodge(width=0.9), width=.1) + geom_text(aes(label=round(mean_corr,digits=2)), position=position_dodge(width=.9), vjust=4) + facet_wrap(vars(Difficulty)) + scale_fill_manual(values=c("#f0f0f0","#636363"))## `summarise()` has grouped output by 'ID', 'Judgment', 'Difficulty'. You can
## override using the `.groups` argument.
## `summarise()` has grouped output by 'Judgment', 'Difficulty'. You can override
## using the `.groups` argument.## Warning: Ignoring unknown parameters: fun.yggsave(qualitative_correct2, file="Twitter-Exp3-Qual-Correct2.jpeg", width=8, height=4)## No summary function supplied, defaulting to `mean_se()`
## No summary function supplied, defaulting to `mean_se()`# contingency table and chi-square analyses
data %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 0, "no","yes")) %>%
count(ID,Judgment,mentions_accuracy) %>%
count(Judgment,mentions_accuracy) %>%
spread(mentions_accuracy,n)## # A tibble: 3 × 3
## Judgment no yes
## <chr> <int> <int>
## 1 Accuracy 16 124
## 2 Interest 94 46
## 3 Like 84 56contig_table <- matrix(c(16,94,84,124,46,56), ncol = 2)
dimnames(contig_table) <- list(judgment = c("Accuracy","Interest", "Like"),mentions_accuracy = c("Yes", "No"))
chisq.test(contig_table)##
## Pearson's Chi-squared test
##
## data: contig_table
## X-squared = 103.53, df = 2, p-value < 2.2e-16chisq.post.hoc(contig_table)## Adjusted p-values used the fdr method.## comparison raw.p adj.p
## 1 Accuracy vs. Interest 0.0000 0.0000
## 2 Accuracy vs. Like 0.0000 0.0000
## 3 Interest vs. Like 0.2637 0.2637# do people who mention accuracy as a strategy reproduce fewer inaccuracies at test?
model_or_error1 <- data %>%
filter(Validity == "FALSE") %>%
mutate(Difficulty = ifelse(Difficulty == "Easy",-.5,.5)) %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 1,-.5,.5)) %>%
glmer(Error ~ Difficulty*mentions_accuracy + (1 + Difficulty | ID) + (1 | Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(model_or_error1)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: Error ~ Difficulty * mentions_accuracy + (1 + Difficulty | ID) +
## (1 | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 8686.3 8747.3 -4335.2 8670.3 15112
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.8383 -0.3305 -0.2069 -0.1291 8.7045
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 0.7902 0.8889
## Difficulty 0.2983 0.5462 0.07
## Item (Intercept) 1.3017 1.1409
## Number of obs: 15120, groups: ID, 420; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -2.8549 0.1484 -19.243 < 2e-16 ***
## Difficulty 0.2682 0.2839 0.944 0.3449
## mentions_accuracy 0.3082 0.1074 2.870 0.0041 **
## Difficulty:mentions_accuracy -0.9499 0.1307 -7.266 3.7e-13 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) Dffclt mntns_
## Difficulty 0.002
## mntns_ccrcy 0.009 0.028
## Dffclty:mn_ 0.040 -0.011 -0.055emmeans(model_or_error1, ~ Difficulty*mentions_accuracy) %>%
pairs(., simple = "mentions_accuracy", reverse = TRUE)## Difficulty = -0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) 0.783 0.129 Inf 6.083 <.0001
##
## Difficulty = 0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) -0.167 0.123 Inf -1.361 0.1736
##
## Results are given on the log odds ratio (not the response) scale.model_or_error2 <- data %>%
filter(Validity == "FALSE") %>%
filter(Judgment != "Accuracy") %>%
mutate(Difficulty = ifelse(Difficulty == "Easy",-.5,.5)) %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 1,-.5,.5)) %>%
glmer(Error ~ Difficulty*mentions_accuracy + (1 + Difficulty | ID) + (1 | Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(model_or_error2)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: Error ~ Difficulty * mentions_accuracy + (1 + Difficulty | ID) +
## (1 | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 6081.9 6139.6 -3033.0 6065.9 10072
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.7178 -0.3389 -0.2148 -0.1296 7.7353
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 0.8205 0.9058
## Difficulty 0.2808 0.5299 0.15
## Item (Intercept) 1.3076 1.1435
## Number of obs: 10080, groups: ID, 280; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -2.8239 0.1563 -18.063 < 2e-16 ***
## Difficulty 0.2804 0.2919 0.961 0.33680
## mentions_accuracy 0.3699 0.1397 2.649 0.00808 **
## Difficulty:mentions_accuracy -1.0986 0.1692 -6.492 8.49e-11 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) Dffclt mntns_
## Difficulty 0.003
## mntns_ccrcy -0.147 0.043
## Dffclty:mn_ 0.061 -0.120 -0.069stdCoef.merMod(model_or_error2)## stdcoef stdse
## (Intercept) 0.0000000 0.0000000
## Difficulty 0.4384433 0.4564729
## mentions_accuracy 0.5568306 0.2102164
## Difficulty:mentions_accuracy -0.8590165 0.1323260emmeans(model_or_error2, ~ Difficulty*mentions_accuracy) %>%
pairs(., simple = "mentions_accuracy", reverse = TRUE)## Difficulty = -0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) 0.919 0.168 Inf 5.464 <.0001
##
## Difficulty = 0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) -0.179 0.158 Inf -1.134 0.2570
##
## Results are given on the log odds ratio (not the response) scale.# do people who mention accuracy as a strategy produce more correct answers at test?
model_or_correct1 <- data %>%
filter(Validity == "FALSE") %>%
mutate(Difficulty = ifelse(Difficulty == "Easy",.5,-.5)) %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 1,.5,-.5)) %>%
glmer(Correct ~ Difficulty*mentions_accuracy + (1 + Difficulty | ID) + (1 | Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(model_or_correct1)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: Correct ~ Difficulty * mentions_accuracy + (1 + Difficulty |
## ID) + (1 | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 14133.8 14194.8 -7058.9 14117.8 15112
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -6.6863 -0.4614 0.1744 0.4883 6.4469
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 1.6052 1.2669
## Difficulty 1.4038 1.1848 -0.50
## Item (Intercept) 0.5009 0.7077
## Number of obs: 15120, groups: ID, 420; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -0.03388 0.10752 -0.315 0.75267
## Difficulty 3.03146 0.18482 16.402 < 2e-16 ***
## mentions_accuracy 0.80640 0.13342 6.044 1.5e-09 ***
## Difficulty:mentions_accuracy 0.39640 0.15324 2.587 0.00969 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) Dffclt mntns_
## Difficulty -0.103
## mntns_ccrcy -0.048 0.034
## Dffclty:mn_ 0.055 -0.032 -0.385emmeans(model_or_correct1, ~ Difficulty*mentions_accuracy) %>%
pairs(., simple = "mentions_accuracy", reverse = TRUE)## Difficulty = -0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) 0.608 0.178 Inf 3.424 0.0006
##
## Difficulty = 0.5:
## contrast estimate SE df z.ratio p.value
## 0.5 - (-0.5) 1.005 0.126 Inf 7.993 <.0001
##
## Results are given on the log odds ratio (not the response) scale.model_or_correct2 <- data %>%
filter(Validity == "FALSE") %>%
filter(Judgment != "Accuracy") %>%
mutate(Difficulty = ifelse(Difficulty == "Easy",.5,-.5)) %>%
mutate(mentions_accuracy = ifelse(mentions_accuracy == 1,.5,-.5)) %>%
glmer(Correct ~ Difficulty*mentions_accuracy + (1 + Difficulty | ID) + (1 | Item), ., family = binomial, control = glmerControl(optimizer = "bobyqa", optCtrl=list(maxfun=2e5)))
summary(model_or_correct2)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula: Correct ~ Difficulty * mentions_accuracy + (1 + Difficulty |
## ID) + (1 | Item)
## Data: .
## Control: glmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
##
## AIC BIC logLik deviance df.resid
## 9565.5 9623.3 -4774.8 9549.5 10072
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -4.5804 -0.4773 0.1813 0.5019 7.2030
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## ID (Intercept) 1.6283 1.2760
## Difficulty 1.4138 1.1890 -0.55
## Item (Intercept) 0.4482 0.6695
## Number of obs: 10080, groups: ID, 280; Item, 72
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.1423 0.1170 1.216 0.224
## Difficulty 2.9258 0.1871 15.640 < 2e-16 ***
## mentions_accuracy 1.1388 0.1707 6.673 2.51e-11 ***
## Difficulty:mentions_accuracy 0.1254 0.1956 0.641 0.522
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) Dffclt mntns_
## Difficulty -0.150
## mntns_ccrcy 0.197 -0.021
## Dffclty:mn_ -0.025 0.139 -0.392Demographics (incl social media use)
# age
data %>%
summarise(mean(age))## # A tibble: 1 × 1
## `mean(age)`
## <dbl>
## 1 39.5# gender
data %>%
count(gender) %>%
mutate(n = n/72)## # A tibble: 4 × 2
## gender n
## <chr> <dbl>
## 1 Female 192
## 2 Male 226
## 3 Non-binary/third gender 1
## 4 Prefer not to answer 1# prop of sample who uses social media apps
data %>%
rename(social_use = social_media_use) %>%
mutate(use_twitter = ifelse(grepl("Twitter", social_use),1,0)) %>%
mutate(use_facebook = ifelse(grepl("Facebook", social_use),1,0)) %>%
mutate(use_reddit = ifelse(grepl("Reddit", social_use),1,0)) %>%
mutate(use_youtube = ifelse(grepl("Youtube", social_use),1,0)) %>%
mutate(use_snapchat = ifelse(grepl("Snapchat", social_use),1,0)) %>%
mutate(use_instagram = ifelse(grepl("Instagram", social_use),1,0)) %>%
mutate(use_pinterest = ifelse(grepl("Pinterest", social_use),1,0)) %>%
summarise(
across(use_twitter:use_pinterest, mean))## # A tibble: 1 × 7
## use_twitter use_facebook use_reddit use_youtube use_snapchat use_instagram
## <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 0.698 0.743 0.638 0.871 0.207 0.643
## # … with 1 more variable: use_pinterest <dbl># avg number of different apps used
data %>%
mutate(social_use_num = as.numeric((str_count(social_media_use, ',') + 1))) %>%
summarise(mean(social_use_num))## # A tibble: 1 × 1
## `mean(social_use_num)`
## <dbl>
## 1 4.15# count number of hours spent on social per day
data %>%
count(use_social) %>%
mutate(n = n/72) %>%
arrange(n) %>%
mutate(prop = n/420)## # A tibble: 9 × 3
## use_social n prop
## <chr> <dbl> <dbl>
## 1 5–6 hours per day 9 0.0214
## 2 More than 6 hours per day 9 0.0214
## 3 4–5 hours per day 16 0.0381
## 4 Less than 10 minutes per day 21 0.05
## 5 3–4 hours per day 34 0.0810
## 6 2–3 hours per day 60 0.143
## 7 10–30 minutes per day 68 0.162
## 8 31–60 minutes per day 88 0.210
## 9 1–2 hours per day 115 0.274