1 Load and clean data

read_sav("Qualitrics/FB Misinfo Stimuli Pilot_July 27, 2023_18.29.sav") |> filter(Q109 == "616fecdd127553593aac3134")
# Load data
data = read_sav("Qualitrics/FB Misinfo Stimuli Pilot_July 27, 2023_18.29.sav") |>
    rename(ProlificID = Q109) |>
    filter(Finished == 1, !is.na(ProlificID)) |> 
    filter(
        !(ResponseId  %in% c("R_1hFLVlyOvoORO7X", "R_3erVa4BDzEtxiWS"))
    )


cat("Number of finished participants:", data |> nrow(), "\n")
## Number of finished participants: 101
# Extract column labels
col_labels = lapply(data, function(x) attributes(x)$label)

col_labels = data.frame(
    Variable = names(data),
    Label = col_labels |> unlist() |> as.character()
    ) |> mutate(
    q_number = str_extract(Variable, "\\d+"))

col_labels = col_labels |> mutate(
    q_type = case_when(
        str_detect(Label, "share") ~ "share",
        str_detect(Label, "manipulative") ~ "manipulative",
        str_detect(Label, "reliable") ~ "reliable",
        str_detect(Label, "concerning") ~ "concerning",
        str_detect(Label, "might see") ~ "might_see",
        str_detect(Label, "easy to understand") ~ "easy_to_understand",
        str_detect(Label, "feedback") ~ "feedback"
    )
)
col_labels$q_type[is.na(col_labels$q_type)] = "other"


opinions = data[, (col_labels$Variable == "ResponseId") | (col_labels$q_type %in% c("share", "manipulative", "reliable", "misleading", "concerning", "might_see", "easy_to_understand"))]

# Convert labeled to numeric
opinions = opinions |> mutate_at(vars(-ResponseId), as.numeric)

# Pivot data
opinions = opinions |> pivot_longer(
    cols = -ResponseId,
    names_to = "Variable",
    values_to = "opinion"
)

# Merge with labels
opinions = opinions |> left_join(col_labels |> select(Variable, q_type, q_number), by = "Variable")

# Group variables into posts
opinions = opinions |> arrange(ResponseId, q_number) |>
    mutate(
        post_number = case_when(
            q_number %in% 255:260 ~ 1,
            q_number %in% 493:498 ~ 2,
            q_number %in% 507:512 ~ 3,
            q_number %in% 521:526 ~ 4,
            q_number %in% 535:540 ~ 5,
            q_number %in% 549:554 ~ 6,
            q_number %in% 563:568 ~ 7,
            q_number %in% 577:582 ~ 8,
            q_number %in% 591:596 ~ 9,
            q_number %in% 605:610 ~ 10,
            q_number %in% 619:624 ~ 11,
            q_number %in% 633:638 ~ 12,
            q_number %in% 647:652 ~ 13,
            q_number %in% 661:666 ~ 14,
            q_number %in% 675:680 ~ 15,
            q_number %in% 689:694 ~ 16,
            q_number %in% 703:708 ~ 17,
            q_number %in% 717:722 ~ 18,
            q_number %in% 731:736 ~ 19,
            q_number %in% 745:750 ~ 20,
            q_number %in% 759:764 ~ 21,
            q_number %in% 773:778 ~ 22,
            q_number %in% 787:792 ~ 23,
            q_number %in% 801:806 ~ 24,
            q_number %in% 815:820 ~ 25,
            q_number %in% 829:834 ~ 26,
            q_number %in% 843:848 ~ 27,
            q_number %in% 857:862 ~ 28,
            q_number %in% 871:876 ~ 29),
        post_type = case_when(
            post_number %in% 1:13 ~ "Non Misinfo",
            post_number %in% 14:18 ~ "Discrediting",
            post_number %in% 19:23 ~ "False Dichotomy",
            post_number %in% 24:29 ~ "Emotional")
    )

nice_labels = data.frame(
    variable = c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"),
    Variable = c("Manipulative", "Reliable", "Concerning", "Easy to Understand", "Might See", "Would Share")
)

post_types = opinions |> select(post_number, post_type) |> distinct()


# Select feedback variables

feedback = data[, (col_labels$Variable == "ResponseId") | (col_labels$q_type == "feedback")] 

feedback = feedback |>
    # Now pivot to long
    pivot_longer(
        cols = -ResponseId,
        names_to = "question",
        values_to = "feedback"
    ) |>
    # Merge with labels
    left_join(
        data.frame(
            post_number = 1:29,
            question = names(feedback)[2:30]
        )
    )
valid_feedback = feedback |> filter(feedback != "", feedback != "no feed back", feedback != "na") |> 
    filter(str_length(feedback) > 6) |>
    group_by(ResponseId) |>
    mutate(n_valid_feedback = n()) |> 
    ungroup() |>
    # Now merge post_type
    left_join(post_types, by = "post_number") 

valid_feedback |> select(ResponseId, post_number, feedback) |>
    arrange(post_number) |> 
    write_csv("Qualitrics/valid_feedback.csv")

valid_feedback |> filter(n_valid_feedback == max(n_valid_feedback)) |> 
    distinct(ResponseId, n_valid_feedback) |> 
    left_join(data |> select(ResponseId, ProlificID)) |>
    write_csv("Qualitrics/valid_feedback_ProlificIds.csv")
# Pivot data to wider by ResponseID+post_number
opinions = opinions |> 
    select(ResponseId, post_number, post_type, opinion, q_type) |>
    pivot_wider(
    id_cols = c(ResponseId, post_number, post_type),
    names_from = q_type,
    values_from = opinion
)

#recode variables

opinions = opinions |> mutate(
    share = case_when(
        share == 1 ~ 1,
        share == 2 ~ 0,
    ),
    reliable_orig = reliable,
    reliable = case_when(
        reliable_orig == 1 ~ 1,
        reliable_orig == 2 ~ 2,
        reliable_orig == 7 ~ 3,
        reliable_orig == 3 ~ 4,
        reliable_orig == 8 ~ 5,
        reliable_orig == 9 ~ 6,
        reliable_orig == 11 ~ 7,
    ),
    might_see_orig = might_see,
    might_see = case_when(
        might_see_orig == 1 ~ 1,
        might_see_orig == 4 ~ 0,
        might_see_orig == 2 ~ 0.5,
        might_see_orig == 3 ~ 0.5,
    )
) |> select(-reliable_orig, -might_see_orig) 

2 Questions list

Post number Screenshot Post Type
1 description of image Non-misinfo
2 description of image Non-misinfo
3 description of image Non-misinfo
4 description of image Non-misinfo
5 description of image Non-misinfo
6 description of image Non-misinfo
7 description of image Non-misinfo
8 description of image Non-misinfo
9 description of image Non-misinfo
10 description of image Non-misinfo
11 description of image Non-misinfo
12 description of image Non-misinfo
13 description of image Non-misinfo
14 description of image Discrediting
15 description of image Discrediting
16 description of image Discrediting
17 description of image Discrediting
18 description of image Discrediting
19 description of image False Dichotomy
20 description of image False Dichotomy
21 description of image False Dichotomy
22 description of image False Dichotomy
23 description of image False Dichotomy
24 description of image Emotional
25 description of image Emotional
26 description of image Emotional
27 description of image Emotional
28 description of image Emotional
29 description of image Emotional

3 Analyze the data

3.1 Descriptive statistics by post

# Calculate mean for each post and number of participants 
sum_stats_post = opinions |> 
    group_by(post_number, post_type) |> 
    summarise(
        across(c("share", "manipulative", "reliable", "concerning", "easy_to_understand", "might_see"), mean, na.r = TRUE),
        n = n()
    ) |>
    rename(
        Manipulative = manipulative,
        Reliable = reliable,
        Concerning = concerning,
        `Easy tounderstand` = easy_to_understand,
        `Might see` = might_see
    ) 

sum_stats_post |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
post_number post_type share Manipulative Reliable Concerning Easy tounderstand Might see n
1 Non Misinfo 0.2673267 4.138614 3.158416 5.524752 5.881188 0.4356436 101
2 Non Misinfo 0.5742574 2.891089 4.772277 5.297030 6.267327 0.7376238 101
3 Non Misinfo 0.2970297 3.178218 3.673267 4.851485 6.099010 0.5346535 101
4 Non Misinfo 0.3168317 3.762376 3.613861 4.891089 6.316832 0.4950495 101
5 Non Misinfo 0.4257426 3.544554 3.792079 5.079208 6.297030 0.5792079 101
6 Non Misinfo 0.6138614 3.128713 4.762376 4.801980 6.277228 0.7227723 101
7 Non Misinfo 0.3267327 2.673267 3.752475 3.188119 5.613861 0.4306931 101
8 Non Misinfo 0.7029703 2.603960 5.148515 4.861386 6.425743 0.7722772 101
9 Non Misinfo 0.5742574 3.554455 5.128713 5.267327 6.435644 0.6633663 101
10 Non Misinfo 0.4950495 3.138614 4.653465 4.910891 6.415842 0.6188119 101
11 Non Misinfo 0.5544554 3.217822 4.504950 5.524752 6.297030 0.6633663 101
12 Non Misinfo 0.2574257 3.108911 3.128713 2.613861 6.168317 0.6485149 101
13 Non Misinfo 0.3168317 3.861386 3.504951 5.297030 5.821782 0.5346535 101
14 Discrediting 0.4950495 3.990099 4.336634 5.128713 6.425743 0.6930693 101
15 Discrediting 0.2376238 3.960396 3.108911 4.643564 4.623762 0.4306931 101
16 Discrediting 0.1188119 5.386139 2.287129 4.920792 5.821782 0.4257426 101
17 Discrediting 0.3168317 4.405941 3.782178 5.475248 6.198020 0.6732673 101
18 Discrediting 0.6039604 3.386139 5.386139 5.227723 6.663366 0.8613861 101
19 False Dichotomy 0.2871287 3.118812 3.386139 3.178218 5.673267 0.6881188 101
20 False Dichotomy 0.1584158 5.574257 2.693069 4.435644 6.336634 0.7227723 101
21 False Dichotomy 0.3069307 5.742574 3.792079 5.227723 6.039604 0.7029703 101
22 False Dichotomy 0.1089109 5.336634 2.564356 5.356436 5.584158 0.6089109 101
23 False Dichotomy 0.4554455 3.920792 4.118812 5.772277 6.297030 0.6683168 101
24 Emotional 0.3960396 4.396040 3.405941 6.069307 6.178218 0.6980198 101
25 Emotional 0.4455446 2.960396 4.504950 6.435644 6.227723 0.7772277 101
26 Emotional 0.4653465 4.435644 3.811881 5.891089 6.415842 0.7475248 101
27 Emotional 0.1287129 5.900990 2.465346 5.118812 6.178218 0.6386139 101
28 Emotional 0.2673267 3.663366 3.495049 5.138614 6.158416 0.6633663 101
29 Emotional 0.4554455 4.079208 4.415842 5.574257 6.485149 0.7376238 101

3.2 Descriptive statistics by post-type

sum_stats_post = opinions |> 
    select(post_type, all_of(c("share", "manipulative", "reliable", "concerning", "easy_to_understand", "might_see"))) |>
    # now pivot long
    pivot_longer(cols = -post_type, names_to = "variable", values_to = "opinion") |>
    group_by(post_type, variable) |>
    # Compute mean, standard deviation, number non-missing, q25 and q75
    summarise(
        mean = mean(opinion, na.rm = TRUE),
        sd = sd(opinion, na.rm = TRUE),
        n = n(),
        q25 = quantile(opinion, probs = 0.25, na.rm = TRUE),
        q75 = quantile(opinion, probs = 0.75, na.rm = TRUE)
    )

sum_stats_post |> left_join(nice_labels, by.x = "variable") |> 
    select(
        `Post Type` = post_type, Variable, `Mean` = mean, `SD` = sd, `N` = n, q25, q75
    ) |>
    kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
Post Type Variable Mean SD N q25 q75
Discrediting Concerning 5.0792079 1.9944525 505 4.0 7
Discrediting Easy to Understand 5.9465347 1.6472418 505 5.0 7
Discrediting Manipulative 4.2257426 2.1868408 505 2.0 6
Discrediting Might See 0.6168317 0.3949255 505 0.5 1
Discrediting Reliable 3.7801980 2.0415340 505 2.0 5
Discrediting Would Share 0.3544554 0.4788223 505 0.0 1
Emotional Concerning 5.7046205 1.7731361 606 5.0 7
Emotional Easy to Understand 6.2739274 1.2626406 606 6.0 7
Emotional Manipulative 4.2392739 2.2157607 606 2.0 7
Emotional Might See 0.7103960 0.2470470 606 0.5 1
Emotional Reliable 3.6831683 2.0004827 606 2.0 5
Emotional Would Share 0.3597360 0.4803194 606 0.0 1
False Dichotomy Concerning 4.7940594 2.1760703 505 3.0 7
False Dichotomy Easy to Understand 5.9861386 1.6564586 505 5.0 7
False Dichotomy Manipulative 4.7386139 2.1846765 505 3.0 7
False Dichotomy Might See 0.6782178 0.2397105 505 0.5 1
False Dichotomy Reliable 3.3108911 1.9294039 505 1.0 5
False Dichotomy Would Share 0.2633663 0.4408962 505 0.0 1
Non Misinfo Concerning 4.7776085 2.0639660 1313 3.0 7
Non Misinfo Easy to Understand 6.1782178 1.2997118 1313 6.0 7
Non Misinfo Manipulative 3.2924600 2.0492591 1313 1.0 5
Non Misinfo Might See 0.6028180 0.3831628 1313 0.5 1
Non Misinfo Reliable 4.1226200 1.9089923 1313 3.0 6
Non Misinfo Would Share 0.4402133 0.4966018 1313 0.0 1

3.3 Distribution of Likert-Scale responses by outcome

bin_counts_post = opinions |> 
    group_by(post_number) |>
    select(all_of(c("manipulative", "reliable", "concerning", "easy_to_understand"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "var", values_to = "opinion") |> #now count responses by post_number and var and opinion
    group_by(post_number, var, opinion) |>
    summarise(n = n())  |> #now pivot to wide
    pivot_wider(names_from = opinion, values_from = n) |>
    `colnames<-`(c("post_number", "variable", "1 -- not at all", "2", "3", "4", "5", "6", "7 -- very")) |> 
    ungroup() |>
    left_join(nice_labels, by = "variable") |> # now replace NAs with 0
    mutate_at(vars(-c("post_number", "variable", "Variable")), ~replace_na(., 0)) |>
    left_join(post_types, by = "post_number") |> # now replace NAs with 0
    select(`Post Number` = post_number, `Post Type` = post_type, all_of(c("Variable", "1 -- not at all", "2", "3", "4", "5", "6", "7 -- very")))

3.3.1 Manipulative

bin_counts_post |> filter(Variable == "Manipulative") |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>% scroll_box(height = "100%")
Post Number Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
1 Non Misinfo Manipulative 18 8 12 19 12 12 20
2 Non Misinfo Manipulative 38 16 8 14 15 3 7
3 Non Misinfo Manipulative 29 11 17 22 9 5 8
4 Non Misinfo Manipulative 26 8 8 22 12 9 16
5 Non Misinfo Manipulative 26 11 11 21 12 7 13
6 Non Misinfo Manipulative 32 15 17 11 8 7 11
7 Non Misinfo Manipulative 41 16 14 12 8 3 7
8 Non Misinfo Manipulative 48 13 8 11 10 6 5
9 Non Misinfo Manipulative 23 17 8 17 15 12 9
10 Non Misinfo Manipulative 39 8 9 16 11 10 8
11 Non Misinfo Manipulative 31 11 15 15 17 2 10
12 Non Misinfo Manipulative 35 15 6 23 5 5 12
13 Non Misinfo Manipulative 19 9 9 25 20 7 12
14 Discrediting Manipulative 22 8 5 20 23 6 17
15 Discrediting Manipulative 22 5 14 15 20 9 16
16 Discrediting Manipulative 8 4 3 15 10 18 43
17 Discrediting Manipulative 13 12 6 20 12 16 22
18 Discrediting Manipulative 36 11 8 11 9 11 15
19 False Dichotomy Manipulative 36 12 9 17 10 9 8
20 False Dichotomy Manipulative 6 4 4 8 15 18 46
21 False Dichotomy Manipulative 5 4 5 5 16 10 56
22 False Dichotomy Manipulative 6 5 11 8 11 17 43
23 False Dichotomy Manipulative 20 10 11 20 12 13 15
24 Emotional Manipulative 16 7 7 22 12 14 23
25 Emotional Manipulative 40 10 15 12 10 2 12
26 Emotional Manipulative 16 10 6 18 12 11 28
27 Emotional Manipulative 3 0 7 8 13 15 55
28 Emotional Manipulative 25 11 10 15 19 9 12
29 Emotional Manipulative 19 7 18 14 12 8 23

3.3.2 Reliable

bin_counts_post |> filter(Variable == "Reliable") |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>% scroll_box(height = "100%")
Post Number Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
1 Non Misinfo Reliable 25 16 18 23 7 3 9
2 Non Misinfo Reliable 5 7 12 19 19 17 22
3 Non Misinfo Reliable 17 13 13 25 16 10 7
4 Non Misinfo Reliable 24 7 18 23 8 6 15
5 Non Misinfo Reliable 15 8 19 31 7 11 10
6 Non Misinfo Reliable 8 4 8 25 17 17 22
7 Non Misinfo Reliable 15 10 15 27 20 7 7
8 Non Misinfo Reliable 5 0 9 18 28 11 30
9 Non Misinfo Reliable 5 2 12 19 14 16 33
10 Non Misinfo Reliable 7 6 9 25 19 16 19
11 Non Misinfo Reliable 9 7 11 22 20 13 19
12 Non Misinfo Reliable 26 19 13 20 9 10 4
13 Non Misinfo Reliable 16 14 16 27 18 6 4
14 Discrediting Reliable 10 7 15 26 12 12 19
15 Discrediting Reliable 27 12 21 21 10 4 6
16 Discrediting Reliable 52 11 13 13 7 4 1
17 Discrediting Reliable 18 10 13 23 18 10 9
18 Discrediting Reliable 1 6 7 15 20 14 38
19 False Dichotomy Reliable 25 11 13 29 9 3 11
20 False Dichotomy Reliable 35 18 16 18 6 5 3
21 False Dichotomy Reliable 17 11 16 21 14 12 10
22 False Dichotomy Reliable 45 13 14 13 6 6 4
23 False Dichotomy Reliable 14 9 10 25 16 15 12
24 Emotional Reliable 25 13 12 22 13 8 8
25 Emotional Reliable 4 9 14 23 21 16 14
26 Emotional Reliable 23 8 9 24 14 8 15
27 Emotional Reliable 50 16 12 6 3 6 8
28 Emotional Reliable 17 19 12 25 10 14 4
29 Emotional Reliable 12 7 8 24 19 12 19

3.3.3 Concerning

bin_counts_post |> filter(Variable == "Concerning") |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>% scroll_box(height = "100%")
Post Number Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
1 Non Misinfo Concerning 7 1 5 9 20 15 44
2 Non Misinfo Concerning 5 4 8 16 13 16 39
3 Non Misinfo Concerning 8 2 13 13 26 16 23
4 Non Misinfo Concerning 14 4 4 18 13 13 35
5 Non Misinfo Concerning 8 4 8 17 17 9 38
6 Non Misinfo Concerning 11 6 4 20 18 14 28
7 Non Misinfo Concerning 27 21 13 11 13 7 9
8 Non Misinfo Concerning 9 9 6 16 15 15 31
9 Non Misinfo Concerning 7 5 4 13 19 15 38
10 Non Misinfo Concerning 10 5 10 11 19 15 31
11 Non Misinfo Concerning 4 3 6 11 17 19 41
12 Non Misinfo Concerning 44 15 13 10 9 4 6
13 Non Misinfo Concerning 5 5 6 15 14 20 36
14 Discrediting Concerning 13 3 4 11 12 23 35
15 Discrediting Concerning 12 4 6 28 13 12 26
16 Discrediting Concerning 12 5 9 11 15 14 35
17 Discrediting Concerning 6 4 3 10 21 14 43
18 Discrediting Concerning 10 2 9 12 11 15 42
19 False Dichotomy Concerning 32 15 10 18 8 9 9
20 False Dichotomy Concerning 19 5 9 13 16 13 26
21 False Dichotomy Concerning 7 6 7 10 17 15 39
22 False Dichotomy Concerning 12 2 1 11 17 13 45
23 False Dichotomy Concerning 6 0 7 5 12 21 50
24 Emotional Concerning 4 0 1 6 15 18 57
25 Emotional Concerning 2 1 1 4 3 18 72
26 Emotional Concerning 4 4 2 7 11 17 56
27 Emotional Concerning 14 3 9 6 12 13 44
28 Emotional Concerning 9 2 7 14 19 16 34
29 Emotional Concerning 6 4 1 10 15 24 41

3.3.4 Easy to understand

bin_counts_post |> filter(Variable == "Easy to Understand") |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>% scroll_box(height = "100%")
Post Number Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
1 Non Misinfo Easy to Understand 3 1 6 8 13 16 54
2 Non Misinfo Easy to Understand 1 1 4 4 10 15 66
3 Non Misinfo Easy to Understand 1 2 2 4 19 17 56
4 Non Misinfo Easy to Understand 0 1 3 6 9 16 66
5 Non Misinfo Easy to Understand 1 1 2 5 10 17 65
6 Non Misinfo Easy to Understand 2 2 1 5 9 14 68
7 Non Misinfo Easy to Understand 3 1 9 8 18 21 41
8 Non Misinfo Easy to Understand 0 1 4 3 5 18 70
9 Non Misinfo Easy to Understand 1 1 0 3 8 21 67
10 Non Misinfo Easy to Understand 0 1 2 3 10 17 68
11 Non Misinfo Easy to Understand 2 0 1 4 11 21 62
12 Non Misinfo Easy to Understand 1 3 1 6 12 17 61
13 Non Misinfo Easy to Understand 2 0 6 11 17 16 49
14 Discrediting Easy to Understand 2 0 2 3 8 13 73
15 Discrediting Easy to Understand 14 6 12 12 14 14 29
16 Discrediting Easy to Understand 4 4 3 11 7 16 56
17 Discrediting Easy to Understand 1 2 4 2 11 21 60
18 Discrediting Easy to Understand 0 0 0 1 10 11 79
19 False Dichotomy Easy to Understand 9 0 5 8 11 14 54
20 False Dichotomy Easy to Understand 2 0 3 4 9 13 70
21 False Dichotomy Easy to Understand 6 3 1 5 6 15 65
22 False Dichotomy Easy to Understand 5 7 3 8 15 12 51
23 False Dichotomy Easy to Understand 2 2 2 1 12 14 68
24 Emotional Easy to Understand 2 2 2 5 8 22 60
25 Emotional Easy to Understand 3 0 3 3 10 19 63
26 Emotional Easy to Understand 2 0 0 6 7 15 71
27 Emotional Easy to Understand 3 1 2 4 12 16 63
28 Emotional Easy to Understand 2 0 1 4 21 15 58
29 Emotional Easy to Understand 1 0 2 1 10 15 72

3.4 Distribution of Likert-Scale responses by post-type

bin_counts = opinions |> 
    group_by(post_type) |>
    select(all_of(c("manipulative", "reliable", "concerning", "easy_to_understand"))) |> #now pivot to long
    pivot_longer(cols = -post_type, names_to = "var", values_to = "opinion") |> #now count responses by post_type and var and opinion
    group_by(post_type, var, opinion) |>
    summarise(n = n())  |> #now pivot to wide
    pivot_wider(names_from = opinion, values_from = n) |>
    `colnames<-`(c("Post Type", "variable", "1 -- not at all", "2", "3", "4", "5", "6", "7 -- very")) |> 
    ungroup() |> 
    left_join(nice_labels, by = "variable") |>
    select(all_of(c("Post Type", "Variable", "1 -- not at all", "2", "3", "4", "5", "6", "7 -- very")))

bin_counts |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
Discrediting Concerning 53 18 31 72 72 78 181
Discrediting Easy to Understand 21 12 21 29 50 75 297
Discrediting Manipulative 101 40 36 81 74 60 113
Discrediting Reliable 108 46 69 98 67 44 73
Emotional Concerning 39 14 21 47 75 106 304
Emotional Easy to Understand 13 3 10 23 68 102 387
Emotional Manipulative 119 45 63 89 78 59 153
Emotional Reliable 131 72 67 124 80 64 68
False Dichotomy Concerning 76 28 34 57 70 71 169
False Dichotomy Easy to Understand 24 12 14 26 53 68 308
False Dichotomy Manipulative 73 35 40 58 64 67 168
False Dichotomy Reliable 136 62 69 106 51 41 40
Non Misinfo Concerning 159 84 100 180 213 178 399
Non Misinfo Easy to Understand 17 15 41 70 151 226 793
Non Misinfo Manipulative 405 158 142 228 154 88 138
Non Misinfo Reliable 177 113 173 304 202 143 201

3.5 Corelations between variables

# Calculate bivariate correlations and present as a heatmap using ggplot
# Rename variables to more readable names
opinions |> 
    select(
        manipulative, reliable, concerning, easy_to_understand, might_see, share
    ) |> 
    na.omit() |>
    cor(use = "pairwise.complete.obs") |>
    as.data.frame() |>
    rename(
        Manipulative = manipulative,
        Reliable = reliable,
        Concerning = concerning,
        `Easy tounderstand` = easy_to_understand,
        `Might see` = might_see
    ) |>
    rownames_to_column("var1") |>
    pivot_longer(cols = -var1, names_to = "var2", values_to = "cor") |>
    ggplot(aes(x = var1, y = var2, fill = cor)) +
    geom_tile() +
    geom_text(aes(label = round(cor, 2)), size = 3) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust = 1)) +
    labs(x = "", y = "", title = "Correlations between variables")

3.5.1 Some observations

  1. Manipulative posts are harder to understand and more concerning but (understandably) seen as less reliable and, less likely to be seen on one’s timeline and less likely to be shared
  2. Sharing intention most correlated with reliability and likelihood of seeing the post on one’s timeline.

3.5.2 Covariate heatmap by post type

mean_diffs = opinions |>
    select(post_type, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_type, names_to = "variable", values_to = "opinion") |>
    # calculate grand means by variable and by variable-post_type
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_type) |>
    mutate(group_mean = mean(opinion, na.rm = TRUE)) |>
    # group diferences from grand mean
    mutate(group_diff = group_mean - grand_mean) |>
    # now pivot to wide
    distinct(
        variable, post_type, group_diff
    ) |>
    pivot_wider(names_from = post_type, values_from = group_diff) |>
    left_join(nice_labels, by = "variable") |> ungroup() |>
    select(Variable, `Non Misinfo`, Discrediting, `False Dichotomy`, Emotional)

# Plot heatmap
mean_diffs |> 
    pivot_longer(cols = -Variable, names_to = "Post Type", values_to = "Difference") |>
    ggplot(aes(x = Variable, y = `Post Type`, fill = Difference)) +
    geom_tile() +
    geom_text(aes(label = round(Difference, 2)), size = 3) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(x = "", y = "", title = "Differences from grand mean by post type")

3.5.3 Coviariate heatmap by post (within post type)

3.5.3.1 Non-misinfo

opinions |>
    filter(post_type == "Non Misinfo") |>
    select(post_number, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "variable", values_to = "opinion") |>
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_number) |>
    mutate(group_mean = mean(opinion, na.rm = TRUE)) |>
    mutate(group_diff = group_mean - grand_mean) |>
    distinct(variable, post_number, group_diff) |>
    pivot_wider(names_from = post_number, values_from = group_diff) |>
    left_join(nice_labels, by = "variable") |> ungroup() |>
    select(-variable) |> 
    pivot_longer(cols = -Variable, names_to = "Post Number", values_to = "Difference", names_prefix = "Non Misinfo.") |>
    ggplot(aes(y = Variable, x = `Post Number`, fill = Difference)) +
    geom_tile() +
    geom_text(aes(label = round(Difference, 2)), size = 3) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(y = "Feeling", x = "Question Number", title = "Differences from Post-Type group mean by post number, Non-Misinfo")

3.5.3.2 Discrediting

opinions |>
    filter(post_type == "Discrediting") |>
    select(post_number, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "variable", values_to = "opinion") |>
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_number) |>
    mutate(group_mean = mean(opinion, na.rm = TRUE)) |>
    mutate(group_diff = group_mean - grand_mean) |>
    distinct(variable, post_number, group_diff) |>
    pivot_wider(names_from = post_number, values_from = group_diff) |>
    left_join(nice_labels, by = "variable") |> ungroup() |>
    select(-variable) |> 
    pivot_longer(cols = -Variable, names_to = "Post Number", values_to = "Difference") |>
    ggplot(aes(y = Variable, x = `Post Number`, fill = Difference)) +
    geom_tile() +
    geom_text(aes(label = round(Difference, 2)), size = 3) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(y = "Feeling", x = "Question Number", title = "Differences from Post-Type group mean by post number, Discrediting")

3.5.3.3 False Dichotomy

opinions |>
    filter(post_type == "False Dichotomy") |>
    select(post_number, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "variable", values_to = "opinion") |>
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_number) |>
    mutate(group_mean = mean(opinion, na.rm = TRUE)) |>
    mutate(group_diff = group_mean - grand_mean) |>
    distinct(variable, post_number, group_diff) |>
    pivot_wider(names_from = post_number, values_from = group_diff) |>
    left_join(nice_labels, by = "variable") |> ungroup() |>
    select(-variable) |> 
    pivot_longer(cols = -Variable, names_to = "Post Number", values_to = "Difference") |>
    ggplot(aes(y = Variable, x = `Post Number`, fill = Difference)) +
    geom_tile() +
    geom_text(aes(label = round(Difference, 2)), size = 3) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(y = "Feeling", x = "Question Number", title = "Differences from Post-Type group mean by post number, False Dichotomy")

3.5.3.4 Emotional

opinions |>
    filter(post_type == "Emotional") |>
    select(post_number, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "variable", values_to = "opinion") |>
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_number) |>
    mutate(group_mean = mean(opinion, na.rm = TRUE)) |>
    mutate(group_diff = group_mean - grand_mean) |>
    distinct(variable, post_number, group_diff) |>
    pivot_wider(names_from = post_number, values_from = group_diff) |>
    left_join(nice_labels, by = "variable") |> ungroup() |>
    select(-variable) |> 
    pivot_longer(cols = -Variable, names_to = "Post Number", values_to = "Difference") |>
    ggplot(aes(y = Variable, x = `Post Number`, fill = Difference)) +
    geom_tile() +
    geom_text(aes(label = round(Difference, 2)), size = 3) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(y = "Feeling", x = "Question Number", title = "Differences from Post-Type group mean by post number, Emotional")

3.6 Regressions of intention to share on other variables

# Run regressions of each variables individually on share and all jointly
library(fixest)

mods = list()
for(var in c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see")){
    mods[[var]] = feols(as.formula(paste0("share ~ ", var)), data = opinions)
}

mods[["all"]] = feols(share ~ manipulative + reliable + concerning + easy_to_understand + might_see, data = opinions)

#rename mods in list to (1), (2), etc
names(mods) = paste0("(", 1:length(mods), ")")

etable(mods, cluster = "ResponseId", digits = 3,
    headers = rep("Share?", length(mods)),
    depvar = FALSE,
    drop = "Constant",
    dict = c(
        manipulative = "Manipulative?",
        reliable = "Reliable?",
        concerning = "Concerning?",
        easy_to_understand = "Easy to understand?",
        might_see = "Might see?"
    )
) |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
Share? Share? Share? Share? Share? Share?
Manipulative? -0.069*** (0.006) -0.033*** (0.005)
Reliable? 0.131*** (0.006) 0.099*** (0.007)
Concerning? 0.043*** (0.007) 0.020*** (0.004)
Easy to understand? 0.064*** (0.009) 0.004 (0.007)
Might see? 0.516*** (0.040) 0.317*** (0.037)
___________________ _________________ ________________ ________________ ________________ ________________ _________________
S.E.: Clustered by: ResponseId by: ResponseId by: ResponseId by: ResponseId by: ResponseId by: ResponseId
Observations 2,929 2,929 2,929 2,929 2,929 2,929
R2 0.09751 0.28674 0.03238 0.03556 0.13237 0.36160
Adj. R2 0.09720 0.28650 0.03204 0.03523 0.13207 0.36051

3.6.1 Takeaways

  1. Stated reliability and post being similar to something one might see on their timelines have strong (and positive) associations with the intentions to share
  2. The remaining variables (easy to understand, manipulative, concerning) have much weaker correlations with the intention to share.
  3. Manipulativeness is the only variable with a negative association with the intention to share.

3.7 Textual feedback

Takeaways: 1. A lot of people say they agree with true posts but some say that they know nothing about given topics and it would not draw their attention 2. On misinfo posts a lot of people comment as: “fake news”, “lie” “crazy” 3. Some people agree with some misinfo posts: “Media lies!”, “Interesting and good question”, “The post raises eyebrows about the reliability of nutritionists.”, “it is sad but it truth” 4. On style: “I like this post because its to the point.” “arrogant pster”, “sad; but vague and unhelpful too”, “Humorous post.”

Specific Posts:

  • Post 18 (deemed very reliable) – “facts -poliiticians do not care”, “politicians are the biggest criminals”, “Unfortunately, a large number of the politicians never deliver their promised projects. It happens all over the world!” – Notice these comments are not about reliability of the post but just people agreeing with the sentiment of the post
  • Post 27 (deemed very maniulative and unreliable) – “This could be propaganda.”, “I learnt something new.”, “Antibiotics work wonders”, “antibiotics are safe” – People mention that the post is manipulative but also just express the opinion that they disagree with the substance (i.e. they don’t think antibiotics are toxic)
  • Post 23 (very reliable and concerning) – “There is truth in this post” “This is just someone venting about lack of help for low income countries but not harmful.”, “the figure makes me trust the post”, “With statistical posts, proof should be provided. That said, if I were to believe the post’s claim, I would get angry on behalf of low-income nations.”, “nothing new, the rich will always come first” – Some people took the number for granted but a lot question the figure and would like to see a source. Still, it seems like a well-written post with a number in it seems more reliable.

3.7.1 Word Clouds

3.7.1.1 Non-misinfo

# Tokenize feedback
tokens <- valid_feedback %>%
    filter(post_type == "Non Misinfo") %>%
    select(feedback) %>%
    unnest_tokens(word, feedback) |>
    anti_join(stop_words)

# Count word occurrences and select top 100
word_counts <- tokens %>%
    count(word, sort = TRUE)|>
    filter(word!= "post") |>
    slice(1:50) 
# Create word cloud
wordcloud::wordcloud(words = word_counts$word, freq = word_counts$n, max.words = 50,
    random.order = FALSE, rot.per = 0.35, colors = brewer.pal(8, "Dark2"))

3.7.1.2 Discrediting

# Tokenize feedback
tokens <- valid_feedback %>%
    filter(post_type == "Discrediting") %>%
    select(feedback) %>%
    unnest_tokens(word, feedback) |>
    anti_join(stop_words)

# Count word occurrences and select top 100
word_counts <- tokens %>%
    count(word, sort = TRUE)|>
    filter(word!= "post") |>
    slice(1:50)


# Create word cloud
wordcloud::wordcloud(words = word_counts$word, freq = word_counts$n, max.words = 50,
    random.order = FALSE, rot.per = 0.35, colors = brewer.pal(8, "Dark2"))

3.7.1.3 Emotional

# Tokenize feedback
tokens <- valid_feedback %>%
    filter(post_type == "Emotional") %>%
    select(feedback) %>%
    unnest_tokens(word, feedback) |>
    anti_join(stop_words)

# Count word occurrences and select top 100
word_counts <- tokens %>%
    count(word, sort = TRUE) |>
    filter(word!= "post")|>
    slice(1:50)

# Create word cloud
wordcloud::wordcloud(words = word_counts$word, freq = word_counts$n, max.words = 50,
    random.order = FALSE, rot.per = 0.35, colors = brewer.pal(8, "Dark2"))

3.7.1.4 False Dichotomy

# Tokenize feedback
tokens <- valid_feedback %>%
    filter(post_type == "False Dichotomy") %>%
    select(feedback) %>%
    unnest_tokens(word, feedback) |>
    anti_join(stop_words)

# Count word occurrences and select top 100
word_counts <- tokens %>%
    count(word, sort = TRUE) |>
    filter(word!= "post") |>
    slice(1:50)

# Create word cloud
wordcloud::wordcloud(words = word_counts$word, freq = word_counts$n, max.words = 50,
    random.order = FALSE, rot.per = 0.35, colors = brewer.pal(8, "Dark2"))