1 Load and clean data

new_path = "Qualitrics/FB Misinfo Stimuli Pilot_July 31, 2023_16.34.sav"
#old_path = "Qualitrics/FB Misinfo Stimuli Pilot_July 27, 2023_18.29.sav"

#old_data = read_sav(old_path) |> mutate(old_data = TRUE)
#new_data = read_sav(new_path) |> mutate(new_data = TRUE)

#dt = merge(old_data, new_data, all = TRUE) |>
#    mutate(across(c(old_data, new_data), ~ifelse(is.na(.x), FALSE, .x)))

# Load data
data = read_sav(new_path) |>
    rename(ProlificID = Q109) |>
    filter(Finished == 1, !is.na(ProlificID)) |> 
    filter(
        !(ResponseId  %in% c("R_1hFLVlyOvoORO7X", "R_3erVa4BDzEtxiWS"))
    )


cat("Number of finished participants:", data |> nrow(), "\n")
## Number of finished participants: 103
# Extract column labels
col_labels = lapply(data, function(x) attributes(x)$label)

col_labels = data.frame(
    Variable = names(data),
    Label = col_labels |> unlist() |> as.character()
    ) |> mutate(
    q_number = str_extract(Variable, "\\d+"))

col_labels = col_labels |> mutate(
    q_type = case_when(
        str_detect(Label, "share") ~ "share",
        str_detect(Label, "manipulative") ~ "manipulative",
        str_detect(Label, "reliable") ~ "reliable",
        str_detect(Label, "concerning") ~ "concerning",
        str_detect(Label, "might see") ~ "might_see",
        str_detect(Label, "easy to understand") ~ "easy_to_understand",
        str_detect(Label, "feedback") ~ "feedback"
    )
)
col_labels$q_type[is.na(col_labels$q_type)] = "other"


opinions = data[, (col_labels$Variable == "ResponseId") | (col_labels$q_type %in% c("share", "manipulative", "reliable", "misleading", "concerning", "might_see", "easy_to_understand"))]

# Convert labeled to numeric
opinions = opinions |> mutate_at(vars(-ResponseId), as.numeric)

# Pivot data
opinions = opinions |> pivot_longer(
    cols = -ResponseId,
    names_to = "Variable",
    values_to = "opinion"
)

# Merge with labels
opinions = opinions |> left_join(col_labels |> select(Variable, q_type, q_number), by = "Variable")

# Group variables into posts
opinions = opinions |> arrange(ResponseId, q_number) |>
    mutate(
        post_number = case_when(
            q_number %in% 255:260 ~ 1,
            q_number %in% 493:498 ~ 2,
            q_number %in% 507:512 ~ 3,
            q_number %in% 521:526 ~ 4,
            q_number %in% 535:540 ~ 5,
            q_number %in% 549:554 ~ 6,
            q_number %in% 563:568 ~ 7,
            q_number %in% 577:582 ~ 8,
            q_number %in% 591:596 ~ 9,
            q_number %in% 605:610 ~ 10,
            q_number %in% 619:624 ~ 11,
            q_number %in% 633:638 ~ 12,
            q_number %in% 647:652 ~ 13,
            q_number %in% 661:666 ~ 14,
            q_number %in% 675:680 ~ 15,
            q_number %in% 689:694 ~ 16,
            q_number %in% 703:708 ~ 17,
            q_number %in% 717:722 ~ 18,
            q_number %in% 731:736 ~ 19,
            q_number %in% 745:750 ~ 20,
            q_number %in% 759:764 ~ 21,
            q_number %in% 773:778 ~ 22,
            q_number %in% 787:792 ~ 23,
            q_number %in% 801:806 ~ 24,
            q_number %in% 815:820 ~ 25,
            q_number %in% 829:834 ~ 26,
            q_number %in% 843:848 ~ 27,
            q_number %in% 857:862 ~ 28,
            q_number %in% 871:876 ~ 29),
        post_type = case_when(
            post_number %in% 1:13 ~ "Non Misinfo",
            post_number %in% 14:18 ~ "Discrediting",
            post_number %in% 19:23 ~ "False Dichotomy",
            post_number %in% 24:29 ~ "Emotional")
    )

nice_labels = data.frame(
    variable = c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"),
    Variable = c("Manipulative", "Reliable", "Concerning", "Easy to Understand", "Might See", "Would Share")
)

post_types = opinions |> select(post_number, post_type) |> distinct()


# Select feedback variables

feedback = data[, (col_labels$Variable == "ResponseId") | (col_labels$q_type == "feedback")] 

feedback = feedback |>
    # Now pivot to long
    pivot_longer(
        cols = -ResponseId,
        names_to = "question",
        values_to = "feedback"
    ) |>
    # Merge with labels
    left_join(
        data.frame(
            post_number = 1:29,
            question = names(feedback)[2:30]
        )
    )
valid_feedback = feedback |> filter(feedback != "", feedback != "no feed back", feedback != "na") |> 
    filter(str_length(feedback) > 6) |>
    group_by(ResponseId) |>
    mutate(n_valid_feedback = n()) |> 
    ungroup() |>
    # Now merge post_type
    left_join(post_types, by = "post_number") 

valid_feedback |> select(ResponseId, post_number, feedback) |>
    arrange(post_number) |> 
    write_csv("Qualitrics/valid_feedback.csv")

valid_feedback |> filter(n_valid_feedback == max(n_valid_feedback)) |> 
    distinct(ResponseId, n_valid_feedback) |> 
    left_join(data |> select(ResponseId, ProlificID)) |>
    write_csv("Qualitrics/valid_feedback_ProlificIds.csv")
# Pivot data to wider by ResponseID+post_number
opinions = opinions |> 
    select(ResponseId, post_number, post_type, opinion, q_type) |>
    pivot_wider(
    id_cols = c(ResponseId, post_number, post_type),
    names_from = q_type,
    values_from = opinion
)

#recode variables

opinions = opinions |> mutate(
    share = case_when(
        share == 1 ~ 1,
        share == 2 ~ 0,
    ),
    reliable_orig = reliable,
    reliable = case_when(
        reliable_orig == 1 ~ 1,
        reliable_orig == 2 ~ 2,
        reliable_orig == 7 ~ 3,
        reliable_orig == 3 ~ 4,
        reliable_orig == 8 ~ 5,
        reliable_orig == 9 ~ 6,
        reliable_orig == 11 ~ 7,
    ),
    might_see_orig = might_see,
    might_see = case_when(
        might_see_orig == 1 ~ 1,
        might_see_orig == 4 ~ 0,
        might_see_orig == 2 ~ 0.5,
        might_see_orig == 3 ~ 0.5,
    )
) |> select(-reliable_orig, -might_see_orig) 

2 Questions list

Post number Screenshot Post Type
1 description of image Non-misinfo
2 description of image Non-misinfo
3 description of image Non-misinfo
4 description of image Non-misinfo
5 description of image Non-misinfo
6 description of image Non-misinfo
7 description of image Non-misinfo
8 description of image Non-misinfo
9 description of image Non-misinfo
10 description of image Non-misinfo
11 description of image Non-misinfo
12 description of image Non-misinfo
13 description of image Non-misinfo
14 description of image Discrediting
15 description of image Discrediting
16 description of image Discrediting
17 description of image Discrediting
18 description of image Discrediting
19 description of image False Dichotomy
20 description of image False Dichotomy
21 description of image False Dichotomy
22 description of image False Dichotomy
23 description of image False Dichotomy
24 description of image Emotional
25 description of image Emotional
26 description of image Emotional
27 description of image Emotional
28 description of image Emotional
29 description of image Emotional

3 Analyze the data

3.1 Descriptive statistics by post

# Calculate mean for each post and number of participants 
sum_stats_post = opinions |> 
    group_by(post_number, post_type) |> 
    summarise(
        across(c("share", "manipulative", "reliable", "concerning", "easy_to_understand", "might_see"), mean, na.r = TRUE),
        n = n()
    ) |> #now round to 2 decimals
    mutate(across(c("share", "manipulative", "reliable", "concerning", "easy_to_understand", "might_see"), round, 2)) |>
    rename(
        Manipulative = manipulative,
        Reliable = reliable,
        Concerning = concerning,
        `Easy tounderstand` = easy_to_understand,
        `Might see` = might_see
    ) 

sum_stats_post |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
post_number post_type share Manipulative Reliable Concerning Easy tounderstand Might see n
1 Non Misinfo 0.27 4.15 3.17 5.51 5.83 0.44 103
2 Non Misinfo 0.58 2.89 4.76 5.32 6.26 0.74 103
3 Non Misinfo 0.31 3.16 3.72 4.89 6.12 0.54 103
4 Non Misinfo 0.31 3.73 3.60 4.85 6.32 0.49 103
5 Non Misinfo 0.43 3.57 3.79 5.12 6.30 0.57 103
6 Non Misinfo 0.60 3.13 4.77 4.80 6.26 0.72 103
7 Non Misinfo 0.33 2.66 3.76 3.18 5.59 0.44 103
8 Non Misinfo 0.70 2.59 5.17 4.85 6.41 0.77 103
9 Non Misinfo 0.56 3.60 5.10 5.29 6.41 0.67 103
10 Non Misinfo 0.50 3.12 4.65 4.91 6.41 0.62 103
11 Non Misinfo 0.55 3.22 4.48 5.53 6.30 0.66 103
12 Non Misinfo 0.26 3.11 3.13 2.66 6.12 0.65 103
13 Non Misinfo 0.31 3.85 3.52 5.32 5.80 0.54 103
14 Discrediting 0.50 4.02 4.35 5.17 6.44 0.69 103
15 Discrediting 0.23 3.97 3.13 4.66 4.63 0.43 103
16 Discrediting 0.12 5.38 2.28 4.90 5.82 0.43 103
17 Discrediting 0.32 4.42 3.78 5.49 6.16 0.67 103
18 Discrediting 0.60 3.38 5.39 5.22 6.66 0.86 103
19 False Dichotomy 0.28 3.14 3.36 3.19 5.64 0.69 103
20 False Dichotomy 0.16 5.56 2.71 4.46 6.34 0.72 103
21 False Dichotomy 0.31 5.68 3.83 5.25 6.06 0.70 103
22 False Dichotomy 0.12 5.35 2.59 5.35 5.55 0.61 103
23 False Dichotomy 0.46 3.88 4.15 5.79 6.29 0.67 103
24 Emotional 0.40 4.44 3.40 6.09 6.14 0.69 103
25 Emotional 0.45 2.96 4.51 6.45 6.24 0.78 103
26 Emotional 0.48 4.41 3.83 5.91 6.43 0.75 103
27 Emotional 0.13 5.91 2.46 5.13 6.18 0.64 103
28 Emotional 0.27 3.67 3.52 5.17 6.16 0.67 103
29 Emotional 0.47 4.04 4.47 5.59 6.49 0.74 103

3.2 Descriptive statistics by post-type

opinions |> 
    select(post_type, all_of(c("share", "manipulative", "reliable", "concerning", "easy_to_understand", "might_see"))) |>
    # now pivot long
    pivot_longer(cols = -post_type, names_to = "variable", values_to = "opinion") |>
    group_by(post_type, variable) |>
    # Compute mean, standard deviation, number non-missing, q25 and q75
    summarise(
        mean = mean(opinion, na.rm = TRUE),
        sd = sd(opinion, na.rm = TRUE),
        n = n(),
        q25 = quantile(opinion, probs = 0.25, na.rm = TRUE),
        q75 = quantile(opinion, probs = 0.75, na.rm = TRUE)
    ) |> #now round to 2 decimals
    mutate(across(c("mean", "sd", "q25", "q75"), round, 2)) |>
    left_join(nice_labels, by.x = "variable") |> 
    select(
        `Post Type` = post_type, Variable, `Mean` = mean, `SD` = sd, `N` = n, q25, q75
    ) |>
    kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
Post Type Variable Mean SD N q25 q75
Discrediting Concerning 5.09 1.99 515 4.0 7.00
Discrediting Easy to Understand 5.94 1.65 515 5.0 7.00
Discrediting Manipulative 4.23 2.18 515 2.0 6.00
Discrediting Might See 0.62 0.39 515 0.5 1.00
Discrediting Reliable 3.78 2.04 515 2.0 5.00
Discrediting Would Share 0.35 0.48 515 0.0 1.00
Emotional Concerning 5.72 1.76 618 5.0 7.00
Emotional Easy to Understand 6.27 1.27 618 6.0 7.00
Emotional Manipulative 4.24 2.21 618 2.0 6.75
Emotional Might See 0.71 0.25 618 0.5 1.00
Emotional Reliable 3.70 2.00 618 2.0 5.00
Emotional Would Share 0.36 0.48 618 0.0 1.00
False Dichotomy Concerning 4.81 2.16 515 3.0 7.00
False Dichotomy Easy to Understand 5.98 1.66 515 5.0 7.00
False Dichotomy Manipulative 4.72 2.18 515 3.0 7.00
False Dichotomy Might See 0.68 0.24 515 0.5 1.00
False Dichotomy Reliable 3.33 1.94 515 1.0 5.00
False Dichotomy Would Share 0.26 0.44 515 0.0 1.00
Non Misinfo Concerning 4.79 2.06 1339 3.0 7.00
Non Misinfo Easy to Understand 6.16 1.31 1339 6.0 7.00
Non Misinfo Manipulative 3.29 2.04 1339 1.0 5.00
Non Misinfo Might See 0.60 0.38 1339 0.5 1.00
Non Misinfo Reliable 4.12 1.91 1339 3.0 6.00
Non Misinfo Would Share 0.44 0.50 1339 0.0 1.00

3.3 Distribution of Likert-Scale responses by outcome

bin_counts_post = opinions |> 
    group_by(post_number) |>
    select(all_of(c("manipulative", "reliable", "concerning", "easy_to_understand"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "var", values_to = "opinion") |> #now count responses by post_number and var and opinion
    group_by(post_number, var, opinion) |>
    summarise(n = n())  |> #now pivot to wide
    pivot_wider(names_from = opinion, values_from = n) |>
    `colnames<-`(c("post_number", "variable", "1 -- not at all", "2", "3", "4", "5", "6", "7 -- very")) |> 
    ungroup() |>
    left_join(nice_labels, by = "variable") |> # now replace NAs with 0
    mutate_at(vars(-c("post_number", "variable", "Variable")), ~replace_na(., 0)) |>
    left_join(post_types, by = "post_number") |> # now replace NAs with 0
    select(`Post Number` = post_number, `Post Type` = post_type, all_of(c("Variable", "1 -- not at all", "2", "3", "4", "5", "6", "7 -- very")))

3.3.1 Manipulative

bin_counts_post |> filter(Variable == "Manipulative") |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>% scroll_box(height = "100%")
Post Number Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
1 Non Misinfo Manipulative 18 8 13 19 12 13 20
2 Non Misinfo Manipulative 38 16 10 14 15 3 7
3 Non Misinfo Manipulative 30 11 18 22 9 5 8
4 Non Misinfo Manipulative 27 8 9 22 12 9 16
5 Non Misinfo Manipulative 26 11 12 21 12 7 14
6 Non Misinfo Manipulative 32 15 19 11 8 7 11
7 Non Misinfo Manipulative 42 16 15 12 8 3 7
8 Non Misinfo Manipulative 49 13 9 11 10 6 5
9 Non Misinfo Manipulative 23 17 8 17 16 12 10
10 Non Misinfo Manipulative 40 8 10 16 11 10 8
11 Non Misinfo Manipulative 31 11 16 16 17 2 10
12 Non Misinfo Manipulative 35 16 6 24 5 5 12
13 Non Misinfo Manipulative 19 9 10 26 20 7 12
14 Discrediting Manipulative 22 8 5 21 23 6 18
15 Discrediting Manipulative 22 5 14 16 21 9 16
16 Discrediting Manipulative 8 4 4 15 10 18 44
17 Discrediting Manipulative 13 12 7 20 12 16 23
18 Discrediting Manipulative 37 11 8 11 10 11 15
19 False Dichotomy Manipulative 36 12 10 17 11 9 8
20 False Dichotomy Manipulative 6 4 5 8 15 18 47
21 False Dichotomy Manipulative 6 4 5 6 16 10 56
22 False Dichotomy Manipulative 6 5 11 8 12 17 44
23 False Dichotomy Manipulative 21 10 12 20 12 13 15
24 Emotional Manipulative 16 7 7 22 12 15 24
25 Emotional Manipulative 40 10 17 12 10 2 12
26 Emotional Manipulative 16 11 6 19 12 11 28
27 Emotional Manipulative 3 0 7 8 13 16 56
28 Emotional Manipulative 25 11 10 17 19 9 12
29 Emotional Manipulative 20 7 19 14 12 8 23

3.3.2 Reliable

bin_counts_post |> filter(Variable == "Reliable") |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>% scroll_box(height = "100%")
Post Number Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
1 Non Misinfo Reliable 25 16 19 24 7 3 9
2 Non Misinfo Reliable 5 7 12 21 19 17 22
3 Non Misinfo Reliable 17 13 13 25 17 10 8
4 Non Misinfo Reliable 24 8 18 24 8 6 15
5 Non Misinfo Reliable 16 8 19 31 7 12 10
6 Non Misinfo Reliable 8 4 8 25 19 17 22
7 Non Misinfo Reliable 15 10 16 27 21 7 7
8 Non Misinfo Reliable 5 0 9 18 29 11 31
9 Non Misinfo Reliable 6 2 12 19 14 17 33
10 Non Misinfo Reliable 7 6 9 26 20 16 19
11 Non Misinfo Reliable 10 7 11 22 21 13 19
12 Non Misinfo Reliable 27 19 13 20 10 10 4
13 Non Misinfo Reliable 16 14 17 27 18 7 4
14 Discrediting Reliable 10 7 15 26 14 12 19
15 Discrediting Reliable 27 12 22 21 11 4 6
16 Discrediting Reliable 53 11 14 13 7 4 1
17 Discrediting Reliable 19 10 13 23 18 11 9
18 Discrediting Reliable 1 6 7 16 20 14 39
19 False Dichotomy Reliable 26 11 14 29 9 3 11
20 False Dichotomy Reliable 35 18 17 19 6 5 3
21 False Dichotomy Reliable 17 11 16 21 15 12 11
22 False Dichotomy Reliable 46 13 14 13 6 6 5
23 False Dichotomy Reliable 14 9 10 25 17 16 12
24 Emotional Reliable 26 13 12 22 14 8 8
25 Emotional Reliable 4 9 14 23 23 16 14
26 Emotional Reliable 23 8 9 24 16 8 15
27 Emotional Reliable 51 16 13 6 3 6 8
28 Emotional Reliable 17 19 12 26 10 15 4
29 Emotional Reliable 12 7 8 24 19 12 21

3.3.3 Concerning

bin_counts_post |> filter(Variable == "Concerning") |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>% scroll_box(height = "100%")
Post Number Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
1 Non Misinfo Concerning 7 1 5 9 22 15 44
2 Non Misinfo Concerning 5 4 8 16 13 17 40
3 Non Misinfo Concerning 8 2 13 13 26 16 25
4 Non Misinfo Concerning 15 4 4 18 14 13 35
5 Non Misinfo Concerning 8 4 8 17 17 9 40
6 Non Misinfo Concerning 11 6 4 21 19 14 28
7 Non Misinfo Concerning 28 21 13 11 14 7 9
8 Non Misinfo Concerning 9 9 6 17 16 15 31
9 Non Misinfo Concerning 7 5 4 13 19 16 39
10 Non Misinfo Concerning 10 5 10 11 21 15 31
11 Non Misinfo Concerning 4 3 6 11 17 21 41
12 Non Misinfo Concerning 44 15 13 11 9 5 6
13 Non Misinfo Concerning 5 5 6 15 14 21 37
14 Discrediting Concerning 13 3 4 11 12 23 37
15 Discrediting Concerning 12 4 6 29 13 12 27
16 Discrediting Concerning 12 5 10 11 16 14 35
17 Discrediting Concerning 6 4 3 10 22 14 44
18 Discrediting Concerning 10 2 10 12 11 15 43
19 False Dichotomy Concerning 32 15 11 18 9 9 9
20 False Dichotomy Concerning 19 5 9 13 17 14 26
21 False Dichotomy Concerning 7 6 7 10 17 16 40
22 False Dichotomy Concerning 12 2 1 12 17 14 45
23 False Dichotomy Concerning 6 0 7 5 12 22 51
24 Emotional Concerning 4 0 1 6 15 18 59
25 Emotional Concerning 2 1 1 4 3 18 74
26 Emotional Concerning 4 4 2 7 11 17 58
27 Emotional Concerning 14 3 9 7 12 13 45
28 Emotional Concerning 9 2 7 14 19 16 36
29 Emotional Concerning 6 4 1 10 15 25 42

3.3.4 Easy to understand

bin_counts_post |> filter(Variable == "Easy to Understand") |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>% scroll_box(height = "100%")
Post Number Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
1 Non Misinfo Easy to Understand 3 1 7 9 13 16 54
2 Non Misinfo Easy to Understand 1 1 4 4 10 17 66
3 Non Misinfo Easy to Understand 1 2 2 4 19 17 58
4 Non Misinfo Easy to Understand 0 1 3 6 9 17 67
5 Non Misinfo Easy to Understand 1 1 2 5 10 18 66
6 Non Misinfo Easy to Understand 2 2 1 5 10 15 68
7 Non Misinfo Easy to Understand 3 2 9 8 18 21 42
8 Non Misinfo Easy to Understand 0 1 4 4 5 18 71
9 Non Misinfo Easy to Understand 1 1 1 3 8 21 68
10 Non Misinfo Easy to Understand 0 1 2 3 11 17 69
11 Non Misinfo Easy to Understand 2 0 1 4 11 22 63
12 Non Misinfo Easy to Understand 2 3 1 6 12 18 61
13 Non Misinfo Easy to Understand 2 0 7 11 17 17 49
14 Discrediting Easy to Understand 2 0 2 3 8 13 75
15 Discrediting Easy to Understand 14 6 13 12 14 14 30
16 Discrediting Easy to Understand 4 4 3 11 8 17 56
17 Discrediting Easy to Understand 1 3 4 2 11 22 60
18 Discrediting Easy to Understand 0 0 0 1 10 12 80
19 False Dichotomy Easy to Understand 9 0 5 10 11 14 54
20 False Dichotomy Easy to Understand 2 0 3 4 9 14 71
21 False Dichotomy Easy to Understand 6 3 1 5 6 15 67
22 False Dichotomy Easy to Understand 6 7 3 8 15 12 52
23 False Dichotomy Easy to Understand 2 2 2 1 13 14 69
24 Emotional Easy to Understand 3 2 2 5 8 22 61
25 Emotional Easy to Understand 3 0 3 3 10 19 65
26 Emotional Easy to Understand 2 0 0 6 7 15 73
27 Emotional Easy to Understand 3 1 2 4 12 17 64
28 Emotional Easy to Understand 2 0 1 4 21 17 58
29 Emotional Easy to Understand 1 0 2 1 10 16 73

3.4 Distribution of Likert-Scale responses by post-type

bin_counts = opinions |> 
    group_by(post_type) |>
    select(all_of(c("manipulative", "reliable", "concerning", "easy_to_understand"))) |> #now pivot to long
    pivot_longer(cols = -post_type, names_to = "var", values_to = "opinion") |> #now count responses by post_type and var and opinion
    group_by(post_type, var, opinion) |>
    summarise(n = n())  |> #now pivot to wide
    pivot_wider(names_from = opinion, values_from = n) |>
    `colnames<-`(c("Post Type", "variable", "1 -- not at all", "2", "3", "4", "5", "6", "7 -- very")) |> 
    ungroup() |> 
    left_join(nice_labels, by = "variable") |>
    select(all_of(c("Post Type", "Variable", "1 -- not at all", "2", "3", "4", "5", "6", "7 -- very")))

bin_counts |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
Post Type Variable 1 – not at all 2 3 4 5 6 7 – very
Discrediting Concerning 53 18 33 73 74 78 186
Discrediting Easy to Understand 21 13 22 29 51 78 301
Discrediting Manipulative 102 40 38 83 76 60 116
Discrediting Reliable 110 46 71 99 70 45 74
Emotional Concerning 39 14 21 48 75 107 314
Emotional Easy to Understand 14 3 10 23 68 106 394
Emotional Manipulative 120 46 66 92 78 61 155
Emotional Reliable 133 72 68 125 85 65 70
False Dichotomy Concerning 76 28 35 58 72 75 171
False Dichotomy Easy to Understand 25 12 14 28 54 69 313
False Dichotomy Manipulative 75 35 43 59 66 67 170
False Dichotomy Reliable 138 62 71 107 53 42 42
Non Misinfo Concerning 161 84 100 183 221 184 406
Non Misinfo Easy to Understand 18 16 44 72 153 234 802
Non Misinfo Manipulative 410 159 155 231 155 89 140
Non Misinfo Reliable 181 114 176 309 210 146 203

3.5 Corelations between variables

# Calculate bivariate correlations and present as a heatmap using ggplot
# Rename variables to more readable names
opinions |> 
    select(
        manipulative, reliable, concerning, easy_to_understand, might_see, share
    ) |> 
    na.omit() |>
    cor(use = "pairwise.complete.obs") |>
    as.data.frame() |>
    rename(
        Manipulative = manipulative,
        Reliable = reliable,
        Concerning = concerning,
        `Easy tounderstand` = easy_to_understand,
        `Might see` = might_see
    ) |>
    rownames_to_column("var1") |>
    pivot_longer(cols = -var1, names_to = "var2", values_to = "cor") |>
    ggplot(aes(x = var1, y = var2, fill = cor)) +
    geom_tile() +
    geom_text(aes(label = round(cor, 2)), size = 3) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust = 1)) +
    labs(x = "", y = "", title = "Correlations between variables")

3.5.1 Some observations

  1. Manipulative posts are harder to understand and more concerning but (understandably) seen as less reliable and, less likely to be seen on one’s timeline and less likely to be shared
  2. Sharing intention most correlated with reliability and likelihood of seeing the post on one’s timeline.

3.5.2 Covariate heatmap by post type

mean_diffs = opinions |>
    select(post_type, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_type, names_to = "variable", values_to = "opinion") |>
    # calculate grand means by variable and by variable-post_type
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_type) |>
    mutate(
        group_mean = mean(opinion, na.rm = TRUE),
        group_sd = sd(opinion, na.rm = TRUE)
        ) |>
    # group diferences from grand mean
    mutate(group_diff = group_mean - grand_mean) |>
    # now pivot to wide
    distinct(
        variable, post_type, group_diff, group_sd
    )
# Plot heatmap with SDs in parentheses below

library(ggtext)

mean_diffs |>
  ggplot(aes(x = post_type, y = variable)) +
  geom_tile(aes(fill = group_diff), color = "white") +
  geom_text(aes(label = paste0(round(group_diff, 2), "\n(", round(group_sd, 2), ")")),
            vjust =0,
            colour = "black",
            size = 2,
            lineheight = 0.9) +
  labs(title = "Opinions Heatmap",
       x = "Post Type",
       y = "Variable") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title = element_text(hjust = 0.5)) +
  scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) 

ggsave("opinions_heatmap.png", width = 10, height = 10, units = "in")

3.5.3 Coviariate heatmap by post (within post type)

3.5.3.1 Non-misinfo

opinions |>
    filter(post_type == "Non Misinfo") |>
    select(post_number, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "variable", values_to = "opinion") |>
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_number) |>
    mutate(
        group_mean = mean(opinion, na.rm = TRUE),
        group_diff = group_mean - grand_mean,
        group_sd = sd(opinion, na.rm = TRUE)
        ) |>
    distinct(variable, post_number, group_diff, group_sd) |> 
    ggplot(aes(y = variable, x = post_number, fill = group_diff)) +
    geom_tile() +
    geom_text(aes(label = paste0(round(group_diff, 2), " \n (", round(group_sd, 2), ")")),
            vjust =0,
            colour = "black",
            size = 2,
            lineheight = 0.9) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(y = "Feeling", x = "Question Number", title = "Differences from Post-Type group mean by post number, Non-Misinfo")

ggsave("opinions_heatmap.png", width = 15, height = 10, units = "in")

3.5.3.2 Discrediting

opinions |>
    filter(post_type == "Non Misinfo") |>
    select(post_number, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "variable", values_to = "opinion") |>
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_number) |>
    mutate(
        group_mean = mean(opinion, na.rm = TRUE),
        group_diff = group_mean - grand_mean,
        group_sd = sd(opinion, na.rm = TRUE)
        ) |>
    distinct(variable, post_number, group_diff, group_sd) |> 
    ggplot(aes(y = variable, x = post_number, fill = group_diff)) +
    geom_tile() +
    geom_text(aes(label = paste0(round(group_diff, 2), " \n (", round(group_sd, 2), ")")),
            vjust = 0,
            colour = "black",
            size = 2,
            lineheight = 0.9) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(y = "Feeling", x = "Question Number", title = "Differences from Post-Type group mean by post number, Discrediting")

3.5.3.3 False Dichotomy

opinions |>
    filter(post_type == "Non Misinfo") |>
    select(post_number, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "variable", values_to = "opinion") |>
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_number) |>
    mutate(
        group_mean = mean(opinion, na.rm = TRUE),
        group_diff = group_mean - grand_mean,
        group_sd = sd(opinion, na.rm = TRUE)
        ) |>
    distinct(variable, post_number, group_diff, group_sd) |> 
    ggplot(aes(y = variable, x = post_number, fill = group_diff)) +
    geom_tile() +
    geom_text(aes(label = paste0(round(group_diff, 2), "\n (", round(group_sd, 2), ")")),
            vjust = 0,
            colour = "black",
            size = 2,
            lineheight = 0.9) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(y = "Feeling", x = "Question Number", title = "Differences from Post-Type group mean by post number, False Dichotomy")

3.5.3.4 Emotional

opinions |>
    filter(post_type == "Non Misinfo") |>
    select(post_number, all_of(c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see", "share"))) |> #now pivot to long
    pivot_longer(cols = -post_number, names_to = "variable", values_to = "opinion") |>
    group_by(variable) |>
    mutate(grand_mean = mean(opinion, na.rm = TRUE)) |>
    group_by(variable, post_number) |>
    mutate(
        group_mean = mean(opinion, na.rm = TRUE),
        group_diff = group_mean - grand_mean,
        group_sd = sd(opinion, na.rm = TRUE)
        ) |>
    distinct(variable, post_number, group_diff, group_sd) |> 
    ggplot(aes(y = variable, x = post_number, fill = group_diff)) +
    geom_tile() +
    geom_text(aes(label = paste0(round(group_diff, 2), "\n (", round(group_sd, 2), ")")),
            vjust = 0,
            colour = "black",
            size = 2,
            lineheight = 0.9) +
    scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0) +
    theme_minimal() +
    theme(axis.text.x = element_text(vjust = 0.5, hjust = 1)) +
    labs(y = "Feeling", x = "Question Number", title = "Differences from Post-Type group mean by post number, Emotional")

3.6 Regressions

3.6.1 Regressions of intention to share on other variables

# Run regressions of each variables individually on share and all jointly
library(fixest)

mods = list()
for(var in c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see")){
    mods[[var]] = feols(as.formula(paste0("share ~ ", var)), data = opinions)
}

mods[["Manipulative_&_Reliable"]] = feols(share ~ manipulative + reliable, data = opinions)
mods[["all"]] = feols(share ~ manipulative + reliable + concerning + easy_to_understand + might_see, data = opinions)

#rename mods in list to (1), (2), etc
names(mods) = paste0("(", 1:length(mods), ")")

etable(mods, cluster = "ResponseId", digits = 3,
    headers = rep("Share?", length(mods)),
    depvar = FALSE,
    drop = "Constant",
    dict = c(
        manipulative = "Manipulative?",
        reliable = "Reliable?",
        concerning = "Concerning?",
        easy_to_understand = "Easy to understand?",
        might_see = "Might see?"
    )
) |> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
Share? Share? Share? Share? Share? Share? Share?
Manipulative? -0.069*** (0.006) -0.031*** (0.005) -0.033*** (0.005)
Reliable? 0.132*** (0.006) 0.119*** (0.007) 0.098*** (0.007)
Concerning? 0.044*** (0.007) 0.021*** (0.004)
Easy to understand? 0.066*** (0.008) 0.005 (0.007)
Might see? 0.520*** (0.040) 0.318*** (0.036)
___________________ _________________ ________________ ________________ ________________ ________________ _________________ _________________
S.E.: Clustered by: ResponseId by: ResponseId by: ResponseId by: ResponseId by: ResponseId by: ResponseId by: ResponseId
Observations 2,987 2,987 2,987 2,987 2,987 2,987 2,987
R2 0.09817 0.28704 0.03354 0.03794 0.13391 0.30400 0.36260
Adj. R2 0.09787 0.28680 0.03322 0.03762 0.13362 0.30354 0.36154

3.6.2 Leave-one-out regressions

Now regress the intention to share on leave-one-out (LOO) means of each variable by post number. This is to see to what extent, say, more reliable posts are shared more widely. This specification lessens the experimenter’s demand concerns.

# Compute the leave-one-out means by question of each variable
opinions = opinions |>
    group_by(post_number) |>
    mutate(
        manipulative_loo = (sum(manipulative) - manipulative) / (n() - 1),
        reliable_loo = (sum(reliable) - reliable) / (n() - 1),
        concerning_loo = (sum(concerning) - concerning) / (n() - 1),
        easy_to_understand_loo = (sum(easy_to_understand) - easy_to_understand) / (n() - 1),
        might_see_loo = (sum(might_see) - might_see) / (n() - 1)
    )

# Run regressions of each variables individually on share and all jointly

mods = list()
for(var in c("manipulative_loo", "reliable_loo", "concerning_loo", "easy_to_understand_loo", "might_see_loo")){
    mods[[var]] = feols(as.formula(paste0("share ~ ", var)), data = opinions)
}

mods[["Manipulative_&_Reliable"]] = feols(share ~ manipulative_loo + reliable_loo, data = opinions)
mods[["all"]] = feols(share ~ manipulative_loo + reliable_loo + concerning_loo + easy_to_understand_loo + might_see_loo, data = opinions)

#rename mods in list to (1), (2), etc
names(mods) = paste0("(", 1:length(mods), ")")

etable(mods, cluster = "post_number", digits = 3,
    headers = rep("Share?", length(mods)),
    depvar = FALSE,
    drop = "Constant",
    dict = c(
        manipulative_loo = "Manipulative LOO?",
        reliable_loo = "Reliable LOO?",
        concerning_loo = "Concerning LOO?",
        easy_to_understand_loo = "Easy to understand LOO?",
        might_see_loo = "Might see LOO?"
    )
)|> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
Share? Share? Share? Share? Share? Share? Share?
Manipulative LOO? -0.108*** (0.023) -0.008 (0.011) -0.025 (0.017)
Reliable LOO? 0.178*** (0.010) 0.172*** (0.012) 0.146*** (0.023)
Concerning LOO? 0.050*** (0.013) 0.015 (0.015)
Easy to understand LOO? 0.221** (0.073) 0.031 (0.019)
Might see LOO? 0.747*** (0.157) 0.026 (0.104)
_______________________ _________________ ________________ ________________ _______________ ________________ ________________ ________________
S.E.: Clustered by: post_number by: post_number by: post_number by: post_number by: post_number by: post_number by: post_number
Observations 2,987 2,987 2,987 2,987 2,987 2,987 2,987
R2 0.04142 0.08846 0.00694 0.03179 0.03007 0.08858 0.08947
Adj. R2 0.04109 0.08815 0.00661 0.03146 0.02974 0.08797 0.08794

3.6.3 Regressions with both LOO and original variables

mods = list()
for(var in c("manipulative", "reliable", "concerning", "easy_to_understand", "might_see")){
    mods[[var]] = feols(as.formula(paste0("share ~ ", var, " + ", var, "_loo")), data = opinions)
}

mods[["Manipulative_&_Reliable"]] = feols(share ~ manipulative + reliable + manipulative_loo + reliable_loo, data = opinions)

mods[[paste0("all")]] = feols(share ~ manipulative + reliable + concerning + easy_to_understand + might_see + manipulative_loo + reliable_loo + concerning_loo + easy_to_understand_loo + might_see_loo, data = opinions)

#rename mods in list to (1), (2), etc
names(mods) = paste0("(", 1:length(mods), ")")

etable(mods, cluster = "post_number", digits = 3,
    headers = rep("Share?", length(mods)),
    depvar = FALSE,
    drop = "Constant",
    dict = c(
        manipulative = "Manipulative?",
        reliable = "Reliable?",
        concerning = "Concerning?",
        easy_to_understand = "Easy to understand?",
        might_see = "Might see?",
        manipulative_loo = "Manipulative LOO?",
        reliable_loo = "Reliable LOO?",
        concerning_loo = "Concerning LOO?",
        easy_to_understand_loo = "Easy to understand LOO?",
        might_see_loo = "Might see LOO?"
    )
)|> kable() |> kable_styling(bootstrap_options = c("striped", "hover")) %>%
  scroll_box(height = "100%")
Share? Share? Share? Share? Share? Share? Share?
Manipulative? -0.061*** (0.005) -0.030*** (0.004) -0.030*** (0.003)
Manipulative LOO? -0.050* (0.023) 0.024* (0.011) 0.005 (0.018)
Reliable? 0.122*** (0.006) 0.112*** (0.005) 0.094*** (0.005)
Reliable LOO? 0.062*** (0.012) 0.067*** (0.013) 0.056* (0.024)
Concerning? 0.042*** (0.007) 0.021*** (0.003)
Concerning LOO? 0.010 (0.014) -0.005 (0.016)
Easy to understand? 0.054*** (0.007) 0.005 (0.004)
Easy to understand LOO? 0.173* (0.076) 0.022 (0.019)
Might see? 0.490*** (0.031) 0.327*** (0.028)
Might see LOO? 0.297. (0.165) -0.271* (0.115)
_______________________ _________________ ________________ ________________ ________________ ________________ _________________ _________________
S.E.: Clustered by: post_number by: post_number by: post_number by: post_number by: post_number by: post_number by: post_number
Observations 2,987 2,987 2,987 2,987 2,987 2,987 2,987
R2 0.10563 0.29622 0.03379 0.05636 0.13822 0.31102 0.36749
Adj. R2 0.10503 0.29575 0.03314 0.05572 0.13764 0.31010 0.36537

3.6.4 Takeaways

  1. Stated reliability and post being similar to something one might see on their timelines have strong (and positive) associations with the intentions to share
  2. The remaining variables (easy to understand, manipulative, concerning) have much weaker correlations with the intention to share.
  3. Manipulativeness is the only variable with a negative association with the intention to share.

3.7 Textual feedback

Takeaways: 1. A lot of people say they agree with true posts but some say that they know nothing about given topics and it would not draw their attention 2. On misinfo posts a lot of people comment as: “fake news”, “lie” “crazy” 3. Some people agree with some misinfo posts: “Media lies!”, “Interesting and good question”, “The post raises eyebrows about the reliability of nutritionists.”, “it is sad but it truth” 4. On style: “I like this post because its to the point.” “arrogant pster”, “sad; but vague and unhelpful too”, “Humorous post.”

Specific Posts:

  • Post 18 (deemed very reliable) – “facts -poliiticians do not care”, “politicians are the biggest criminals”, “Unfortunately, a large number of the politicians never deliver their promised projects. It happens all over the world!” – Notice these comments are not about reliability of the post but just people agreeing with the sentiment of the post
  • Post 27 (deemed very maniulative and unreliable) – “This could be propaganda.”, “I learnt something new.”, “Antibiotics work wonders”, “antibiotics are safe” – People mention that the post is manipulative but also just express the opinion that they disagree with the substance (i.e. they don’t think antibiotics are toxic)
  • Post 23 (very reliable and concerning) – “There is truth in this post” “This is just someone venting about lack of help for low income countries but not harmful.”, “the figure makes me trust the post”, “With statistical posts, proof should be provided. That said, if I were to believe the post’s claim, I would get angry on behalf of low-income nations.”, “nothing new, the rich will always come first” – Some people took the number for granted but a lot question the figure and would like to see a source. Still, it seems like a well-written post with a number in it seems more reliable.

3.7.1 Word Clouds

3.7.1.1 Non-misinfo

# Tokenize feedback
tokens <- valid_feedback %>%
    filter(post_type == "Non Misinfo") %>%
    select(feedback) %>%
    unnest_tokens(word, feedback) |>
    anti_join(stop_words)

# Count word occurrences and select top 100
word_counts <- tokens %>%
    count(word, sort = TRUE)|>
    filter(word!= "post") |>
    slice(1:50) 
# Create word cloud
wordcloud::wordcloud(words = word_counts$word, freq = word_counts$n, max.words = 50,
    random.order = FALSE, rot.per = 0.35, colors = brewer.pal(8, "Dark2"))

3.7.1.2 Discrediting

# Tokenize feedback
tokens <- valid_feedback %>%
    filter(post_type == "Discrediting") %>%
    select(feedback) %>%
    unnest_tokens(word, feedback) |>
    anti_join(stop_words)

# Count word occurrences and select top 100
word_counts <- tokens %>%
    count(word, sort = TRUE)|>
    filter(word!= "post") |>
    slice(1:50)


# Create word cloud
wordcloud::wordcloud(words = word_counts$word, freq = word_counts$n, max.words = 50,
    random.order = FALSE, rot.per = 0.35, colors = brewer.pal(8, "Dark2"))

3.7.1.3 Emotional

# Tokenize feedback
tokens <- valid_feedback %>%
    filter(post_type == "Emotional") %>%
    select(feedback) %>%
    unnest_tokens(word, feedback) |>
    anti_join(stop_words)

# Count word occurrences and select top 100
word_counts <- tokens %>%
    count(word, sort = TRUE) |>
    filter(word!= "post")|>
    slice(1:50)

# Create word cloud
wordcloud::wordcloud(words = word_counts$word, freq = word_counts$n, max.words = 50,
    random.order = FALSE, rot.per = 0.35, colors = brewer.pal(8, "Dark2"))

3.7.1.4 False Dichotomy

# Tokenize feedback
tokens <- valid_feedback %>%
    filter(post_type == "False Dichotomy") %>%
    select(feedback) %>%
    unnest_tokens(word, feedback) |>
    anti_join(stop_words)

# Count word occurrences and select top 100
word_counts <- tokens %>%
    count(word, sort = TRUE) |>
    filter(word!= "post") |>
    slice(1:50)

# Create word cloud
wordcloud::wordcloud(words = word_counts$word, freq = word_counts$n, max.words = 50,
    random.order = FALSE, rot.per = 0.35, colors = brewer.pal(8, "Dark2"))