Pre Process

Load Data

data <- read.csv("Data/Copy_of_output_with_diffs.csv")
#head(data)

Load Libraries

library(dplyr)
library(datawizard)
library(ggplot2)
library(lmerTest) # for MLMs
library(performance) # for icc
library(parameters)
library(emmeans) # for contrasts and more
library(tidyverse)

Clean Data

We will clean our data to contain only relevant variables.

data_clean <- data |>
  select(Prolific.ID, contains("opinionsCollection"), contains("Type"), contains("Diff") ) |>
  mutate(Prolific.ID = c(1:175)) |> # Set ID for Subject
  select(-Signature_Type, -TimeDiff) # Omit non-relevant variables

Convert Data to Long Format

We transformed the wide-format data_clean dataset into a tidy long-format data frame by reshaping the opinion collection variables and ‘Diff’ variables using pivot_longer and pivot_wider. We extracted the Category and Variable information from the column names, allowing us to organize the data by subject and category. After pivoting, we merged the opinion variables, ‘Diff’ variables, and Manipulation_Type into a single data frame called final_data, selecting and arranging the relevant columns. This process resulted in a streamlined dataset where each row represents a subject’s responses for a specific category, facilitating easier mixed models analysis.

# Pivot the opinion collection variables
opinions_long <- data_clean %>%
  select(Prolific.ID, matches('opinionsCollection_.*_Manipulation_.*_Value$')) %>%
  pivot_longer(
    cols = -Prolific.ID,
    names_to = c('Category', 'Variable'),
    names_pattern = 'opinionsCollection_(.+)_Manipulation_(.+)_Value',
    values_to = 'Value'
  ) %>%
  pivot_wider(
    names_from = Variable,
    values_from = Value
  )

# Pivot the Diff variables
diff_long <- data_clean %>%
  select(Prolific.ID, matches('Diff_.*_Manipulation_.*_Value$')) %>%
  pivot_longer(
    cols = -Prolific.ID,
    names_to = c('Category', 'Variable'),
    names_pattern = 'Diff_(.+)_Manipulation_(.+)_Value',
    values_to = 'Diff_Value'
  ) %>%
  pivot_wider(
    names_from = Variable,
    values_from = Diff_Value,
    names_prefix = 'Diff_'
  )

# Pivot the Manipulation_Type variables
manipulation_long <- data_clean %>%
  select(Prolific.ID, ends_with('_Manipulation_Type')) %>%
  pivot_longer(
    cols = -Prolific.ID,
    names_to = 'Category',
    names_pattern = '(.+)_Manipulation_Type',
    values_to = 'Manipulation_Type'
  )

# Merge all long-format data frames
final_data <- opinions_long %>%
  left_join(diff_long, by = c('Prolific.ID', 'Category')) %>%
  left_join(manipulation_long, by = c('Prolific.ID', 'Category')) %>%
  rename(SubjectID = Prolific.ID)

# Select and arrange the columns
final_data <- final_data %>%
  select(
    SubjectID,
    Category,
    Manipulation_Type,
    Agreement,
    Confidence,
    PersonalSignificance,
    IdentityConnection,
    Diff_Agreement,
    Diff_Confidence,
    Diff_PersonalSignificance,
    Diff_IdentityConnection
  )

# View the transformed data
head(final_data)
## # A tibble: 6 × 11
##   SubjectID Category Manipulation_Type Agreement Confidence PersonalSignificance
##       <int> <chr>    <chr>                 <int>      <int>                <int>
## 1         1 Abortion Against_Moderate         NA         NA                   NA
## 2         1 Climate… Against_Extreme         100        100                   77
## 3         1 GunCont… Against_Extreme         100        100                  100
## 4         1 Immigra… Support_Moderate          8        100                   48
## 5         1 BidenVS… Against_Moderate         69         45                   67
## 6         1 Pregnan… Support_Extreme          NA         NA                   NA
## # ℹ 5 more variables: IdentityConnection <int>, Diff_Agreement <int>,
## #   Diff_Confidence <int>, Diff_PersonalSignificance <int>,
## #   Diff_IdentityConnection <int>

Mixed Effects

Interclass Cross Correlation (ICC) - Analyze the variance of Diff_Agreement explained by each level differences (Conditional R2):

  • Manipulation Type Level
  • Subject Level
  • Category Level
Manipulation_type <- lmer(Diff_Agreement ~ (1|Manipulation_Type), data = final_data)
RandomSubject <- lmer(Diff_Agreement ~ (1|SubjectID), data = final_data)
RandomCategory <- lmer (Diff_Agreement ~ (1|Category), data = final_data)

sjPlot::tab_model(Manipulation_type, RandomSubject, RandomCategory,
                  df.method= "satterthwaite")
  Diff_Agreement Diff_Agreement Diff_Agreement
Predictors Estimates CI p Estimates CI p Estimates CI p
(Intercept) -6.25 -9.87 – -2.63 0.013 -6.17 -7.43 – -4.91 <0.001 -6.17 -7.38 – -4.96 <0.001
Random Effects
σ2 381.16 340.81 382.12
τ00 3.78 Manipulation_Type 42.83 SubjectID 1.43 Category
ICC 0.01 0.11 0.00
N 4 Manipulation_Type 175 SubjectID 12 Category
Observations 2100 2100 2100
Marginal R2 / Conditional R2 0.000 / 0.010 0.000 / 0.112 0.000 / 0.004

Plot opinion difference by Manipulation Type

ggplot(final_data, aes(x = factor(Manipulation_Type), y = Diff_Agreement)) +
  geom_boxplot() +
  labs(
    x = "Manipulation Type (Moderate = 0, Extreme = 1)",
    y = "Change in Agreement (Diff_Agreement)",
    title = "Change in Agreement by Manipulation Type"
  ) +
  theme_minimal()

Convert Manipulation_Type into a dummy variable: Moderate = 0, Exterme = 1

final_data_bin <- final_data %>%
  mutate(
    Manipulation_Type = ifelse(
      str_detect(Manipulation_Type, "Extreme"),
      1, 0))

Plot Differences: Moderate VS Extreme

ggplot(final_data_bin, aes(x = factor(Manipulation_Type), y = Diff_Agreement)) +
  geom_boxplot() +
  labs(
    x = "Manipulation Type (Moderate = 0, Extreme = 1)",
    y = "Change in Agreement (Diff_Agreement)",
    title = "Change in Agreement by Manipulation Type"
  ) +
  theme_minimal()

ggplot(final_data_bin, aes(PersonalSignificance, Diff_Agreement, colour = factor(Manipulation_Type))) + 
  geom_line() +
  stat_smooth(method = 'lm', se = FALSE,
              colour = "black") +
  facet_grid(cols = vars(Manipulation_Type)) + 
  guides(colour = "none")
## `geom_smooth()` using formula = 'y ~ x'
## Warning: Removed 344 rows containing non-finite outside the scale range
## (`stat_smooth()`).
## Warning: Removed 344 rows containing missing values or values outside the scale range
## (`geom_line()`).

Final Models

Model1

Predict Diff_Agreement with PersonalSignificance + Manipulation_Type + (1|SubjectID)

Model1 <- lmer(Diff_Agreement ~ PersonalSignificance + Manipulation_Type + (1|SubjectID), data = final_data_bin)
summary(Model1)
## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: Diff_Agreement ~ PersonalSignificance + Manipulation_Type + (1 |  
##     SubjectID)
##    Data: final_data_bin
## 
## REML criterion at convergence: 15363.2
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -5.0133 -0.3207  0.1991  0.4417  3.3746 
## 
## Random effects:
##  Groups    Name        Variance Std.Dev.
##  SubjectID (Intercept)  32.66    5.715  
##  Residual              345.16   18.578  
## Number of obs: 1756, groups:  SubjectID, 175
## 
## Fixed effects:
##                        Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)          -6.106e+00  1.084e+00  8.099e+02  -5.634 2.43e-08 ***
## PersonalSignificance  4.226e-03  1.517e-02  1.650e+03   0.279    0.781    
## Manipulation_Type    -7.845e-01  9.092e-01  1.709e+03  -0.863    0.388    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) PrsnlS
## PrsnlSgnfcn -0.704       
## Manpltn_Typ -0.385 -0.051
ranova(Model1)
## ANOVA-like table for random-effects: Single term deletions
## 
## Model:
## Diff_Agreement ~ PersonalSignificance + Manipulation_Type + (1 | SubjectID)
##                 npar  logLik   AIC   LRT Df Pr(>Chisq)    
## <none>             5 -7681.6 15373                        
## (1 | SubjectID)    4 -7703.3 15414 43.36  1  4.554e-11 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
sjPlot::tab_model(Model1, # Plot Model1`
                  df.method= "satterthwaite", show.std = TRUE)
  Diff_Agreement
Predictors Estimates std. Beta CI standardized CI p
(Intercept) -6.11 0.00 -8.23 – -3.98 -0.06 – 0.06 <0.001
PersonalSignificance 0.00 0.01 -0.03 – 0.03 -0.04 – 0.06 0.781
Manipulation Type -0.78 -0.02 -2.57 – 1.00 -0.07 – 0.03 0.388
Random Effects
σ2 345.16
τ00 SubjectID 32.66
ICC 0.09
N SubjectID 175
Observations 1756
Marginal R2 / Conditional R2 0.000 / 0.087 
ggplot(final_data, aes(PersonalSignificance, Diff_Agreement, colour = factor(Manipulation_Type))) + 
  geom_line() +
  stat_smooth(method = 'lm', se = FALSE,
              colour = "black") +
  facet_grid(cols = vars(Manipulation_Type)) + 
  guides(colour = "none")
## `geom_smooth()` using formula = 'y ~ x'
## Warning: Removed 344 rows containing non-finite outside the scale range
## (`stat_smooth()`).
## Warning: Removed 344 rows containing missing values or values outside the scale range
## (`geom_line()`).

Model2

Predict Diff_Agreement with Confidence + Manipulation_Type + (1|SubjectID)

Model2 <- lmer(Diff_Agreement ~ Confidence + Manipulation_Type + (1|SubjectID), data = final_data_bin)

summary(Model2)
## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: Diff_Agreement ~ Confidence + Manipulation_Type + (1 | SubjectID)
##    Data: final_data_bin
## 
## REML criterion at convergence: 15359.2
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -4.9804 -0.3284  0.2076  0.4614  3.2425 
## 
## Random effects:
##  Groups    Name        Variance Std.Dev.
##  SubjectID (Intercept)  32.78    5.725  
##  Residual              344.37   18.557  
## Number of obs: 1756, groups:  SubjectID, 175
## 
## Fixed effects:
##                     Estimate Std. Error         df t value Pr(>|t|)  
## (Intercept)         -3.10528    1.66266 1304.06078  -1.868   0.0620 .
## Confidence          -0.03646    0.01927 1658.29149  -1.892   0.0587 .
## Manipulation_Type   -0.77709    0.90701 1709.63391  -0.857   0.3917  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) Cnfdnc
## Confidence  -0.887       
## Manpltn_Typ -0.277  0.003
ranova(Model2)
## ANOVA-like table for random-effects: Single term deletions
## 
## Model:
## Diff_Agreement ~ Confidence + Manipulation_Type + (1 | SubjectID)
##                 npar  logLik   AIC    LRT Df Pr(>Chisq)    
## <none>             5 -7679.6 15369                         
## (1 | SubjectID)    4 -7701.5 15411 43.816  1  3.608e-11 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
sjPlot::tab_model(Model2, # Plot Model2
                  df.method= "satterthwaite", show.std = TRUE)
  Diff_Agreement
Predictors Estimates std. Beta CI standardized CI p
(Intercept) -3.11 0.00 -6.37 – 0.16 -0.06 – 0.06 0.062
Confidence -0.04 -0.05 -0.07 – 0.00 -0.09 – 0.00 0.059
Manipulation Type -0.78 -0.02 -2.56 – 1.00 -0.07 – 0.03 0.392
Random Effects
σ2 344.37
τ00 SubjectID 32.78
ICC 0.09
N SubjectID 175
Observations 1756
Marginal R2 / Conditional R2 0.003 / 0.089

Model3

Predict Diff_Confidence with PersonalSignificance + Manipulation_Type + (1|SubjectID)

Model3 <- lmer(Diff_Confidence ~ PersonalSignificance + Manipulation_Type + (1|SubjectID), data = final_data_bin)
summary(Model3)
## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: Diff_Confidence ~ PersonalSignificance + Manipulation_Type +  
##     (1 | SubjectID)
##    Data: final_data_bin
## 
## REML criterion at convergence: 15720.1
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -4.5351 -0.3940  0.0549  0.4089  4.9118 
## 
## Random effects:
##  Groups    Name        Variance Std.Dev.
##  SubjectID (Intercept)  59.09    7.687  
##  Residual              413.92   20.345  
## Number of obs: 1756, groups:  SubjectID, 175
## 
## Fixed effects:
##                        Estimate Std. Error         df t value Pr(>|t|)    
## (Intercept)            -0.10492    1.24555  750.65320  -0.084 0.932893    
## PersonalSignificance   -0.05867    0.01691 1713.32691  -3.469 0.000535 ***
## Manipulation_Type       2.11303    1.00066 1690.05689   2.112 0.034864 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Correlation of Fixed Effects:
##             (Intr) PrsnlS
## PrsnlSgnfcn -0.684       
## Manpltn_Typ -0.368 -0.051
ranova(Model3)
## ANOVA-like table for random-effects: Single term deletions
## 
## Model:
## Diff_Confidence ~ PersonalSignificance + Manipulation_Type + (1 | SubjectID)
##                 npar  logLik   AIC    LRT Df Pr(>Chisq)    
## <none>             5 -7860.0 15730                         
## (1 | SubjectID)    4 -7900.3 15809 80.588  1  < 2.2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
sjPlot::tab_model(Model3, # Plot Model3
                  df.method= "satterthwaite", show.std = TRUE)
  Diff_Confidence
Predictors Estimates std. Beta CI standardized CI p
(Intercept) -0.10 0.00 -2.55 – 2.34 -0.07 – 0.07 0.933
PersonalSignificance -0.06 -0.09 -0.09 – -0.03 -0.13 – -0.04 0.001
Manipulation Type 2.11 0.05 0.15 – 4.08 0.00 – 0.09 0.035
Random Effects
σ2 413.92
τ00 SubjectID 59.09
ICC 0.12
N SubjectID 175
Observations 1756
Marginal R2 / Conditional R2 0.009 / 0.133

Plot All Models

sjPlot::tab_model(Model1, Model2, Model3,
                  df.method= "satterthwaite", show.std = TRUE)
  Diff_Agreement Diff_Agreement Diff_Confidence
Predictors Estimates std. Beta CI standardized CI p Estimates std. Beta CI standardized CI p Estimates std. Beta CI standardized CI p
(Intercept) -6.11 0.00 -8.23 – -3.98 -0.06 – 0.06 <0.001 -3.11 0.00 -6.37 – 0.16 -0.06 – 0.06 0.062 -0.10 0.00 -2.55 – 2.34 -0.07 – 0.07 0.933
PersonalSignificance 0.00 0.01 -0.03 – 0.03 -0.04 – 0.06 0.781 -0.06 -0.09 -0.09 – -0.03 -0.13 – -0.04 0.001
Manipulation Type -0.78 -0.02 -2.57 – 1.00 -0.07 – 0.03 0.388 -0.78 -0.02 -2.56 – 1.00 -0.07 – 0.03 0.392 2.11 0.05 0.15 – 4.08 0.00 – 0.09 0.035
Confidence -0.04 -0.05 -0.07 – 0.00 -0.09 – 0.00 0.059
Random Effects
σ2 345.16 344.37 413.92
τ00 32.66 SubjectID 32.78 SubjectID 59.09 SubjectID
ICC 0.09 0.09 0.12
N 175 SubjectID 175 SubjectID 175 SubjectID
Observations 1756 1756 1756
Marginal R2 / Conditional R2 0.000 / 0.087 0.003 / 0.089 0.009 / 0.133