Hypothesis 2:

A zero-sum mindset is relatively stable over time.

We utilized the Intraclass Correlation Coefficient (ICC) to determine the test-retest reliability of a psychometric scale. We chose the ICC for its ability to measure both the degree of correlation and the absolute agreement between the mean scores across two time points. The ‘twoway’ model was selected to control for random effects associated with time-point measurements and systematic differences between the scores. Our choice of the ‘agreement’ type over ‘consistency’ was to ensure the estimation of absolute score agreement, which is more stringent and indicative of a scale’s reliability in yielding consistent results over time.

We applied the ‘average’ measures unit to indicate our use of 7-item scale index. The specification of r0 as 7 in our model was set to test the null hypothesis that the true ICC is less than or equal to 0.7, (not to be confused with the 7-items in the scale) reflecting a substantial and acceptable level of stability over time. Lastly, we established a confidence level of 95% for our estimates, ensuring that the interval reflects a precise level of certainty in our reliability measure.

Libraries

library(readr)
library(here)
library(tidyverse)
library(psych)
library(BlandAltmanLeh)
library(irr)
library(pander)

Loading Longitudinal Datasets

s1 <- read_csv(here("data", "s1b.csv"))
s2 <- read_csv(here("data", "s2.csv"))
s3b <- read_csv(here("data", "s3b.csv"))
s4 <- read_csv(here("data", "s4.csv"))
s5 <- read_csv(here("data", "s5.csv"))

Analysis

# Create a list to store the ICC results
icc_results <- list()

# Loop through the datasets
datasets <- c("s1", "s2", "s3b", "s4", "s5")
for (dataset in datasets) {
  # Combine the ratings into a data frame
  ratings <- data.frame(T1 = eval(parse(text = paste0(dataset, "$zsm_mean_T1"))),
                        T2 = eval(parse(text = paste0(dataset, "$zsm_mean_T2"))))
  
 
   # Compute the ICC using icc() function from irr package
  icc_result <- icc(ratings, model = "twoway", type = "agreement", unit = "average", r0 = 7, conf.level = 0.95)
  
  # Store the ICC result in the list
  icc_results[[dataset]] <- icc_result
}

# Create a data frame to store the summary results
summary_df <- data.frame(
  Sample = datasets,
  ICC = sapply(icc_results, function(x) x$value),
  Lower_CI = sapply(icc_results, function(x) x$lbound),
  Upper_CI = sapply(icc_results, function(x) x$ubound)
)

# Print the summary table
print(summary_df)
##     Sample       ICC  Lower_CI  Upper_CI
## s1      s1 0.8193921 0.7772542 0.8535464
## s2      s2 0.7753126 0.7215934 0.8186610
## s3b    s3b 0.7806074 0.7205452 0.8277532
## s4      s4 0.8158118 0.7763534 0.8482515
## s5      s5 0.8342496 0.7854481 0.8719486
pander(summary_df)
  Sample ICC Lower_CI Upper_CI
s1 s1 0.8194 0.7773 0.8535
s2 s2 0.7753 0.7216 0.8187
s3b s3b 0.7806 0.7205 0.8278
s4 s4 0.8158 0.7764 0.8483
s5 s5 0.8342 0.7854 0.8719 
library(irr)

# Create a list to store the ICC results and descriptive statistics
icc_results <- list()
means <- list()
sds <- list()

# Loop through the datasets
datasets <- c("s1", "s2", "s3b", "s4", "s5")
for (dataset in datasets) {
  # Combine the ratings into a data frame
  ratings <- data.frame(T1 = eval(parse(text = paste0(dataset, "$zsm_mean_T1"))),
                        T2 = eval(parse(text = paste0(dataset, "$zsm_mean_T2"))))

  # Handle missing values by excluding them from the analysis
  valid_indices <- complete.cases(ratings$T1, ratings$T2)
  ratings <- ratings[valid_indices, ]

  # Compute the ICC using icc() function from irr package
  icc_result <- icc(ratings, model = "twoway", type = "agreement", unit = "average", r0 = 7, conf.level = 0.95)
  
  # Store the ICC result in the list
  icc_results[[dataset]] <- icc_result
  
  # Calculate mean and SD across both time points for each dataset, excluding missing values
  means[[dataset]] <- mean(rowMeans(ratings, na.rm = TRUE))
  sds[[dataset]] <- sqrt((var(ratings$T1, na.rm = TRUE) + var(ratings$T2, na.rm = TRUE))/2) # Pooled SD for two time points
}

# Create a data frame to store the summary results
summary_df <- data.frame(
  Sample = datasets,
  ICC = sapply(icc_results, function(x) x$value),
  Mean = unlist(means),
  SD = unlist(sds),
  Lower_CI = sapply(icc_results, function(x) x$lbound),
  Upper_CI = sapply(icc_results, function(x) x$ubound)
)

# Print the summary table
print(summary_df)
##     Sample       ICC     Mean        SD  Lower_CI  Upper_CI
## s1      s1 0.8193921 2.632295 1.0961483 0.7772542 0.8535464
## s2      s2 0.7753126 2.624036 0.9609415 0.7215934 0.8186610
## s3b    s3b 0.7806074 2.617736 1.0999452 0.7205452 0.8277532
## s4      s4 0.8158118 2.928571 1.1772381 0.7763534 0.8482515
## s5      s5 0.8342496 2.701104 1.0887373 0.7854481 0.8719486
pander(summary_df)
  Sample ICC Mean SD Lower_CI Upper_CI
s1 s1 0.8194 2.632 1.096 0.7773 0.8535
s2 s2 0.7753 2.624 0.9609 0.7216 0.8187
s3b s3b 0.7806 2.618 1.1 0.7205 0.8278
s4 s4 0.8158 2.929 1.177 0.7764 0.8483
s5 s5 0.8342 2.701 1.089 0.7854 0.8719