Multi-project assessment of sample quality in cross-national surveys

Replication materials

library(tidyverse) # for manipulating data
library(labelled) # for using labels
library(flextable) # for formatting output tables
library(readr) # for reading csv files
library(broom) # for dealing with annoying output chunks

##SUPLEMENTARY MATERIALS

This document presents details of data transformations, statistical procedures, and exact results of analyses performed in the paper’s main body. Code chunks are hidden by default but can be accessed by clicking the relevant buttons. All calculations rely on the publically available integrated .csv data file.

ESM2e02 <- read_delim("ESM2e02.csv", ";",
                      escape_double = FALSE, 
                      na = "Not applicable", 
                      trim_ws = TRUE, 
                      col_types = cols(
  .default = col_character(),
  T_SURVEY_YEAR = col_double(),
  Q_PROP_FEM_POPULATION_18_74 = col_double(),
  Q_EFFECT_SIZE_PROP_FEMALE_EXTERNAL_18_74_psweights = col_double(),
  Q_EFFECT_SIZE_PROP_FEMALE_EXTERNAL_18_74_dweights_ESS = col_double(),
  Q_EFFECT_SIZE_PROP_FEMALE_EXTERNAL_18_74_no_weights = col_double(),
  Q_PROP_FEM_INTERNAL_KOHLER_psweights_strict = col_double(),
  Q_PROP_FEM_INTERNAL_KOHLER_no_weights_strict = col_double(),
  Q_SUBSAMPLE_SIZE_INTERNAL_KOHLER_strict = col_double(),
  Q_EFFECT_SIZE_PROP_FEMALE_INTERNAL_KOHLER_psweights_strict = col_double(),
  Q_EFFECT_SIZE_PROP_FEMALE_INTERNAL_KOHLER_no_weights_strict = col_double(),
  Q_ABS_BIAS_INTERNAL_KOHLER_psweights_strict = col_double(),
  Q_ABS_BIAS_INTERNAL_KOHLER_no_weights_strict = col_double(),
  Q_BIAS_INTERNAL_KOHLER_psweights_strict = col_double(),
  Q_BIAS_INTERNAL_KOHLER_no_weights_strict = col_double(),
  Q_ABS_BIAS_PROP_FEMALE_EXTERNAL_dweights_ESS = col_double(),
  Q_ABS_BIAS_PROP_FEMALE_EXTERNAL_no_weights = col_double(),
  Q_ABS_BIAS_INTERNAL_KOHLER_dweights_ESS = col_double(),
  Q_ABS_BIAS_INTERNAL_KOHLER_no_weights = col_double(),
  Q_ABS_BIAS_PROP_FEMALE_EXTERNAL_psweights = col_double(),
  Q_ABS_BIAS_PROP_FEMALE_EXTERNAL_no_weights = col_double(),
  Q_ABS_BIAS_INTERNAL_KOHLER_psweights = col_double(),
  Q_ABS_BIAS_INTERNAL_KOHLER_no_weights = col_double()
))

Distribution of weights available across the four survey projects - raw counts: (see Figure 1)

ESM2e02 %>%
  count(T_SURVEY_NAME, T_Weighting_factor2) %>%
  pivot_wider(names_from = T_SURVEY_NAME, values_from = n, values_fill = 0) %>%
  qflextable()

NA

Incidence of weight types within projects - column percentages (see Figure 1)

table_1 <- ESM2e02 %>%
  count(T_SURVEY_NAME, T_Weighting_factor2) %>%
  group_by(T_SURVEY_NAME) %>%
  mutate(Percent = percent(n/sum(n), accuracy = 0.1)) %>% 
  select(T_SURVEY_NAME, T_Weighting_factor2, Percent) %>% 
  ungroup() %>%
  pivot_wider(names_from = T_SURVEY_NAME, values_from = Percent, values_fill = "0%")

disp_table_1 <- qflextable(table_1) %>% theme_booktabs()
disp_table_1

NA

Pearson correlation coefficients between biases resulting internal criteria without weights in a) strict manner and b) according to the lenient approach. Calculations performed for surveys within the two projects where data for implementing the strict approach was available.

??? A tu nie powinniśmy dać zmiany w czasie tak jak na Figure 2???

table_3 <- ESM2e02 %>% filter(T_SURVEY_NAME == "ESS", T_DEFFp_ESS == "DEFF>1") %>%
  mutate(delta1 = Q_ABS_BIAS_PROP_FEMALE_EXTERNAL_dweights_ESS  - Q_ABS_BIAS_PROP_FEMALE_EXTERNAL_no_weights,
         delta2 = Q_ABS_BIAS_INTERNAL_KOHLER_dweights_ESS - Q_ABS_BIAS_INTERNAL_KOHLER_no_weights) %>%
  summarise(meanXdifference_external = mean(delta1),
            medianXdifference_external = median(delta1),
            meanXdifference_internal = mean(delta2),
            medianXdifference_internal = median(delta2), .groups = 'drop') %>%
  pivot_longer(1:4, names_to = c("fun", "difference"), values_to = "values", names_sep = "X") %>%
  pivot_wider(names_from = fun, values_from = values)

  
disp_table_3 <- qflextable(table_3) %>% theme_booktabs()
disp_table_3 <- colformat_num(x = disp_table_3, big.mark = ",", digits = 2, na_str = "N/A")

disp_table_3

Differences between weighted an unweighted bias estimates.

table_4 <- ESM2e02 %>% filter(T_Weighing_factor == "Total weights present in dataset") %>% 
  mutate(delta1 = Q_ABS_BIAS_PROP_FEMALE_EXTERNAL_psweights - Q_ABS_BIAS_PROP_FEMALE_EXTERNAL_no_weights,
         delta2 = Q_ABS_BIAS_INTERNAL_KOHLER_psweights - Q_ABS_BIAS_INTERNAL_KOHLER_no_weights) %>%
  group_by(T_SURVEY_NAME) %>%
  summarise(mean_external = mean(delta1),
            median_external = median(delta1),
            mean_internal = mean(delta2, na.rm = T),
            median_internal = median(delta2, na.rm = T), .groups = 'drop')

disp_table_4 <- qflextable(table_4) %>% theme_booktabs() %>% align_nottext_col(align = "center")
disp_table_4 <- colformat_num(x = disp_table_4, big.mark = ",", digits = 2, na_str = "N/A")

disp_table_4

NA
NA