# Function to perform the test of proportions
perform_prop_test <- function(group1, group2, variable) {
  result <- df %>%
    select(treat, {{variable}}) %>% 
    filter(treat %in% c(group1, group2)) %>%
    group_by(treat) %>%
    summarise(
      n_obs = n(),
      proportion = sum({{variable}}) / n(),
      se = sqrt(proportion * (1 - proportion) / n())
    ) %>%
    ungroup() %>%
    mutate(group = ifelse(treat == group1, group1, group2))
  
  p_value <- prop.test(c(result$proportion * result$n_obs), result$n_obs, correct = FALSE)$p.value
  
  result <- result %>%
    mutate(p_value = p_value) %>%
    select(group, n_obs, proportion, se, p_value)
  
  return(result)
}

# Function to perform the t-test
perform_t_test <- function(group1, group2, variable) {
  result <- df %>%
    filter(treat %in% c(group1, group2)) %>%
    group_by(treat) %>%
    summarise(
      n_obs = n(),
      mean = sum(!!sym(variable)) / n(),
      sd = sd(!!sym(variable), na.rm = TRUE),
      se = sd / sqrt(n_obs)
    ) %>%
    ungroup() %>%
    mutate(group = ifelse(treat == group1, group1, group2))
  
  formula <- formula(paste(as.name(variable), "~ treat"))
  p_value <- t.test(formula, data = df, subset = treat %in% c(group1, group2))$p.value
  
  result <- result %>%
    mutate(p_value = p_value) %>%
    select(group, n_obs, mean, sd, se, p_value)
  
  return(result)
}

Introduction

This script conducts initial experimental comparisons and examines descriptive statistics from pilots 3 and 4 executed on Prolific. Pilot 3 recruited 1,600 participants to test 4 interventions against a control, but the control group was not usable due to a technical issue. In Pilot 4, another 900 participants were recruited to test 2 interventions against a control. Participants were recruited to complete the Main survey for a $0.50 payment. Then, participants were asked to donate their time to complete an additional survey for a $2 donation to the International Rescue Committee.

In Pilot 3, participants in the Main survey were exposed to either:

Control
Light-touch Obligation
Light-touch Opportunity
Hard-core Obligation
Hard-core Opportunity

In Pilot 4, participants in the Main survey were exposed to either:

Control
Medium Obligation
Medium Opportunity

Note on time trends/selection: We were able to accurately observe reported intent to donate (Donate) in the control for Pilot 3 and it was 5 percentage points higher than the control we include in this script from Pilot 4. In addition, the recruiting on Prolific went noticeably slower this pilot, suggesting we are reaching people who are marginally less interested in surveys on charitable giving (or, less interested in surveys in general). Altogether, the much lower rates we observe below for Medium Obligation/Opportunity (and the control) are likely at least partially due to this time trend/selection.

Summary

This pilot suggests that we should use Hard-core Obligation/Opportunity. First, recall that we want Obligation to do NO WORSE than Opportunity in our experiment (either the two are equally good at increasing donation behavior, or Obligation is better). For the Hard-core pair of interventions, we find that:

Obligation does better than Opportunity in reported intention to donate by 0.8pp (SE = 3.3), which is not large, but at least does not go in the opposite direction.
Obligation does better than Opportunity in completed donations by 1.3pp (SE = 3.7), with the same caveats as the last bullet.

Second, and critical to the experimental design, we want Opportunity to do better than Obligation in willingness to be recontacted. We find that:

Opportunity does better than Obligation in willingness to be recontacted by 2.5pp (SE = 3.8).

Lastly, we want to verify that the selected interventions have their intended emotional impact.

Opportunity increases positive emotions by 0.46 (SE = 0.138), while Obligation decreases positive emotion by 0.10 (SE = 0.138), and the difference between the two is statistically distinguishable.
Similarly, Opportunity decreases negative emotions by 0.291 (SE = 0.089), while Obligation increases negative emotion by 0.32 (SE = 0.105), which is again statistically distinguishable.

In comparison, the Light-touch Obligation/Opportunity does worse in changing emotions and in completing donations (i.e., Hard-core generates more donations), although the difference in recontact is larger and statistically distinguishable. The Medium Obligation/Opportunity does generally worse and, in particular, Opportunity consistently outperforms Obligation.

Main Outcomes of Interest

We begin by comparing the main outcomes of interest across treatment groups:

Donate: report wanting to donate in the Main survey.
Complete Vol: actually donate by completing the Volunteer survey.
Recontact: report wanting to be recontacted in the Main survey.
Link: click the link to the IRC website in the Main survey.

In addition, we report the ratio of donate to complete volunteer survey.

# Create table of all outcomes of interest, by treatment group
outcome_table <- df %>%
  group_by(treat) %>%
  select(donation, recontact, link, complete_vol) %>% 
  summarise(
    n_obs = n(),
    donation_mean = mean(donation, na.rm = TRUE),
    donation_se = sd(donation, na.rm = TRUE) / sqrt(n()),
    complete_vol_mean = mean(complete_vol, na.rm = TRUE),
    complete_vol_se = sd(complete_vol, na.rm = TRUE) / sqrt(n()),
    prop_complete = complete_vol_mean / donation_mean,
    recontact_mean = mean(recontact, na.rm = TRUE),
    recontact_se = sd(recontact, na.rm = TRUE) / sqrt(n()),
    link_mean = mean(link, na.rm = TRUE),
    link_se = sd(link, na.rm = TRUE) / sqrt(n())
  ) %>% 
  mutate(
    row_names = c('Control', 'Hard-core Obligation', 'Hard-core Opportunity','Light-touch Obligation', 'Light-touch Opportunity','Medium Obligation', 'Medium Opportunity')
    ) %>% 
  select(row_names,n_obs,donation_mean,donation_se,complete_vol_mean,complete_vol_se,prop_complete,recontact_mean,recontact_se,link_mean,link_se)

# Generate Table
kable(outcome_table, "simple", caption = "Main Outcomes, by Treatment", digits = 3, col.names = c('','# of Obs.','Donate Mean', 'SE', 'Complete Vol Mean','SE','Donate / Complete Vol','Recontact Mean','SE','Link Mean','SE'))

Main Outcomes, by Treatment
	# of Obs.	Donate Mean	SE	Complete Vol Mean	SE	Donate / Complete Vol	Recontact Mean	SE	Link Mean	SE
Control	200	0.605	0.035	0.320	0.033	0.529	0.420	0.035	0.095	0.021
Hard-core Obligation	345	0.751	0.023	0.383	0.026	0.510	0.458	0.027	0.128	0.018
Hard-core Opportunity	354	0.743	0.023	0.370	0.026	0.498	0.483	0.027	0.127	0.018
Light-touch Obligation	356	0.733	0.023	0.337	0.025	0.460	0.424	0.026	0.098	0.016
Light-touch Opportunity	346	0.749	0.023	0.329	0.025	0.440	0.497	0.027	0.150	0.019
Medium Obligation	339	0.687	0.025	0.271	0.024	0.395	0.504	0.027	0.118	0.018
Medium Opportunity	344	0.695	0.025	0.302	0.025	0.435	0.483	0.027	0.102	0.016

Tests of Proportions

For each outcome we test the following comparisons:

Light-touch Obligation v. Opportunity
Hard-core Obligation v. Opportunity
Light-touch v. Hard-core Obligation
Light-touch v. Hard-core Opportunity

# Outcomes list
outcomes <- c("donation", "complete_vol", "recontact", "link")

# List of comparisons to make for each outcome
comparisons <- list(
  c("l_obligation", "l_opportunity"),
  c("h_obligation", "h_opportunity"),
  c("m_obligation", "m_opportunity"),
  c("l_obligation", "h_obligation"),
  c("l_opportunity", "h_opportunity"),
  c("m_obligation", "h_obligation"),
  c("m_obligation", "l_obligation"),
  c("m_opportunity", "h_opportunity"),
  c("m_opportunity", "l_opportunity")
)

# Create an empty data frame to hold the results
tests_proportions <- data.frame(
  "Outcome" = character(),
  "Group 1 Name" = character(),
  "Group 1 N" = integer(),
  "Group 1 Proportion" = numeric(),
  "Group 1 SE" = numeric(),
  "Group 2 Name" = character(),
  "Group 2 N" = integer(),
  "Group 2 Proportion" = numeric(),
  "Group 2 SE" = numeric(),
  "Group 1 - 2 Diff" = numeric(),
  "Diff SE" = numeric(),
  "p-value" = numeric(),
  stringsAsFactors = FALSE
)

# Execute tests for each comparison for each outcome
for (outcome in outcomes) {
  for (comparison in comparisons) {
    group1 <- comparison[1]
    group2 <- comparison[2]
    result <- perform_prop_test(group1, group2, !!sym(outcome))
    
    # Add the results to the data frame
    result_df <- data.frame(
      "Outcome" = outcome,
      "Group 1 Name" = group1,
      "Group 1 N" = result$n_obs[1],
      "Group 1 Proportion" = result$proportion[1],
      "Group 1 SE" = result$se[1],
      "Group 2 Name" = group2,
      "Group 2 N" = result$n_obs[2],
      "Group 2 Proportion" = result$proportion[2],
      "Group 2 SE" = result$se[2],
      "Group 1 - 2 Diff" = result$proportion[1] - result$proportion[2],
      "Diff SE" = sqrt((result$proportion[1] * (1 - result$proportion[1]) / result$n_obs[1]) + (result$proportion[2] * (1 - result$proportion[2]) / result$n_obs[2])),
      "p-value" = result$p_value,
      stringsAsFactors = FALSE
    )
    
    tests_proportions <- rbind(tests_proportions, result_df)
  }
}

# Generate Table
kable(distinct(tests_proportions), "simple", caption = "Tests of Proportions", digits = 3)

Tests of Proportions
Outcome	Group.1.Name	Group.1.N	Group.1.Proportion	Group.1.SE	Group.2.Name	Group.2.N	Group.2.Proportion	Group.2.SE	Group.1…2.Diff	Diff.SE	p.value
donation	l_obligation	356	0.733	0.023	l_opportunity	346	0.749	0.023	-0.015	0.033	0.641
donation	h_obligation	345	0.751	0.023	h_opportunity	354	0.743	0.023	0.008	0.033	0.813
donation	m_obligation	339	0.687	0.025	m_opportunity	344	0.695	0.025	-0.007	0.035	0.833
donation	l_obligation	345	0.751	0.023	h_obligation	356	0.733	0.023	0.018	0.033	0.595
donation	l_opportunity	354	0.743	0.023	h_opportunity	346	0.749	0.023	-0.006	0.033	0.865
donation	m_obligation	345	0.751	0.023	h_obligation	339	0.687	0.025	0.063	0.034	0.065
donation	m_obligation	356	0.733	0.023	l_obligation	339	0.687	0.025	0.046	0.034	0.183
donation	m_opportunity	354	0.743	0.023	h_opportunity	344	0.695	0.025	0.048	0.034	0.157
donation	m_opportunity	346	0.749	0.023	l_opportunity	344	0.695	0.025	0.054	0.034	0.115
complete_vol	l_obligation	356	0.337	0.025	l_opportunity	346	0.329	0.025	0.008	0.036	0.831
complete_vol	h_obligation	345	0.383	0.026	h_opportunity	354	0.370	0.026	0.013	0.037	0.732
complete_vol	m_obligation	339	0.271	0.024	m_opportunity	344	0.302	0.025	-0.031	0.035	0.371
complete_vol	l_obligation	345	0.383	0.026	h_obligation	356	0.337	0.025	0.046	0.036	0.209
complete_vol	l_opportunity	354	0.370	0.026	h_opportunity	346	0.329	0.025	0.041	0.036	0.260
complete_vol	m_obligation	345	0.383	0.026	h_obligation	339	0.271	0.024	0.111	0.036	0.002
complete_vol	m_obligation	356	0.337	0.025	l_obligation	339	0.271	0.024	0.066	0.035	0.060
complete_vol	m_opportunity	354	0.370	0.026	h_opportunity	344	0.302	0.025	0.068	0.036	0.058
complete_vol	m_opportunity	346	0.329	0.025	l_opportunity	344	0.302	0.025	0.027	0.035	0.443
recontact	l_obligation	356	0.424	0.026	l_opportunity	346	0.497	0.027	-0.073	0.038	0.053
recontact	h_obligation	345	0.458	0.027	h_opportunity	354	0.483	0.027	-0.025	0.038	0.507
recontact	m_obligation	339	0.504	0.027	m_opportunity	344	0.483	0.027	0.022	0.038	0.568
recontact	l_obligation	345	0.458	0.027	h_obligation	356	0.424	0.026	0.034	0.037	0.367
recontact	l_opportunity	354	0.483	0.027	h_opportunity	346	0.497	0.027	-0.014	0.038	0.710
recontact	m_obligation	345	0.458	0.027	h_obligation	339	0.504	0.027	-0.046	0.038	0.224
recontact	m_obligation	356	0.424	0.026	l_obligation	339	0.504	0.027	-0.080	0.038	0.034
recontact	m_opportunity	354	0.483	0.027	h_opportunity	344	0.483	0.027	0.000	0.038	0.990
recontact	m_opportunity	346	0.497	0.027	l_opportunity	344	0.483	0.027	0.015	0.038	0.702
link	l_obligation	356	0.098	0.016	l_opportunity	346	0.150	0.019	-0.052	0.025	0.037
link	h_obligation	345	0.128	0.018	h_opportunity	354	0.127	0.018	0.000	0.025	0.987
link	m_obligation	339	0.118	0.018	m_opportunity	344	0.102	0.016	0.016	0.024	0.497
link	l_obligation	345	0.128	0.018	h_obligation	356	0.098	0.016	0.029	0.024	0.221
link	l_opportunity	354	0.127	0.018	h_opportunity	346	0.150	0.019	-0.023	0.026	0.375
link	m_obligation	345	0.128	0.018	h_obligation	339	0.118	0.018	0.010	0.025	0.704
link	m_obligation	356	0.098	0.016	l_obligation	339	0.118	0.018	-0.020	0.024	0.403
link	m_opportunity	354	0.127	0.018	h_opportunity	344	0.102	0.016	0.025	0.024	0.293
link	m_opportunity	346	0.150	0.019	l_opportunity	344	0.102	0.016	0.049	0.025	0.055

Change in Emotions

We next look at post- minus pre-survey emotions using the PANAS questions. We report each emotion individually, as well as aggregated by positive/negative emotion.

df <- df %>%
  mutate(across(starts_with("pre_"), 
                list(~ get(paste0("post_", sub("pre_", "", cur_column()))) - .),
                .names = "change_{sub('pre_', '', .col)}"))

df <- df %>% 
  mutate(
    change_pos = change_inspired + change_enthusiastic + change_purposeful + change_happy,
    change_neg = change_irritable + change_hostile + change_guilty + change_distressed
  )
  
  
emotions_table <- df %>% 
  group_by(treat) %>% 
  select(treat, starts_with("change_")) %>% 
  summarise(across(everything(), list(mean = ~mean(., na.rm = TRUE), se = ~sd(., na.rm = TRUE) / sqrt(n())))) %>% 
  select_if(is.numeric) %>% 
  t()
  
# Generate Table

kable(emotions_table, "simple", caption = "Emotion Outcomes, by Treatment", col.names = c('Control','Hard-core Obligation', 'Hard-core Opportunity','Light-touch Obligation', 'Light-touch Opportunity','Medium Obligation', 'Medium Opportunity'), digits = 3)

Emotion Outcomes, by Treatment
	Control	Hard-core Obligation	Hard-core Opportunity	Light-touch Obligation	Light-touch Opportunity	Medium Obligation	Medium Opportunity
change_inspired_mean	0.165	0.096	0.218	0.110	0.194	0.088	0.218
change_inspired_se	0.053	0.053	0.045	0.045	0.052	0.044	0.044
change_enthusiastic_mean	0.140	0.014	0.113	0.020	0.202	0.012	0.122
change_enthusiastic_se	0.055	0.045	0.042	0.035	0.046	0.040	0.043
change_irritable_mean	-0.130	-0.003	-0.161	-0.107	-0.040	0.027	-0.084
change_irritable_se	0.042	0.039	0.037	0.038	0.034	0.038	0.037
change_purposeful_mean	-0.010	-0.026	0.113	0.011	0.075	-0.068	0.017
change_purposeful_se	0.051	0.051	0.043	0.045	0.046	0.044	0.053
change_happy_mean	-0.010	-0.188	-0.037	-0.073	-0.014	-0.130	-0.035
change_happy_se	0.040	0.040	0.036	0.033	0.042	0.037	0.038
change_distressed_mean	-0.140	0.049	-0.051	-0.124	-0.038	-0.021	-0.119
change_distressed_se	0.033	0.040	0.032	0.037	0.032	0.034	0.032
change_hostile_mean	-0.015	0.075	-0.065	-0.006	0.009	0.065	-0.009
change_hostile_se	0.027	0.035	0.025	0.028	0.027	0.033	0.029
change_guilty_mean	0.070	0.197	-0.014	0.020	0.026	0.236	0.032
change_guilty_se	0.035	0.042	0.035	0.037	0.027	0.045	0.033
change_pos_mean	0.285	-0.104	0.407	0.067	0.457	-0.097	0.323
change_pos_se	0.136	0.138	0.118	0.107	0.138	0.115	0.130
change_neg_mean	-0.215	0.319	-0.291	-0.216	-0.043	0.307	-0.180
change_neg_se	0.089	0.105	0.089	0.093	0.077	0.098	0.082

Tests