Funnel Analysis
Sarah McDonald
Updated: July 05, 2022
# discrete colorblind palette
cb_colors <- brewer.pal(n = 8, name = "Dark2")
#show_col(cb_colors)
# custom ggplot2 theme
custom_theme <- theme_minimal() +
theme(strip.text = element_text(size = 16),
axis.text=element_text(size=16),
axis.title=element_text(size=16,face="bold"),
panel.spacing = unit(1, "lines"),
legend.title = element_text(size=16,face="bold"),
legend.text = element_text(size=16),
plot.caption = element_text(size = 16),
plot.title = element_text(size = 20, face = "bold"),
plot.title.position = "plot",
plot.subtitle = element_text(size = 16))
source(here("analysis_scripts", "factor_cols.R"))The purpose of the script is to run funnel statistics for the full pilot. For now, we set up the script using pilot version 7 data.
This script is step 3 of our pilot analysis process. So far we have:
1 Data Description
full <- read.csv(here("cleaned_data", "clean_full_v7.csv"))
full <- factor_cols(full)
completers <- read.csv(here("cleaned_data", "clean_complete_v7.csv"))
noncompleters <- read.csv(here("cleaned_data","clean_noncomplete_v7.csv"))
ads <- read.csv(here("cleaned_data", "clean_ads_v7.csv"))The full dataset is only filtered to include observations in the month of May 2022 and removes empty rows, and has 5,070 observations. The completers data (clean_complete_v7.csv) filters to observations where full_complete indicator is “yes”, vax_status is non-missing, and observations with non-distinct phone numbers are removed. There are 2,357 observations in this dataset. The noncompleters (clean_noncomplete_v7.csv) data filters to observations where full_complete indicator is missing, and there is no criteria on vaccination status and phone number. There are 2,636 observations in this dataset. There is no filtering to the ads data.
The code follows much of the code in this script.
We use the following definitions in our funnel dropoff statistics:
Impressions (Total Count)= the total number of times our ad has been viewedClickthrough (%)= #clicks / #impressionsMessages Sent (%)= #conversations / #clicksConsent Obtained (%)= #consents / #conversationsCore Survey Complete (%)= #forking section completed / #consentsTreatment Complete (%)= #treatment section completed / #forking section completedDemo Questions Complete (%)= #demog section completed / #treatment section completedFull Survey Complete (%)= #full chat completed / #demog section completed
form_percent <- function(dec){
return(paste0(round(dec* 100, 1), "%"))
}
form_cost <- function(num){
if (num == 0) {
return(NA_character_)
} else {
return(paste0("$", round(cost / num, 3)))
}
}2 Funnel Dropoff Statistics, Full Data
In this section, we will show the funnel drop off statistics for the full dataset, and compare dropoff by variables. The first three columns of each table are the drop off statistics for the full data, and doesn’t change between tables. % Previous calculates the percent of users who answered the current question given that they answered the previous question. The % Total Obs columns calculates the percent of users who answer a question.
In the subsetted columns, the percent of total observations is calculated where the divisor is the number of users with that criteria - for example, for the “vax” subset, the divisor is total number of vaccinated users.
full_impressions <- sum(ads$Impressions, na.rm = T)
full_clicks <- sum(ads$Link.clicks, na.rm = T)
full_conversations <- sum(ads$Results, na.rm = T)
full_consents <- sum(full$consent == "yes", na.rm = T)
full_core_complete <- sum(!is.na(full$main_complete), na.rm = T)
full_treatment_complete <- sum(full$treatment_complete == "yes", na.rm = T)
full_demog_complete <- sum(!is.na(full$demog_complete), na.rm = T)
full_full_complete <- sum(full$full_complete == "yes", na.rm = T)
full_dropoff_current <- data.frame(
metric = c("Impressions", "Link Clicks", "Messages Sent",
"Consent Obtained", "Core Survey Complete", "Treatment complete",
"Demographic Questions Complete", "Full Survey Complete"),
total = scales::comma(c(full_impressions, full_clicks, full_conversations, full_consents, full_core_complete, full_treatment_complete, full_demog_complete, full_full_complete)),
perc_of_prev = c("\\-", form_percent(full_clicks / full_impressions),
form_percent(full_conversations / full_clicks),
form_percent(full_consents / full_conversations),
form_percent(full_core_complete / full_consents),
form_percent(full_treatment_complete/full_core_complete),
form_percent(full_demog_complete / full_treatment_complete),
form_percent(full_full_complete / full_demog_complete)),
perc_of_df = c("\\-", "\\-","\\-",
form_percent(full_consents / nrow(full)),
form_percent(full_core_complete / nrow(full)),
form_percent(full_treatment_complete/nrow(full)),
form_percent(full_demog_complete / nrow(full)),
form_percent(full_full_complete/nrow(full))
)
)full_funnel <- function(variable){
if(is.factor(full[[variable]])){
var_list <- levels(full[[variable]])
} else{
var_list <- full %>%
select(variable) %>%
drop_na() %>%
pull() %>%
unique()
}
tbl <- data.frame(metric = c("Consent Obtained", "Core Survey Complete", "Treatment complete",
"Demographic Questions Complete", "Full Survey Complete"))
for(i in var_list){
df <- full[full[[variable]] == i & !is.na(full[[variable]]), ]
consents <- sum(df[, "consent"] == "yes", na.rm = T)
core_complete <- sum(!is.na(df[, "main_complete"]), na.rm = T)
treatment_complete <- sum(df[, "treatment_complete"] == "yes", na.rm = T)
demog_complete <- sum(!is.na(df[, "demog_complete"]), na.rm = T)
full_complete <- sum(df[, "full_complete"] == "yes", na.rm = T)
drop_off <- data.frame(
metric = c("Consent Obtained", "Core Survey Complete", "Treatment complete",
"Demographic Questions Complete", "Full Survey Complete"),
total = scales::comma(c( consents, core_complete, treatment_complete, demog_complete, full_complete)),
perc_of_prev = c("\\-",
form_percent(core_complete / consents),
form_percent(treatment_complete/core_complete),
form_percent(demog_complete / treatment_complete),
form_percent(full_complete / demog_complete)),
perc_of_df = c(
form_percent(consents / nrow(df)),
form_percent(core_complete / nrow(df)),
form_percent(treatment_complete/nrow(df)),
form_percent(demog_complete / nrow(df)),
form_percent(full_complete/nrow(df)))
)
colnames(drop_off)[colnames(drop_off) != "metric"] <- paste(colnames(drop_off)[colnames(drop_off) != "metric"], i, sep = "_")
tbl <- merge(tbl, drop_off, by = "metric")
}
headers <- c()
for(i in 1:length(var_list)){
headers <- append(headers, setNames(3, str_to_title(var_list[i])))
}
final_tbl <- merge(full_dropoff_current, tbl,by = "metric", all = T) %>%
mutate(metric = factor(metric, levels = c("Impressions", "Link Clicks", "Messages Sent",
"Consent Obtained", "Core Survey Complete", "Treatment complete",
"Demographic Questions Complete", "Full Survey Complete"))) %>%
arrange(metric) %>%
replace(is.na(.), "\\-")
kable(final_tbl, caption = "", digits = 3,
col.names = c("Metric", rep(c("Total", "% Previous", "% Total Obs"), length(var_list)+1)))%>%
kable_styling(bootstrap_options = c("striped", "hover")) %>%
add_header_above(c(" " = 4, headers)) %>%
scroll_box(height = "100%")
}2.1 Vax Status
full_funnel("vax_status")| Metric | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs |
|---|---|---|---|---|---|---|---|---|---|
| Impressions | 1,502,722 | - | - | - | - | - | - | - | - |
| Link Clicks | 12,181 | 0.8% | - | - | - | - | - | - | - |
| Messages Sent | 5,137 | 42.2% | - | - | - | - | - | - | - |
| Consent Obtained | 3,305 | 64.3% | 65.2% | 1,819 | - | 99.5% | 1,430 | - | 99.6% |
| Core Survey Complete | 2,791 | 84.4% | 55% | 1,595 | 87.7% | 87.3% | 1,195 | 83.6% | 83.2% |
| Treatment complete | 2,606 | 93.4% | 51.4% | 1,497 | 93.9% | 81.9% | 1,108 | 92.7% | 77.2% |
| Demographic Questions Complete | 2,488 | 95.5% | 49.1% | 1,434 | 95.8% | 78.4% | 1,053 | 95% | 73.3% |
| Full Survey Complete | 2,434 | 97.8% | 48% | 1,401 | 97.7% | 76.6% | 1,032 | 98% | 71.9% |
2.2 Motivation
full_funnel("motive_main")| Metric | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Impressions | 1,502,722 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Link Clicks | 12,181 | 0.8% | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Messages Sent | 5,137 | 42.2% | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Consent Obtained | 3,305 | 64.3% | 65.2% | 584.0 | - | 99.2% | 295.0 | - | 99.3% | 733 | - | 100% | 408.0 | - | 100% |
| Core Survey Complete | 2,791 | 84.4% | 55% | 523.0 | 89.6% | 88.8% | 265.0 | 89.8% | 89.2% | 626 | 85.4% | 85.4% | 349.0 | 85.5% | 85.5% |
| Treatment complete | 2,606 | 93.4% | 51.4% | 498.0 | 95.2% | 84.6% | 247.0 | 93.2% | 83.2% | 590 | 94.2% | 80.5% | 332.0 | 95.1% | 81.4% |
| Demographic Questions Complete | 2,488 | 95.5% | 49.1% | 477.0 | 95.8% | 81% | 234.0 | 94.7% | 78.8% | 566 | 95.9% | 77.2% | 321.0 | 96.7% | 78.7% |
| Full Survey Complete | 2,434 | 97.8% | 48% | 470.0 | 98.5% | 79.8% | 229.0 | 97.9% | 77.1% | 553 | 97.7% | 75.4% | 315.0 | 98.1% | 77.2% |
2.3 Impediment Theme
full_funnel("Analysis.3...impediment.theme")| Metric | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Impressions | 1,502,722 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Link Clicks | 12,181 | 0.8% | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Messages Sent | 5,137 | 42.2% | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Consent Obtained | 3,305 | 64.3% | 65.2% | 575 | - | 67.4% | 686.0 | - | 69.6% | 407.0 | - | 58.3% | 468.0 | - | 62.8% | 614.0 | - | 69.8% | 415.0 | - | 54.9% |
| Core Survey Complete | 2,791 | 84.4% | 55% | 498 | 86.6% | 58.4% | 587.0 | 85.6% | 59.5% | 332.0 | 81.6% | 47.6% | 394.0 | 84.2% | 52.9% | 521.0 | 84.9% | 59.2% | 322.0 | 77.6% | 42.6% |
| Treatment complete | 2,606 | 93.4% | 51.4% | 477 | 95.8% | 55.9% | 549.0 | 93.5% | 55.7% | 299.0 | 90.1% | 42.8% | 361.0 | 91.6% | 48.5% | 489.0 | 93.9% | 55.6% | 295.0 | 91.6% | 39% |
| Demographic Questions Complete | 2,488 | 95.5% | 49.1% | 460 | 96.4% | 53.9% | 529.0 | 96.4% | 53.7% | 283.0 | 94.6% | 40.5% | 338.0 | 93.6% | 45.4% | 468.0 | 95.7% | 53.2% | 276.0 | 93.6% | 36.5% |
| Full Survey Complete | 2,434 | 97.8% | 48% | 446 | 97% | 52.3% | 520.0 | 98.3% | 52.7% | 276.0 | 97.5% | 39.5% | 329.0 | 97.3% | 44.2% | 459.0 | 98.1% | 52.2% | 270.0 | 97.8% | 35.7% |
2.4 Ad Image
full_funnel("ad_image")| Metric | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Impressions | 1,502,722 | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Link Clicks | 12,181 | 0.8% | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Messages Sent | 5,137 | 42.2% | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Consent Obtained | 3,305 | 64.3% | 65.2% | 1,875 | - | 69% | 407.0 | - | 58.3% | 468.0 | - | 62.8% | 415.0 | - | 54.9% |
| Core Survey Complete | 2,791 | 84.4% | 55% | 1,606 | 85.7% | 59.1% | 332.0 | 81.6% | 47.6% | 394.0 | 84.2% | 52.9% | 322.0 | 77.6% | 42.6% |
| Treatment complete | 2,606 | 93.4% | 51.4% | 1,515 | 94.3% | 55.7% | 299.0 | 90.1% | 42.8% | 361.0 | 91.6% | 48.5% | 295.0 | 91.6% | 39% |
| Demographic Questions Complete | 2,488 | 95.5% | 49.1% | 1,457 | 96.2% | 53.6% | 283.0 | 94.6% | 40.5% | 338.0 | 93.6% | 45.4% | 276.0 | 93.6% | 36.5% |
| Full Survey Complete | 2,434 | 97.8% | 48% | 1,425 | 97.8% | 52.4% | 276.0 | 97.5% | 39.5% | 329.0 | 97.3% | 44.2% | 270.0 | 97.8% | 35.7% |
3 Time Variation
Next, we look at funnel statistics by time variation. We look at a few different categories:
| Category | Description |
|---|---|
| Time | Two 12-hour periords, for 8pm to 8am and 8am to 8pm |
| Day | 7 days of the week |
| Day-Time | 14 combinations of time and day |
| Vax Status-Day-Time | 28 combinations of vaccine status, time, and day |
3.1 Day-Time Group
full_funnel("day_time_group")| Metric | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Impressions | 1,502,722 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Link Clicks | 12,181 | 0.8% | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Messages Sent | 5,137 | 42.2% | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Consent Obtained | 3,305 | 64.3% | 65.2% | 41.0 | - | 97.6% | 42.0 | - | 100% | 32.0 | - | 97% | 18.0 | - | 100% | 557 | - | 98.9% | 156.0 | - | 97.5% | 598 | - | 99.5% | 287.0 | - | 99.3% | 512.0 | - | 99.2% | 248.0 | - | 100% | 194.0 | - | 99.5% | 128.0 | - | 99.2% | 383.0 | - | 99.5% | 109.0 | - | 98.2% |
| Core Survey Complete | 2,791 | 84.4% | 55% | 37.0 | 90.2% | 88.1% | 26.0 | 61.9% | 61.9% | 27.0 | 84.4% | 81.8% | 16.0 | 88.9% | 88.9% | 495 | 88.9% | 87.9% | 132.0 | 84.6% | 82.5% | 515 | 86.1% | 85.7% | 233.0 | 81.2% | 80.6% | 417.0 | 81.4% | 80.8% | 206.0 | 83.1% | 83.1% | 158.0 | 81.4% | 81% | 105.0 | 82% | 81.4% | 329.0 | 85.9% | 85.5% | 95.0 | 87.2% | 85.6% |
| Treatment complete | 2,606 | 93.4% | 51.4% | 34.0 | 91.9% | 81% | 26.0 | 100% | 61.9% | 22.0 | 81.5% | 66.7% | 15.0 | 93.8% | 83.3% | 455 | 91.9% | 80.8% | 128.0 | 97% | 80% | 492 | 95.5% | 81.9% | 221.0 | 94.8% | 76.5% | 385.0 | 92.3% | 74.6% | 187.0 | 90.8% | 75.4% | 146.0 | 92.4% | 74.9% | 95.0 | 90.5% | 73.6% | 310.0 | 94.2% | 80.5% | 90.0 | 94.7% | 81.1% |
| Demographic Questions Complete | 2,488 | 95.5% | 49.1% | 32.0 | 94.1% | 76.2% | 26.0 | 100% | 61.9% | 22.0 | 100% | 66.7% | 15.0 | 100% | 83.3% | 438 | 96.3% | 77.8% | 120.0 | 93.8% | 75% | 460 | 93.5% | 76.5% | 215.0 | 97.3% | 74.4% | 366.0 | 95.1% | 70.9% | 176.0 | 94.1% | 71% | 143.0 | 97.9% | 73.3% | 90.0 | 94.7% | 69.8% | 302.0 | 97.4% | 78.4% | 83.0 | 92.2% | 74.8% |
| Full Survey Complete | 2,434 | 97.8% | 48% | 32.0 | 100% | 76.2% | 25.0 | 96.2% | 59.5% | 22.0 | 100% | 66.7% | 15.0 | 100% | 83.3% | 427 | 97.5% | 75.8% | 116.0 | 96.7% | 72.5% | 450 | 97.8% | 74.9% | 214.0 | 99.5% | 74% | 358.0 | 97.8% | 69.4% | 171.0 | 97.2% | 69% | 140.0 | 97.9% | 71.8% | 88.0 | 97.8% | 68.2% | 298.0 | 98.7% | 77.4% | 78.0 | 94% | 70.3% |
3.2 Day
full_funnel("wday")| Metric | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Impressions | 1,502,722 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Link Clicks | 12,181 | 0.8% | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Messages Sent | 5,137 | 42.2% | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Consent Obtained | 3,305 | 64.3% | 65.2% | 83.0 | - | 98.8% | 50.0 | - | 98% | 713 | - | 98.6% | 885 | - | 99.4% | 760 | - | 99.5% | 322.0 | - | 99.4% | 492.0 | - | 99.2% |
| Core Survey Complete | 2,791 | 84.4% | 55% | 63.0 | 75.9% | 75% | 43.0 | 86% | 84.3% | 627 | 87.9% | 86.7% | 748 | 84.5% | 84% | 623 | 82% | 81.5% | 263.0 | 81.7% | 81.2% | 424.0 | 86.2% | 85.5% |
| Treatment complete | 2,606 | 93.4% | 51.4% | 60.0 | 95.2% | 71.4% | 37.0 | 86% | 72.5% | 583 | 93% | 80.6% | 713 | 95.3% | 80.1% | 572 | 91.8% | 74.9% | 241.0 | 91.6% | 74.4% | 400.0 | 94.3% | 80.6% |
| Demographic Questions Complete | 2,488 | 95.5% | 49.1% | 58.0 | 96.7% | 69% | 37.0 | 100% | 72.5% | 558 | 95.7% | 77.2% | 675 | 94.7% | 75.8% | 542 | 94.8% | 70.9% | 233.0 | 96.7% | 71.9% | 385.0 | 96.2% | 77.6% |
| Full Survey Complete | 2,434 | 97.8% | 48% | 57.0 | 98.3% | 67.9% | 37.0 | 100% | 72.5% | 543 | 97.3% | 75.1% | 664 | 98.4% | 74.6% | 529 | 97.6% | 69.2% | 228.0 | 97.9% | 70.4% | 376.0 | 97.7% | 75.8% |
3.3 Time
full_funnel("time_group")| Metric | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs |
|---|---|---|---|---|---|---|---|---|---|
| Impressions | 1,502,722 | - | - | - | - | - | - | - | - |
| Link Clicks | 12,181 | 0.8% | - | - | - | - | - | - | - |
| Messages Sent | 5,137 | 42.2% | - | - | - | - | - | - | - |
| Consent Obtained | 3,305 | 64.3% | 65.2% | 2,317 | - | 99.2% | 988 | - | 99.1% |
| Core Survey Complete | 2,791 | 84.4% | 55% | 1,978 | 85.4% | 84.7% | 813 | 82.3% | 81.5% |
| Treatment complete | 2,606 | 93.4% | 51.4% | 1,844 | 93.2% | 79% | 762 | 93.7% | 76.4% |
| Demographic Questions Complete | 2,488 | 95.5% | 49.1% | 1,763 | 95.6% | 75.5% | 725 | 95.1% | 72.7% |
| Full Survey Complete | 2,434 | 97.8% | 48% | 1,727 | 98% | 74% | 707 | 97.5% | 70.9% |
3.4 Vax Status-Day-Time Group
full_funnel("vax_day_time_group")| Metric | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs | Total | % Previous | % Total Obs |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Impressions | 1,502,722 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Link Clicks | 12,181 | 0.8% | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Messages Sent | 5,137 | 42.2% | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
| Consent Obtained | 3,305 | 64.3% | 65.2% | 13.0 | - | 100% | 16.0 | - | 100% | 10.0 | - | 100% | 10.0 | - | 100% | 240.0 | - | 99.6% | 72.0 | - | 98.6% | 266.0 | - | 99.6% | 129.0 | - | 100% | 206.0 | - | 100% | 106.0 | - | 100% | 90.0 | - | 98.9% | 56.0 | - | 98.2% | 173.0 | - | 99.4% | 43.0 | - | 100% | 27.0 | - | 96.4% | 26 | - | 100% | 21.0 | - | 100% | 8.0 | - | 100% | 311.0 | - | 99.4% | 80.0 | - | 97.6% | 318.0 | - | 99.7% | 152.0 | - | 99.3% | 294.0 | - | 99.7% | 139.0 | - | 100% | 102.0 | - | 100% | 72.0 | - | 100% | 204.0 | - | 99.5% | 65.0 | - | 100% |
| Core Survey Complete | 2,791 | 84.4% | 55% | 11.0 | 84.6% | 84.6% | 11.0 | 68.8% | 68.8% | 7.0 | 70% | 70% | 9.0 | 90% | 90% | 208.0 | 86.7% | 86.3% | 59.0 | 81.9% | 80.8% | 227.0 | 85.3% | 85% | 106.0 | 82.2% | 82.2% | 168.0 | 81.6% | 81.6% | 86.0 | 81.1% | 81.1% | 72.0 | 80% | 79.1% | 46.0 | 82.1% | 80.7% | 147.0 | 85% | 84.5% | 38.0 | 88.4% | 88.4% | 26.0 | 96.3% | 92.9% | 15 | 57.7% | 57.7% | 20.0 | 95.2% | 95.2% | 7.0 | 87.5% | 87.5% | 287.0 | 92.3% | 91.7% | 72.0 | 90% | 87.8% | 288.0 | 90.6% | 90.3% | 127.0 | 83.6% | 83% | 249.0 | 84.7% | 84.4% | 120.0 | 86.3% | 86.3% | 86.0 | 84.3% | 84.3% | 59.0 | 81.9% | 81.9% | 182.0 | 89.2% | 88.8% | 57.0 | 87.7% | 87.7% |
| Treatment complete | 2,606 | 93.4% | 51.4% | 11.0 | 100% | 84.6% | 11.0 | 100% | 68.8% | 6.0 | 85.7% | 60% | 8.0 | 88.9% | 80% | 189.0 | 90.9% | 78.4% | 57.0 | 96.6% | 78.1% | 218.0 | 96% | 81.6% | 100.0 | 94.3% | 77.5% | 155.0 | 92.3% | 75.2% | 80.0 | 93% | 75.5% | 65.0 | 90.3% | 71.4% | 39.0 | 84.8% | 68.4% | 132.0 | 89.8% | 75.9% | 37.0 | 97.4% | 86% | 23.0 | 88.5% | 82.1% | 15 | 100% | 57.7% | 16.0 | 80% | 76.2% | 7.0 | 100% | 87.5% | 266.0 | 92.7% | 85% | 70.0 | 97.2% | 85.4% | 274.0 | 95.1% | 85.9% | 121.0 | 95.3% | 79.1% | 230.0 | 92.4% | 78% | 107.0 | 89.2% | 77% | 81.0 | 94.2% | 79.4% | 56.0 | 94.9% | 77.8% | 178.0 | 97.8% | 86.8% | 53.0 | 93% | 81.5% |
| Demographic Questions Complete | 2,488 | 95.5% | 49.1% | 11.0 | 100% | 84.6% | 11.0 | 100% | 68.8% | 6.0 | 100% | 60% | 8.0 | 100% | 80% | 180.0 | 95.2% | 74.7% | 53.0 | 93% | 72.6% | 199.0 | 91.3% | 74.5% | 97.0 | 97% | 75.2% | 150.0 | 96.8% | 72.8% | 74.0 | 92.5% | 69.8% | 65.0 | 100% | 71.4% | 36.0 | 92.3% | 63.2% | 130.0 | 98.5% | 74.7% | 33.0 | 89.2% | 76.7% | 21.0 | 91.3% | 75% | 15 | 100% | 57.7% | 16.0 | 100% | 76.2% | 7.0 | 100% | 87.5% | 258.0 | 97% | 82.4% | 66.0 | 94.3% | 80.5% | 261.0 | 95.3% | 81.8% | 118.0 | 97.5% | 77.1% | 216.0 | 93.9% | 73.2% | 102.0 | 95.3% | 73.4% | 78.0 | 96.3% | 76.5% | 54.0 | 96.4% | 75% | 172.0 | 96.6% | 83.9% | 50.0 | 94.3% | 76.9% |
| Full Survey Complete | 2,434 | 97.8% | 48% | 11.0 | 100% | 84.6% | 10.0 | 90.9% | 62.5% | 6.0 | 100% | 60% | 8.0 | 100% | 80% | 175.0 | 97.2% | 72.6% | 52.0 | 98.1% | 71.2% | 197.0 | 99% | 73.8% | 97.0 | 100% | 75.2% | 147.0 | 98% | 71.4% | 73.0 | 98.6% | 68.9% | 63.0 | 96.9% | 69.2% | 35.0 | 97.2% | 61.4% | 127.0 | 97.7% | 73% | 31.0 | 93.9% | 72.1% | 21.0 | 100% | 75% | 15 | 100% | 57.7% | 16.0 | 100% | 76.2% | 7.0 | 100% | 87.5% | 252.0 | 97.7% | 80.5% | 63.0 | 95.5% | 76.8% | 253.0 | 96.9% | 79.3% | 117.0 | 99.2% | 76.5% | 211.0 | 97.7% | 71.5% | 98.0 | 96.1% | 70.5% | 77.0 | 98.7% | 75.5% | 53.0 | 98.1% | 73.6% | 171.0 | 99.4% | 83.4% | 47.0 | 94% | 72.3% |
4 Noncompleters
Next, we look more closely at non-completers.
We assign individuals to forks. To do this, each user must have non missing values for vaccination status, motivation, and ability. There are 559 out of 2,636 (21%) noncompleters for which this is the case.
First, let’s look at the noncompleters with missing values in either vaccination status, motivation, or ability (2,077 out of 2,636 (79%)).
fork_tbl <- noncompleters %>%
filter(is.na(vax_status) | is.na(motive) | is.na(ability)) %>%
select(continue_2,continue_3,vax_status,motive,ability,vax_future,best_treatment,ethnicity,income,education,religion,religiosity,politics,location,comfortable,enjoyable) %>%
summarize_all(~ mean(!is.na(.))) %>%
pivot_longer(cols = everything(), names_to = "question", values_to = "completion_rate") %>%
arrange(-completion_rate) %>%
transmute(
completion_rate,
question = case_when(
question == "continue_2" ~ "Consent 1: Ask your opinion on vaccines",
question == "continue_3" ~ "Consent 2: Continue if you're over 18 and wish to start",
question == "vax_status" ~ "Vaccination status",
question == "motive" ~ "Motive to get vaccine",
question == "ability" ~ "Ability to get vaccine",
question == "vax_future" ~ "Get vaccine in future",
question == "best_treatment" ~ "Preferred treatment",
question == "ethnicity" ~ "Ethnicity",
question == "income" ~ "Income",
question == "education" ~ "Education",
question == "religion" ~ "Religion",
question == "religiosity" ~ "Religiosity",
question == "politics" ~ "Politics",
question == "location" ~ "Location",
question == "comfortable" ~ "Chat comfortability",
question == "enjoyable" ~ "Chat enjoyability",
TRUE ~ NA_character_
)
) %>%
mutate(question = factor(question, levels = c(
"Consent 1: Ask your opinion on vaccines",
"Consent 2: Continue if you're over 18 and wish to start",
"Vaccination status",
"Motive to get vaccine",
"Ability to get vaccine",
"Get vaccine in future",
"Preferred treatment",
"Ethnicity",
"Income",
"Education",
"Religion",
"Religiosity","Politics","Location","Chat comfortability","Chat enjoyability")) %>% fct_rev())
ggplot(fork_tbl, aes(x= question, y = completion_rate, group = 1)) +
geom_point(size = 3) +
geom_line() +
scale_y_continuous(limits = c(0, 1), labels = scales::percent_format(accuracy = 1)) +
coord_flip() +
custom_theme +
labs(
x = "",
y = "Completion rate (%)",
caption = paste("Number of Observations:", scales::comma(nrow(noncompleters[is.na(noncompleters$vax_status) |is.na(noncompleters$motive_fork) | is.na(noncompleters$ability), ])))
)
We define the motive fork the following way:
| vax_status | motive_raw | Motive Fork |
|---|---|---|
| unvax | 1: definitely not | definitely not |
| unvax | 2: probably not | maybe |
| unvax | 3: possibly | maybe |
| unvax | 4: probably | maybe |
| unvax | 5: very probably | maybe |
| unvax | 6: definitely | definitely |
| vax | yes! | definitely |
| vax | I was unsure | maybe |
| vax | no, i didn’t | definitely not |
noncompleters_summary <- noncompleters %>%
filter(!is.na(vax_status), !is.na(motive_fork), !is.na(ability)) %>%
group_by(vax_status, motive_fork, ability) %>%
count() %>%
group_by(vax_status) %>%
mutate(per_in_vax_status = form_percent(n/sum(n))) %>%
ungroup() %>%
mutate(per_overall = form_percent(n/sum(n)))
kable(noncompleters_summary, caption = "Noncompleters by Fork", digits = 3,
col.names = c("Vaccination Status",
"Motive", "Ability", "Number of Observations",
"Percent within Vaccination Status",
"Percent Overall"))%>%
kable_styling(bootstrap_options = c("striped", "hover"))| Vaccination Status | Motive | Ability | Number of Observations | Percent within Vaccination Status | Percent Overall |
|---|---|---|---|---|---|
| unvax | definitely | no | 8 | 3.5% | 1.4% |
| unvax | definitely | yes | 19 | 8.3% | 3.4% |
| unvax | definitely not | no | 60 | 26.1% | 10.7% |
| unvax | definitely not | yes | 46 | 20% | 8.2% |
| unvax | maybe | no | 43 | 18.7% | 7.7% |
| unvax | maybe | yes | 54 | 23.5% | 9.7% |
| vax | definitely | no | 40 | 12.2% | 7.2% |
| vax | definitely | yes | 224 | 68.1% | 40.1% |
| vax | definitely not | no | 9 | 2.7% | 1.6% |
| vax | definitely not | yes | 12 | 3.6% | 2.1% |
| vax | maybe | no | 23 | 7% | 4.1% |
| vax | maybe | yes | 21 | 6.4% | 3.8% |
4.1 Dropoff for Unvaccinated Noncompleters by Fork
Next, we look at drop off for unvaccinated noncompleters by our 12 forks. We look at specific questions rather than groups of questions.
fork_analysis <- function(v, m, a){
fork_tbl <- noncompleters %>%
filter(vax_status == v & motive_fork == m & ability == a) %>%
select(continue_2,continue_3,vax_status,motive,ability,vax_future,best_treatment,ethnicity,income,education,religion,religiosity,politics,location,comfortable,enjoyable) %>%
summarize_all(~ mean(!is.na(.))) %>%
pivot_longer(cols = everything(), names_to = "question", values_to = "completion_rate") %>%
arrange(-completion_rate) %>%
transmute(
completion_rate,
question = case_when(
question == "continue_2" ~ "Consent 1: Ask your opinion on vaccines",
question == "continue_3" ~ "Consent 2: Continue if you're over 18 and wish to start",
question == "vax_status" ~ "Vaccination status",
question == "motive" ~ "Motive to get vaccine",
question == "ability" ~ "Ability to get vaccine",
question == "vax_future" ~ "Get vaccine in future",
question == "best_treatment" ~ "Preferred treatment",
question == "ethnicity" ~ "Ethnicity",
question == "income" ~ "Income",
question == "education" ~ "Education",
question == "religion" ~ "Religion",
question == "religiosity" ~ "Religiosity",
question == "politics" ~ "Politics",
question == "location" ~ "Location",
question == "comfortable" ~ "Chat comfortability",
question == "enjoyable" ~ "Chat enjoyability",
TRUE ~ NA_character_
)
) %>%
mutate(question = factor(question, levels = c(
"Consent 1: Ask your opinion on vaccines",
"Consent 2: Continue if you're over 18 and wish to start",
"Vaccination status",
"Motive to get vaccine",
"Ability to get vaccine",
"Get vaccine in future",
"Preferred treatment",
"Ethnicity",
"Income",
"Education",
"Religion",
"Religiosity","Politics","Location","Chat comfortability","Chat enjoyability")) %>% fct_rev())
ggplot(fork_tbl, aes(x= question, y = completion_rate, group = 1)) +
geom_point(size = 3) +
geom_line() +
scale_y_continuous(limits = c(0, 1), labels = scales::percent_format(accuracy = 1)) +
coord_flip() +
custom_theme +
labs(
x = "",
y = "Completion rate (%)",
caption = paste("Number of Observations:", noncompleters_summary[noncompleters_summary$vax_status == v & noncompleters_summary$motive_fork == m & noncompleters_summary$ability == a, "n"])
)
}4.1.1 No Motive, Easy Ability
fork_analysis(v = "unvax", m = "definitely not", a = "yes")
4.1.2 No Motive, Difficult Ability
fork_analysis(v = "unvax", m = "definitely not", a = "no")
4.1.3 Maybe Motive, Easy Ability
fork_analysis(v = "unvax", m = "maybe", a = "yes")
4.1.4 Maybe Motive, Difficult Ability
fork_analysis(v = "unvax", m = "maybe", a = "no")
4.1.5 Definitely Motive, Easy Ability
fork_analysis(v = "unvax", m = "definitely", a = "yes")
4.1.6 Definitely Motive, Difficult Ability
fork_analysis(v = "unvax", m = "definitely", a = "no")
5 Enjoyability and Comfortability
The following figure shows the percentage of completers that chose different levels (very/moderate/not) for the comfortability and enjoyability questions.
5.1 Completers
comfortable <- completers %>%
select(comfortable) %>%
mutate(
comfortable = str_to_sentence(comfortable) %>% str_squish()) %>%
filter(comfortable %in% c("Very comfortable", "Somewhat comfortable", "Not comfortable")) %>%
mutate(comfortable = factor(comfortable, levels = c("Very comfortable", "Somewhat comfortable", "Not comfortable"))) %>%
group_by(comfortable) %>%
count() %>%
ungroup() %>%
mutate(percent = paste(round(n/sum(n) * 100, 0), "%", sep = ""),
percent = ifelse(n/sum(n) * 100 < 6, "", percent),
p = rev(cumsum(rev(n/sum(n)))),
variable= "comfortable")%>%
rename(label = comfortable)%>%
mutate(short_label = case_when(label == "Very comfortable" ~ "very",
label == "Somewhat comfortable" ~"moderately",
label == "Not comfortable" ~ "not"))
enjoyable <- completers %>%
select(enjoyable) %>%
mutate(
enjoyable= str_to_sentence(enjoyable) %>% str_squish()) %>%
filter(enjoyable %in% c("Very enjoyable", "Moderately enjoyable", "Not enjoyable")) %>%
mutate(enjoyable = factor(enjoyable, levels = c("Very enjoyable", "Moderately enjoyable", "Not enjoyable"))) %>%
group_by(enjoyable) %>%
count() %>%
ungroup() %>%
mutate(percent = paste(round(n/sum(n) * 100, 0), "%", sep = ""),
percent = ifelse(n/sum(n) * 100 < 6, "", percent),
p = rev(cumsum(rev(n/sum(n)))),
variable = "enjoyable")%>%
rename(label = enjoyable) %>%
mutate(short_label = case_when(label == "Very enjoyable" ~ "very",
label == "Moderately enjoyable" ~"moderately",
label == "Not enjoyable" ~ "not"))
comfort_enjoy <- rbind(comfortable, enjoyable) %>%
mutate(short_label = factor(short_label, levels = c("very", "moderately", "not")))
comfort_enjoy %>%
mutate(variable_num = as.numeric(as.factor(variable))) %>%
ggplot(aes(x = variable_num, y = n, fill = short_label)) +
geom_bar(position = "fill", stat = "identity",
color = "white", width = .5) +
custom_theme +
scale_fill_manual(values = cb_colors, labels = str_to_title)+
geom_text(aes(label = percent),
color = "white",
position = position_fill(vjust = 0.5),
size = 16/.pt, fontface = "bold") +
scale_x_continuous(limits = c(NA, 2.7),
breaks = c(1, 2),
labels = c("Comfortability", "Enjoyability")) +
scale_y_continuous(breaks = c(0, 1),
labels = c("0%", "100%"))+
labs(x = "", y = "",
title = "Percent of Respondents by Comfortability\nand Enjoyability for Completers",
caption = paste("Nonmissing Comfortability Observations:", scales::comma(sum(comfortable$n)), "\n", "Nonmissing Enjoyability Observations:", scales::comma(sum(enjoyable$n)))) +
geom_text_repel(data = . %>% filter(variable == "enjoyable"),
aes(x= variable_num, y = p- 0.01,
label = str_to_title(short_label),
color = short_label),
nudge_x = 0.4, direction = "y",
hjust = "left", size = 16/.pt) +
scale_color_manual(values = cb_colors) +
theme(legend.position = "none",
panel.grid.major.x = element_blank(),
panel.grid.minor = element_blank()) 
5.2 Noncompleters
comfortable <- noncompleters %>%
select(comfortable) %>%
mutate(
comfortable = str_to_sentence(comfortable) %>% str_squish()) %>%
filter(comfortable %in% c("Very comfortable", "Somewhat comfortable", "Not comfortable")) %>%
mutate(comfortable = factor(comfortable, levels = c("Very comfortable", "Somewhat comfortable", "Not comfortable"))) %>%
group_by(comfortable) %>%
count() %>%
ungroup() %>%
mutate(percent = paste(round(n/sum(n) * 100, 0), "%", sep = ""),
percent = ifelse(n/sum(n) * 100 < 6, "", percent),
p = rev(cumsum(rev(n/sum(n)))),
variable= "comfortable")%>%
rename(label = comfortable)%>%
mutate(short_label = case_when(label == "Very comfortable" ~ "very",
label == "Somewhat comfortable" ~"moderately",
label == "Not comfortable" ~ "not"))
enjoyable <- noncompleters %>%
select(enjoyable) %>%
mutate(
enjoyable= str_to_sentence(enjoyable) %>% str_squish()) %>%
filter(enjoyable %in% c("Very enjoyable", "Moderately enjoyable", "Not enjoyable")) %>%
mutate(enjoyable = factor(enjoyable, levels = c("Very enjoyable", "Moderately enjoyable", "Not enjoyable"))) %>%
group_by(enjoyable) %>%
count() %>%
ungroup() %>%
mutate(percent = paste(round(n/sum(n) * 100, 0), "%", sep = ""),
percent = ifelse(n/sum(n) * 100 < 6, "", percent),
p = rev(cumsum(rev(n/sum(n)))),
variable = "enjoyable")%>%
rename(label = enjoyable) %>%
mutate(short_label = case_when(label == "Very enjoyable" ~ "very",
label == "Moderately enjoyable" ~"moderately",
label == "Not enjoyable" ~ "not"))
comfort_enjoy <- rbind(comfortable, enjoyable) %>%
mutate(short_label = factor(short_label, levels = c("very", "moderately", "not")))
comfort_enjoy %>%
mutate(variable_num = as.numeric(as.factor(variable))) %>%
ggplot(aes(x = variable_num, y = n, fill = short_label)) +
geom_bar(position = "fill", stat = "identity",
color = "white", width = .5) +
custom_theme +
scale_fill_manual(values = cb_colors, labels = str_to_title)+
geom_text(aes(label = percent),
color = "white",
position = position_fill(vjust = 0.5),
size = 16/.pt, fontface = "bold") +
scale_x_continuous(limits = c(NA, 2.7),
breaks = c(1, 2),
labels = c("Comfortability", "Enjoyability")) +
scale_y_continuous(breaks = c(0, 1),
labels = c("0%", "100%"))+
labs(x = "", y = "",
title = "Percent of Respondents by Comfortability\nand Enjoyability for Non-completers",
caption = paste("Nonmissing Comfortability Observations:", scales::comma(sum(comfortable$n)), "\n", "Nonmissing Enjoyability Observations:", scales::comma(sum(enjoyable$n)))) +
geom_text_repel(data = . %>% filter(variable == "enjoyable"),
aes(x= variable_num, y = p- 0.01,
label = str_to_title(short_label),
color = short_label),
nudge_x = 0.4, direction = "y",
hjust = "left", size = 16/.pt) +
scale_color_manual(values = cb_colors) +
theme(legend.position = "none",
panel.grid.major.x = element_blank(),
panel.grid.minor = element_blank()) 