ds <- ds %>% rename(researcher = Q1)
ds <- ds %>% rename(grant_id = Q2)
ds <- ds %>% rename(grant_title = Q3)
ds <- ds %>% rename(eligibility = Q4)
ds <- ds %>% rename(research_grant = Q5)
ds <- ds %>% rename(max_award = Q6)
ds <- ds %>% rename(max_directonly = Q7)
ds <- ds %>% rename(min_award = Q8)
ds <- ds %>% rename(numb_awards_peryear = Q9)
ds <- ds %>% rename(total_funding = Q10)
ds <- ds %>% rename(indirect_costs = Q11)
ds <- ds %>% rename(indirect_costs_ratespecify = Q11_2_TEXT)
ds <- ds %>% rename(indirect_costs_demin = Q12)
ds <- ds %>% rename(indirect_costs_demin_specify = Q12_1_TEXT)
ds <- ds %>% rename(payment_method = Q13)
ds <- ds %>% rename(payment_method_other = Q13_4_TEXT)
ds <- ds %>% rename(match_required = Q14)
ds <- ds %>% rename(match_requirement_description = Q15)
ds <- ds %>% rename(multi_year_funding = Q16)
ds <- ds %>% rename(multi_year_funding_description = Q17)
ds <- ds %>% rename(loi = Q18)
ds <- ds %>% rename(need = Q19)
ds <- ds %>% rename(program = Q20)
ds <- ds %>% rename(goals = Q21)
ds <- ds %>% rename(logic_model = Q22)
ds <- ds %>% rename(org_statement = Q23)
ds <- ds %>% rename(budget = Q24)
ds <- ds %>% rename(budget_narrative = Q25)
ds <- ds %>% rename(financial_statements = Q26)
ds <- ds %>% rename(audit = Q27)
ds <- ds %>% rename(evaluation = Q28)
ds <- ds %>% rename(sustainability = Q29)
ds <- ds %>% rename(dei = Q30)
ds <- ds %>% rename(partnerships = Q31)
ds <- ds %>% rename(mou = Q32)
ds <- ds %>% rename(letter_of_support = Q33)
ds <- ds %>% rename(number_attachments = Q34)
ds <- ds %>% rename(other_attachments = Q35)
ds <- ds %>% rename(other_attachments_specify = Q36)
ds <- ds %>% rename(outcome_reports = Q37)
ds <- ds %>% rename(outcome_reports_frequency = Q38)
ds <- ds %>% rename(outcome_reports_frequency_other = Q38_6_TEXT)
ds <- ds %>% rename(financial_reports = Q39)
ds <- ds %>% rename(financial_reports_frequency = Q40)
ds <- ds %>% rename(financial_reports_frequency_other = Q40_6_TEXT)
ds <- ds %>% rename(scoring_rubric = Q41)
ds <- ds %>% rename(scoring_rubric_specify = Q42)
ds <- ds %>% rename(webinar = Q43)
ds <- ds %>% rename(qna = Q44)
ds <- ds %>% rename(office_hours = Q45)
ds <- ds %>% rename(recipient_listing = Q46)
ds <- ds %>% rename(application_tips = Q47)
ds <- ds %>% rename(email = Q48)
ds <- ds %>% rename(phone = Q49)
ds <- ds %>% rename(other_assistance = Q50)
ds <- ds %>% rename(other_assistance_specify = Q51)
# Keep only grants with exactly 2 coders
ds <- ds %>%
group_by(grant_id) %>%
filter(n_distinct(researcher) == 2) %>%
ungroup()colSums(is.na(ds))## researcher grant_id grant_title
## 0 0 0
## eligibility research_grant max_award
## 0 0 8
## max_directonly min_award numb_awards_peryear
## 0 26 6
## total_funding indirect_costs indirect_costs_ratespecify
## 4 0 0
## indirect_costs_demin indirect_costs_demin_specify payment_method
## 0 0 0
## payment_method_other match_required match_requirement_description
## 0 0 0
## multi_year_funding multi_year_funding_description loi
## 0 0 0
## need program goals
## 0 0 0
## logic_model org_statement budget
## 0 0 0
## budget_narrative financial_statements audit
## 0 0 0
## evaluation sustainability dei
## 0 0 0
## partnerships mou letter_of_support
## 0 0 0
## number_attachments other_attachments other_attachments_specify
## 0 0 0
## outcome_reports outcome_reports_frequency outcome_reports_frequency_other
## 0 0 0
## financial_reports financial_reports_frequency financial_reports_frequency_other
## 0 0 0
## scoring_rubric scoring_rubric_specify webinar
## 0 0 0
## qna office_hours recipient_listing
## 0 0 0
## application_tips email phone
## 0 0 0
## other_assistance other_assistance_specify
## 0 0sapply(ds, function(x) sum(x == "", na.rm = TRUE))## researcher grant_id grant_title
## 0 0 0
## eligibility research_grant max_award
## 0 0 0
## max_directonly min_award numb_awards_peryear
## 0 0 0
## total_funding indirect_costs indirect_costs_ratespecify
## 0 0 36
## indirect_costs_demin indirect_costs_demin_specify payment_method
## 4 7 0
## payment_method_other match_required match_requirement_description
## 38 0 34
## multi_year_funding multi_year_funding_description loi
## 0 3 0
## need program goals
## 0 0 0
## logic_model org_statement budget
## 0 0 0
## budget_narrative financial_statements audit
## 0 0 0
## evaluation sustainability dei
## 0 0 0
## partnerships mou letter_of_support
## 0 0 0
## number_attachments other_attachments other_attachments_specify
## 0 0 15
## outcome_reports outcome_reports_frequency outcome_reports_frequency_other
## 0 0 37
## financial_reports financial_reports_frequency financial_reports_frequency_other
## 0 0 37
## scoring_rubric scoring_rubric_specify webinar
## 0 0 0
## qna office_hours recipient_listing
## 0 0 0
## application_tips email phone
## 0 0 0
## other_assistance other_assistance_specify
## 0 33subset(ds, research_grant == "" & researcher == "Corbin Walls")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, research_grant == "" & researcher == "Grant Mobley")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, eligibility == "" & researcher == "Corbin Walls")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, eligibility == "" & researcher == "Grant Mobley")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, letter_of_support == "" & researcher == "Corbin Walls")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, letter_of_support == "" & researcher == "Grant Mobley")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, recipient_listing == "")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, application_tips == "")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, email == "")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr>subset(ds, phone == "")[, c("researcher", "grant_id")]## # A tibble: 0 × 2
## # ℹ 2 variables: researcher <chr>, grant_id <chr># drop all variables that are open-ended text responses
colnames(ds)## [1] "researcher" "grant_id" "grant_title"
## [4] "eligibility" "research_grant" "max_award"
## [7] "max_directonly" "min_award" "numb_awards_peryear"
## [10] "total_funding" "indirect_costs" "indirect_costs_ratespecify"
## [13] "indirect_costs_demin" "indirect_costs_demin_specify" "payment_method"
## [16] "payment_method_other" "match_required" "match_requirement_description"
## [19] "multi_year_funding" "multi_year_funding_description" "loi"
## [22] "need" "program" "goals"
## [25] "logic_model" "org_statement" "budget"
## [28] "budget_narrative" "financial_statements" "audit"
## [31] "evaluation" "sustainability" "dei"
## [34] "partnerships" "mou" "letter_of_support"
## [37] "number_attachments" "other_attachments" "other_attachments_specify"
## [40] "outcome_reports" "outcome_reports_frequency" "outcome_reports_frequency_other"
## [43] "financial_reports" "financial_reports_frequency" "financial_reports_frequency_other"
## [46] "scoring_rubric" "scoring_rubric_specify" "webinar"
## [49] "qna" "office_hours" "recipient_listing"
## [52] "application_tips" "email" "phone"
## [55] "other_assistance" "other_assistance_specify"ds <- ds %>% select(-payment_method_other, -match_requirement_description,
-multi_year_funding_description, -other_attachments_specify,
-outcome_reports_frequency_other, -financial_reports_frequency_other,
-scoring_rubric_specify, -other_assistance_specify)
colnames(ds)## [1] "researcher" "grant_id" "grant_title"
## [4] "eligibility" "research_grant" "max_award"
## [7] "max_directonly" "min_award" "numb_awards_peryear"
## [10] "total_funding" "indirect_costs" "indirect_costs_ratespecify"
## [13] "indirect_costs_demin" "indirect_costs_demin_specify" "payment_method"
## [16] "match_required" "multi_year_funding" "loi"
## [19] "need" "program" "goals"
## [22] "logic_model" "org_statement" "budget"
## [25] "budget_narrative" "financial_statements" "audit"
## [28] "evaluation" "sustainability" "dei"
## [31] "partnerships" "mou" "letter_of_support"
## [34] "number_attachments" "other_attachments" "outcome_reports"
## [37] "outcome_reports_frequency" "financial_reports" "financial_reports_frequency"
## [40] "scoring_rubric" "webinar" "qna"
## [43] "office_hours" "recipient_listing" "application_tips"
## [46] "email" "phone" "other_assistance"# Check that all variables are constistently/properly formatted and structured
str(ds$researcher)## chr [1:40] "Corbin Walls" "Corbin Walls" "Corbin Walls" "Corbin Walls" "Corbin Walls" "Corbin Walls" ...ds$eligibility <- factor(ds$eligibility, levels = c("No, neither nonprofits having a 501(c)(3) status with the IRS, other than institutions of higher education, nor city or township governments are eligible",
"No, but city or township governments are eligible",
"Yes"))
ds$research_grant <- factor(ds$research_grant, levels = c("I don't know", "No", "Yes"))
ds$researcher <- factor(ds$researcher, levels = c("Corbin Walls", "Grant Mobley"))
str(ds$max_award)## chr [1:40] "500,000" "2,699,298" "425000" NA "200000" "1500000" NA "75000" "250000" NA "500000" "6730860" ...ds$eligibility <- factor(ds$eligibility, levels = c("Yes", "No"))
ds$max_award <- ds$max_award %>%
str_remove_all("[$,]") %>% # Remove $ and commas
str_trim() %>% # Trim whitespace
as.numeric()
ds$max_directonly <- factor(ds$max_directonly, levels = c(
"Neither the grant synopsis nor any linked documents clarify whether this maximum includes both direct and indirect costs.",
"No, this is the maximum award amount for total costs, including both indirect and direct costs.",
"Not Applicable as this grant doesn't list a maximum award amount.",
"Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."))
ds$min_award## [1] NA "1000000" NA NA NA "700000" NA "50000"
## [9] "245000" NA NA "1000000" "0" NA NA NA
## [17] NA NA NA NA "$1,000,000 " "$0 " NA NA
## [25] "$0 " NA "0" NA NA NA NA "$1,000,000 "
## [33] NA NA NA "$700,000 " NA "$50,000 " "$245,000 " NAds$min_award <- ds$min_award %>%
str_remove_all("[$,]") %>% # Remove $ and commas
str_trim() %>% # Trim whitespace
as.numeric()
str(ds$numb_awards_peryear)## chr [1:40] "5" "2" "3" "9" NA "75" NA "1" "7" NA "3" "2" "10" "8.5" "12" "6.5" "7" "6" "32" "6" "2" "32" "6" ...ds$numb_awards_peryear <- as.numeric(ds$numb_awards_peryear)## Warning: NAs introduced by coercionds$total_funding <- ds$total_funding %>%
str_remove_all("[$,]") %>% # Remove $ and commas
str_trim() %>% # Trim whitespace
as.numeric()
table(ds$indirect_costs)##
## Administrative or indirect costs are allowed at each applicant's negotiated rate
## 36
## Administrative or indirect costs are capped at: (e.g., 10%, 15%)
## 4ds$indirect_costs <- factor(ds$indirect_costs, levels = c(
"Not Specified", "Administrative costs are not allowed as part of the program budget (I.e., 0%)",
"Administrative or indirect costs are allowed at each applicant's negotiated rate",
"Administrative or indirect costs are capped at: (e.g., 10%, 15%)",
"Administrative or indirect costs are uncapped (i.e., there is no limit)"))
ds$indirect_costs_ratespecify <- as.numeric(gsub("%.*", "", ds$indirect_costs_ratespecify)) / 100
ds$indirect_costs_demin <- factor(ds$indirect_costs_demin, levels = c("Yes (please specify the rate, e.g., 10%)", "No"))
str(ds$indirect_costs_demin)## Factor w/ 2 levels "Yes (please specify the rate, e.g., 10%)",..: 1 1 1 1 1 1 1 1 2 1 ...ds$indirect_costs_demin_specify## [1] "10%" "10%" "10%" "10%" "10%" "10%" "10%" "10%" "" "40%" "" "10%" "10%" "" "10%" "10%" "10%" ""
## [19] "10%" "10%" "10%" "10%" "10%" "10%" "10%" "" "10" "10%" "10%" "" "10%" "10" "10%" "10%" "10%" "10%"
## [37] "10%" "10%" "" "10%"ds$indirect_costs_demin_specify <- as.numeric(gsub("%.*", "", ds$indirect_costs_demin_specify)) / 100
ds$payment_method <- factor(ds$payment_method, levels = c("Other (please specify)",
"Not specified",
"Paid as a reimbursement",
"Paid in advance"))
ds$match_required <- factor(ds$match_required, levels = c("Yes", "No"))
ds$multi_year_funding <- factor(ds$multi_year_funding, levels = c(
"Not specified",
"Multi-year funding is not available, and recipients cannot re-apply for grant funds again in the future",
"Multi-year funding is not available, but recipients can re-apply for grant funds again in the future",
"Multi-year funding is available but not required",
"Multi-year funding is the only available option for recipients"))
ds$loi <- factor(ds$loi, levels = c(
"No/Not Mentioned",
"Encouraged but not required",
"Yes"),
ordered = TRUE)
ds$need <- factor(ds$need, levels = c(
"No/Not Mentioned",
"Encouraged as part of a broader statement or document",
"Encouraged as its own statement or document",
"Required as part of a broader statement or document",
"Required as its own statement or document"),
ordered = TRUE)
ds$program <- factor(ds$program, levels = c(
"No/Not Mentioned",
"Encouraged as part of a broader statement or document",
"Encouraged as its own statement or document",
"Required as part of a broader statement or document",
"Required as its own statement or document"),
ordered = TRUE)
ds$goals <- factor(ds$goals, levels = c(
"No/Not Mentioned",
"Encouraged as part of a broader statement or document",
"Encouraged as its own statement or document",
"Required as part of a broader statement or document",
"Required as its own statement or document"),
ordered = TRUE)
ds$logic_model <- factor(ds$logic_model, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds$org_statement <- factor(ds$org_statement, levels = c(
"No/Not Mentioned",
"Encouraged as part of a broader statement or document",
"Encouraged as its own statement or document",
"Required as part of a broader statement or document",
"Required as its own statement or document"),
ordered = TRUE)
ds$budget <- factor(ds$budget, levels = c(
"No/Not Mentioned",
"Encouraged submission of a budget form provided by the granting organization that includes the information found in a program budget",
"Encouraged submission of the applying organization's own documents",
"Required submission of a budget form provided by the granting organization that includes the information found in a program budget",
"Required submission of the applying organization's own documents"),
ordered = TRUE)
ds$budget_narrative <- factor(ds$budget_narrative, levels = c(
"No/Not Mentioned",
"Encouraged as part of a broader statement or document",
"Encouraged as its own statement or document",
"Required as part of a broader statement or document",
"Required as its own statement or document"),
ordered = TRUE)
ds$financial_statements <- factor(ds$financial_statements, levels = c(
"No/Not Mentioned",
"Encouraged submission of a form provided by the granting organization that includes the information found in financial statements",
"Encouraged submission of the applying organization's own documents",
"Required submission of a form provided by the granting organization that includes the information found in financial statements",
"Required submission of the applying organization's own documents"),
ordered = TRUE)
ds$audit <- factor(ds$audit, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds$evaluation <- factor(ds$evaluation, levels = c(
"No/Not Mentioned",
"Encouraged as part of a broader statement or document",
"Encouraged as its own statement or document",
"Required as part of a broader statement or document",
"Required as its own statement or document"),
ordered = TRUE)
ds$sustainability <- factor(ds$sustainability, levels = c(
"No/Not Mentioned",
"Encouraged as part of a broader statement or document",
"Encouraged as its own statement or document",
"Required as part of a broader statement or document",
"Required as its own statement or document"),
ordered = TRUE)
ds$dei <- factor(ds$dei, levels = c(
"No/Not Mentioned",
"Encouraged as part of a broader statement or document",
"Encouraged as its own statement or document",
"Required as part of a broader statement or document",
"Required as its own statement or document"),
ordered = TRUE)
ds$partnerships <- factor(ds$partnerships, levels = c(
"Neither requires nor prioritizes partnerships or MOUs",
"Prioritizes partnerships or MOUs",
"Requires partnerships or MOUs"),
ordered = TRUE)
ds$mou <- factor(ds$mou, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds$letter_of_support <- factor(ds$letter_of_support, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds$number_attachments <- as.numeric(ds$number_attachments)
ds$other_attachments <- factor(ds$other_attachments, levels = c("Yes", "No"))
ds$outcome_reports <- factor(ds$outcome_reports, levels = c(
"Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds$outcome_reports_frequency <- factor(ds$outcome_reports_frequency, levels = c(
"Other (please specify)",
"Once, after the grant period has concluded",
"Annually",
"Semi-Annually",
"Quarterly",
"Monthly"))
ds$financial_reports <- factor(ds$financial_reports, levels = c(
"Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds$financial_reports_frequency <- factor(ds$financial_reports_frequency, levels = c(
"Other (please specify)",
"Once, after the grant period has concluded",
"Annually",
"Semi-Annually",
"Quarterly",
"Monthly"))
ds$scoring_rubric <- factor(ds$scoring_rubric, levels = c("Yes", "No"))
ds$webinar <- factor(ds$webinar, levels = c("Yes", "No"))
ds$qna <- factor(ds$qna, levels = c("Yes", "No"))
ds$office_hours <- factor(ds$office_hours, levels = c("Yes", "No"))
ds$recipient_listing <- factor(ds$recipient_listing, levels = c("Yes", "No"))
ds$application_tips <- factor(ds$application_tips, levels = c("Yes", "No"))
ds$email <- factor(ds$email, levels = c("Yes", "No"))
ds$phone <- factor(ds$phone, levels = c("Yes", "No"))
ds$other_assistance <- factor(ds$other_assistance, levels = c("Yes", "No"))# List of variables with moderate or low percent agreement (<85%)
vars <- setdiff(names(ds), c("researcher", "grant_id", "grant_title"))
# Create a list of matrices, one per variable
matrix_list <- list()
matrix_list_raw <- list()
for (var in vars) {
temp_matrix <- ds %>%
select(grant_id, researcher, all_of(var)) %>%
pivot_wider(names_from = researcher, values_from = all_of(var)) %>%
select(-grant_id) %>%
as.matrix()
# Store matrix with original values (character or factor)
matrix_list_raw[[var]] <- temp_matrix
}# Get variable names to check (exclude identifying variables)
vars_to_check <- ds %>%
select(-researcher, -grant_id, -grant_title) %>%
colnames()
# Pivot data to coder-columns
ds_wide <- ds %>%
select(researcher, grant_id, all_of(vars_to_check)) %>%
pivot_wider(names_from = researcher, values_from = all_of(vars_to_check),
names_sep = "_")
# Calculate percent agreement for each variable
percent_agreement <- sapply(vars_to_check, function(var) {
coder1 <- ds_wide[[paste0(var, "_Corbin Walls")]]
coder2 <- ds_wide[[paste0(var, "_Grant Mobley")]]
agree <- mapply(function(x, y) {
if (is.na(x) && is.na(y)) {
return(TRUE) # Treat both missing as agreement
} else if (is.na(x) || is.na(y)) {
return(FALSE) # One missing, one not = disagreement
} else {
return(identical(x, y)) # Exact match
}
}, coder1, coder2)
if (length(agree) == 0) {
return(NA)
} else {
return(mean(agree) * 100)
}
})
# Format as a dataframe
percent_agreement_df_disaggregate <- data.frame(
Variable = vars_to_check,
PercentAgreement = round(percent_agreement, 1)
)
percent_agreement_df_disaggregate <- percent_agreement_df_disaggregate %>% mutate(agreement_cat = case_when(
PercentAgreement >= 90 ~ "Very Strong",
PercentAgreement < 90 & PercentAgreement >= 80 ~ "Strong",
PercentAgreement < 80 & PercentAgreement >= 70 ~ "Moderate",
PercentAgreement < 70 ~ "Low"))
percent_agreement_df_disaggregate$agreement_cat <- factor(percent_agreement_df_disaggregate$agreement_cat, levels = c("Low", "Moderate", "Strong", "Very Strong"), ordered = TRUE)
arrange(percent_agreement_df_disaggregate, desc(agreement_cat), desc(PercentAgreement))## Variable PercentAgreement agreement_cat
## eligibility eligibility 100 Very Strong
## research_grant research_grant 100 Very Strong
## indirect_costs indirect_costs 100 Very Strong
## indirect_costs_ratespecify indirect_costs_ratespecify 100 Very Strong
## match_required match_required 100 Very Strong
## budget budget 100 Very Strong
## outcome_reports outcome_reports 100 Very Strong
## financial_reports financial_reports 100 Very Strong
## scoring_rubric scoring_rubric 100 Very Strong
## recipient_listing recipient_listing 100 Very Strong
## total_funding total_funding 95 Very Strong
## indirect_costs_demin indirect_costs_demin 95 Very Strong
## audit audit 95 Very Strong
## outcome_reports_frequency outcome_reports_frequency 95 Very Strong
## office_hours office_hours 95 Very Strong
## email email 95 Very Strong
## max_award max_award 90 Very Strong
## min_award min_award 90 Very Strong
## numb_awards_peryear numb_awards_peryear 90 Very Strong
## indirect_costs_demin_specify indirect_costs_demin_specify 90 Very Strong
## multi_year_funding multi_year_funding 90 Very Strong
## loi loi 90 Very Strong
## mou mou 90 Very Strong
## financial_reports_frequency financial_reports_frequency 90 Very Strong
## max_directonly max_directonly 85 Strong
## need need 85 Strong
## financial_statements financial_statements 85 Strong
## phone phone 85 Strong
## logic_model logic_model 80 Strong
## webinar webinar 80 Strong
## evaluation evaluation 75 Moderate
## qna qna 75 Moderate
## goals goals 70 Moderate
## org_statement org_statement 70 Moderate
## sustainability sustainability 70 Moderate
## payment_method payment_method 65 Low
## letter_of_support letter_of_support 65 Low
## application_tips application_tips 65 Low
## other_assistance other_assistance 65 Low
## partnerships partnerships 50 Low
## other_attachments other_attachments 35 Low
## dei dei 25 Low
## budget_narrative budget_narrative 20 Low
## program program 10 Low
## number_attachments number_attachments 0 LowA lot of the application content variables are underperforming what I had expected. I think we might want to just aggregate this to Required, Encouraged, or Not Mentioned. If we do that, does the ICR get better?
ds <- ds %>% mutate(need = case_when(
need == "No/Not Mentioned" ~ "No/Not Mentioned",
need == "Encouraged as part of a broader statement or document" ~ "Encouraged",
need == "Encouraged as its own statement or document" ~ "Encouraged",
need == "Required as part of a broader statement or document" ~ "Required",
need == "Required as its own statement or document" ~ "Required"))
ds$need <- factor(ds$need, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds <- ds %>% mutate(program = case_when(
program == "No/Not Mentioned" ~ "No/Not Mentioned",
program == "Encouraged as part of a broader statement or document" ~ "Encouraged",
program == "Encouraged as its own statement or document" ~ "Encouraged",
program == "Required as part of a broader statement or document" ~ "Required",
program == "Required as its own statement or document" ~ "Required"))
ds$program <- factor(ds$program, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds <- ds %>% mutate(goals = case_when(
goals == "No/Not Mentioned" ~ "No/Not Mentioned",
goals == "Encouraged as part of a broader statement or document" ~ "Encouraged",
goals == "Encouraged as its own statement or document" ~ "Encouraged",
goals == "Required as part of a broader statement or document" ~ "Required",
goals == "Required as its own statement or document" ~ "Required"))
ds$goals <- factor(ds$goals, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds <- ds %>% mutate(org_statement = case_when(
org_statement == "No/Not Mentioned" ~ "No/Not Mentioned",
org_statement == "Encouraged as part of a broader statement or document" ~ "Encouraged",
org_statement == "Encouraged as its own statement or document" ~ "Encouraged",
org_statement == "Required as part of a broader statement or document" ~ "Required",
org_statement == "Required as its own statement or document" ~ "Required"))
ds$org_statement <- factor(ds$org_statement, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds <- ds %>% mutate(budget_narrative = case_when(
budget_narrative == "No/Not Mentioned" ~ "No/Not Mentioned",
budget_narrative == "Encouraged as part of a broader statement or document" ~ "Encouraged",
budget_narrative == "Encouraged as its own statement or document" ~ "Encouraged",
budget_narrative == "Required as part of a broader statement or document" ~ "Required",
budget_narrative == "Required as its own statement or document" ~ "Required"))
ds$budget_narrative## [1] "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required"
## [11] "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required"
## [21] "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required"
## [31] "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required" "Required"ds$budget_narrative <- factor(ds$budget_narrative, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds <- ds %>% mutate(financial_statements = case_when(
financial_statements == "No/Not Mentioned" ~ "No/Not Mentioned",
financial_statements == "Encouraged submission of a form provided by the granting organization that includes the information found in financial statements" ~ "Encouraged",
financial_statements == "Encouraged submission of the applying organization's own documents" ~ "Encouraged",
financial_statements == "Required submission of a form provided by the granting organization that includes the information found in financial statements" ~ "Required",
financial_statements == "Required submission of the applying organization's own documents" ~ "Required"))
ds$financial_statements <- factor(ds$financial_statements, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds <- ds %>% mutate(evaluation = case_when(
evaluation == "No/Not Mentioned" ~ "No/Not Mentioned",
evaluation == "Encouraged as part of a broader statement or document" ~ "Encouraged",
evaluation == "Encouraged as its own statement or document" ~ "Encouraged",
evaluation == "Required as part of a broader statement or document" ~ "Required",
evaluation == "Required as its own statement or document" ~ "Required"))
ds$evaluation <- factor(ds$evaluation, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds <- ds %>% mutate(sustainability = case_when(
sustainability == "No/Not Mentioned" ~ "No/Not Mentioned",
sustainability == "Encouraged as part of a broader statement or document" ~ "Encouraged",
sustainability == "Encouraged as its own statement or document" ~ "Encouraged",
sustainability == "Required as part of a broader statement or document" ~ "Required",
sustainability == "Required as its own statement or document" ~ "Required"))
ds$sustainability <- factor(ds$sustainability, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
ds <- ds %>% mutate(dei = case_when(
dei == "No/Not Mentioned" ~ "No/Not Mentioned",
dei == "Encouraged as part of a broader statement or document" ~ "Encouraged",
dei == "Encouraged as its own statement or document" ~ "Encouraged",
dei == "Required as part of a broader statement or document" ~ "Required",
dei == "Required as its own statement or document" ~ "Required"))
ds$dei <- factor(ds$dei, levels = c(
"No/Not Mentioned",
"Encouraged",
"Required"),
ordered = TRUE)
colnames(ds)## [1] "researcher" "grant_id" "grant_title"
## [4] "eligibility" "research_grant" "max_award"
## [7] "max_directonly" "min_award" "numb_awards_peryear"
## [10] "total_funding" "indirect_costs" "indirect_costs_ratespecify"
## [13] "indirect_costs_demin" "indirect_costs_demin_specify" "payment_method"
## [16] "match_required" "multi_year_funding" "loi"
## [19] "need" "program" "goals"
## [22] "logic_model" "org_statement" "budget"
## [25] "budget_narrative" "financial_statements" "audit"
## [28] "evaluation" "sustainability" "dei"
## [31] "partnerships" "mou" "letter_of_support"
## [34] "number_attachments" "other_attachments" "outcome_reports"
## [37] "outcome_reports_frequency" "financial_reports" "financial_reports_frequency"
## [40] "scoring_rubric" "webinar" "qna"
## [43] "office_hours" "recipient_listing" "application_tips"
## [46] "email" "phone" "other_assistance"# List of variables with moderate or low percent agreement (<85%)
vars <- setdiff(names(ds), c("researcher", "grant_id", "grant_title"))
# Create a list of matrices, one per variable
matrix_list <- list()
matrix_list_raw <- list()
for (var in vars) {
temp_matrix <- ds %>%
select(grant_id, researcher, all_of(var)) %>%
pivot_wider(names_from = researcher, values_from = all_of(var)) %>%
select(-grant_id) %>%
as.matrix()
# Store matrix with original values (character or factor)
matrix_list_raw[[var]] <- temp_matrix
}# Get variable names to check (exclude identifying variables)
vars_to_check <- ds %>%
select(-researcher, -grant_id, -grant_title) %>%
colnames()
# Pivot data to coder-columns
ds_wide <- ds %>%
select(researcher, grant_id, all_of(vars_to_check)) %>%
pivot_wider(names_from = researcher, values_from = all_of(vars_to_check),
names_sep = "_")
# Calculate percent agreement for each variable
percent_agreement <- sapply(vars_to_check, function(var) {
coder1 <- ds_wide[[paste0(var, "_Corbin Walls")]]
coder2 <- ds_wide[[paste0(var, "_Grant Mobley")]]
agree <- mapply(function(x, y) {
if (is.na(x) && is.na(y)) {
return(TRUE) # Treat both missing as agreement
} else if (is.na(x) || is.na(y)) {
return(FALSE) # One missing, one not = disagreement
} else {
return(identical(x, y)) # Exact match
}
}, coder1, coder2)
if (length(agree) == 0) {
return(NA)
} else {
return(mean(agree) * 100)
}
})
# Format as a dataframe
percent_agreement_df <- data.frame(
Variable = vars_to_check,
PercentAgreement = round(percent_agreement, 1)
)
percent_agreement_df <- percent_agreement_df %>% mutate(agreement_cat = case_when(
PercentAgreement >= 90 ~ "Very Strong",
PercentAgreement < 90 & PercentAgreement >= 80 ~ "Strong",
PercentAgreement < 80 & PercentAgreement >= 70 ~ "Moderate",
PercentAgreement < 70 ~ "Low"))
percent_agreement_df$agreement_cat <- factor(percent_agreement_df$agreement_cat, levels = c("Low", "Moderate", "Strong", "Very Strong"), ordered = TRUE)
arrange(percent_agreement_df, desc(agreement_cat), desc(PercentAgreement))## Variable PercentAgreement agreement_cat
## eligibility eligibility 100 Very Strong
## research_grant research_grant 100 Very Strong
## indirect_costs indirect_costs 100 Very Strong
## indirect_costs_ratespecify indirect_costs_ratespecify 100 Very Strong
## match_required match_required 100 Very Strong
## program program 100 Very Strong
## org_statement org_statement 100 Very Strong
## budget budget 100 Very Strong
## budget_narrative budget_narrative 100 Very Strong
## outcome_reports outcome_reports 100 Very Strong
## financial_reports financial_reports 100 Very Strong
## scoring_rubric scoring_rubric 100 Very Strong
## recipient_listing recipient_listing 100 Very Strong
## total_funding total_funding 95 Very Strong
## indirect_costs_demin indirect_costs_demin 95 Very Strong
## goals goals 95 Very Strong
## audit audit 95 Very Strong
## outcome_reports_frequency outcome_reports_frequency 95 Very Strong
## office_hours office_hours 95 Very Strong
## email email 95 Very Strong
## max_award max_award 90 Very Strong
## min_award min_award 90 Very Strong
## numb_awards_peryear numb_awards_peryear 90 Very Strong
## indirect_costs_demin_specify indirect_costs_demin_specify 90 Very Strong
## multi_year_funding multi_year_funding 90 Very Strong
## loi loi 90 Very Strong
## need need 90 Very Strong
## mou mou 90 Very Strong
## financial_reports_frequency financial_reports_frequency 90 Very Strong
## max_directonly max_directonly 85 Strong
## financial_statements financial_statements 85 Strong
## phone phone 85 Strong
## logic_model logic_model 80 Strong
## evaluation evaluation 80 Strong
## webinar webinar 80 Strong
## qna qna 75 Moderate
## sustainability sustainability 70 Moderate
## payment_method payment_method 65 Low
## letter_of_support letter_of_support 65 Low
## application_tips application_tips 65 Low
## other_assistance other_assistance 65 Low
## partnerships partnerships 50 Low
## other_attachments other_attachments 35 Low
## dei dei 25 Low
## number_attachments number_attachments 0 LowWhen aggregating, the following variables move upward in the ICR categories
Low to Very Strong
programModerate to Very Strong
org_statement
goals
Moderate to Strong
evaluationStrong to Very Strong
needAccordingly, we will aggregate these codes moving forward. We should, however, review the evaluation code since it still isn’t in the Very Strong cetagory.
The following variables are not in the Very Strong category:
Currently Strong
max_directonly
financial_statements
research_grant
logic_model
evaluation
webinar
Currently Moderate
qna
sustainability
Currently Low
payment_method
other_assistance
partnerships
other_attachments
dei
number_attachments
We need to review the discrepancies for 10 variables:
max_directonly,
financial_statements,
research_grant,
logic_model,
evaluation,
qna,
sustainability,
payment_method,
partnerships,
dei.
Removal
I think we should remove other_assistance from the
coding scheme and simply add in variables for the additional kinds of
assistance that we have identified in the open text variable here. I
also think we should remove other_attachments from the
coding scheme and simply add in variables for the attachments/statements
that we have identified in the open text variable here. Lastly, I think
we should remove number_attachments from the codebook since
that one is kind of hard to calculate, and, due to differences in
intensities of each question, it really isn’t that insightful of a
measure.
eligibility_matrix <- ds %>%
select(researcher, grant_id, eligibility) %>%
pivot_wider(names_from = researcher, values_from = eligibility) %>%
select(-grant_id) %>%
as.matrix()
eligibility_vec <- as.vector(eligibility_matrix)
eligibility_factor <- factor(
eligibility_vec,
levels = c("No, neither nonprofits having a 501(c)(3) status with the IRS, other than institutions of higher education, nor city or township governments are eligible",
"No, but city or township governments are eligible",
"Yes"),
ordered = FALSE
)
eligibility_matrix_num <- matrix(as.numeric(eligibility_factor), ncol = 2)
colnames(eligibility_matrix_num) <- colnames(eligibility_matrix)
result_eligibility <- krippendorffs.alpha(
eligibility_matrix_num,
level = "ordinal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_eligibility)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = eligibility_matrix_num, level = "ordinal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NaN NAtable(eligibility_matrix_num[, 1])##
## 3
## 20table(eligibility_matrix_num[, 2])##
## 3
## 20all(eligibility_matrix_num[, 1] == eligibility_matrix_num[, 2], na.rm = TRUE)## [1] TRUEEligibility has 100% agreement, so no need to run the Krippendorff’s Alpha.
research_grant_matrix <- ds %>%
select(researcher, grant_id, research_grant) %>%
pivot_wider(names_from = researcher, values_from = research_grant) %>%
select(-grant_id) %>%
as.matrix()
research_grant_vec <- as.vector(research_grant_matrix)
research_grant_factor <- factor(
research_grant_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
research_grant_matrix_num <- matrix(as.numeric(research_grant_factor), ncol = 2)
colnames(research_grant_matrix_num) <- colnames(research_grant_matrix)
result_research_grant <- krippendorffs.alpha(
research_grant_matrix_num,
level = "ordinal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_research_grant)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = research_grant_matrix_num, level = "ordinal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NA NAtable(research_grant_matrix_num[, 1])##
## 1 2
## 13 7table(research_grant_matrix_num[, 2])##
## 1 2
## 13 7all(research_grant_matrix_num[, 1] == research_grant_matrix_num[, 2], na.rm = TRUE)## [1] TRUEResearch Grant has 100% agreement, so no need to run the Krippendorff’s Alpha.
max_award_matrix <- ds %>%
select(researcher, grant_id, max_award) %>%
pivot_wider(names_from = researcher, values_from = max_award) %>%
select(-grant_id) %>%
as.matrix()
max_award_matrix_num <- apply(max_award_matrix, 2, as.numeric)
result_max_award <- krippendorffs.alpha(
max_award_matrix_num,
level = "interval",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_max_award)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = max_award_matrix_num, level = "interval",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.9999 0.9997 1max_directonly_matrix <- ds %>%
select(researcher, grant_id, max_directonly) %>%
pivot_wider(names_from = researcher, values_from = max_directonly) %>%
select(-grant_id) %>%
as.matrix()
max_directonly_vec <- as.vector(max_directonly_matrix)
max_directonly_factor <- factor(
max_directonly_vec,
levels = c(
"Neither the grant synopsis nor any linked documents clarify whether this maximum includes both direct and indirect costs.",
"No, this is the maximum award amount for total costs, including both indirect and direct costs.",
"Not Applicable as this grant doesn't list a maximum award amount.",
"Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
),
ordered = FALSE
)
max_directonly_matrix_num <- matrix(as.numeric(max_directonly_factor), ncol = 2)
colnames(max_directonly_matrix_num) <- colnames(max_directonly_matrix)
result_max_directonly <- krippendorffs.alpha(
max_directonly_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_max_directonly)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = max_directonly_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.7763 0.5526 1min_award_matrix <- ds %>%
select(researcher, grant_id, min_award) %>%
pivot_wider(names_from = researcher, values_from = min_award) %>%
select(-grant_id) %>%
as.matrix()
min_award_matrix_num <- apply(min_award_matrix, 2, function(x) as.numeric(as.factor(x)))
# Remove rows with any NAs
min_award_matrix_num_clean <- min_award_matrix_num[complete.cases(min_award_matrix_num), ]
# Re-run Krippendorff’s alpha
result_min_award <- krippendorffs.alpha(
min_award_matrix_num_clean,
level = "interval",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_min_award)##
## Krippendorff's Alpha
##
## Data: 6 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = min_award_matrix_num_clean, level = "interval",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NA NAtable(min_award_matrix_num[, 1])##
## 1 2 3 4 5
## 1 1 1 1 2table(min_award_matrix_num[, 2])##
## 1 2 3 4 5
## 3 1 1 1 2all(min_award_matrix_num[, 1] == min_award_matrix_num[, 2], na.rm = TRUE)## [1] TRUEMinimum Award has 100% agreement, so no need to run the Krippendorff’s Alpha.
numb_awards_peryear_matrix <- ds %>%
select(researcher, grant_id, numb_awards_peryear) %>%
pivot_wider(names_from = researcher, values_from = numb_awards_peryear) %>%
select(-grant_id) %>%
as.matrix()
numb_awards_peryear_matrix_num <- apply(numb_awards_peryear_matrix, 2, as.numeric)
result_numb_awards_peryear <- krippendorffs.alpha(
numb_awards_peryear_matrix_num,
level = "interval",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_numb_awards_peryear)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = numb_awards_peryear_matrix_num, level = "interval",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NA NAtable(numb_awards_peryear_matrix_num[, 1])##
## 1 2 3 5 6 6.5 7 8.5 9 10 12 32 75
## 1 2 2 1 2 1 2 1 1 1 1 1 1table(numb_awards_peryear_matrix_num[, 2])##
## 1 2 3 5 6 7 9 10 12 32 75
## 1 2 2 1 2 2 1 1 1 1 1all(numb_awards_peryear_matrix_num[, 1] == numb_awards_peryear_matrix_num[, 2], na.rm = TRUE)## [1] TRUENumber of awards per year has 100% agreement, so no need to run the Krippendorff’s Alpha.
FLAG: I CHANGED THE CODING INSTRUCTIONS ON THE SURVEY.
total_funding_matrix <- ds %>%
select(researcher, grant_id, total_funding) %>%
pivot_wider(names_from = researcher, values_from = total_funding) %>%
select(-grant_id) %>%
as.matrix()
total_funding_matrix_num <- apply(total_funding_matrix, 2, as.numeric)
result_total_funding <- krippendorffs.alpha(
total_funding_matrix_num,
level = "interval",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_total_funding)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = total_funding_matrix_num, level = "interval",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.9999 0.9998 1indirect_costs_matrix <- ds %>%
select(researcher, grant_id, indirect_costs) %>%
pivot_wider(names_from = researcher, values_from = indirect_costs) %>%
select(-grant_id) %>%
as.matrix()
indirect_costs_vec <- as.vector(indirect_costs_matrix)
indirect_costs_factor <- factor(
indirect_costs_vec,
levels = c(
"Not Specified",
"Administrative costs are not allowed as part of the program budget (I.e., 0%)",
"Administrative or indirect costs are allowed at each applicant's negotiated rate",
"Administrative or indirect costs are capped at: (e.g., 10%, 15%)",
"Administrative or indirect costs are uncapped (i.e., there is no limit)"
),
ordered = FALSE
)
indirect_costs_matrix_num <- matrix(as.numeric(indirect_costs_factor), ncol = 2)
colnames(indirect_costs_matrix_num) <- colnames(indirect_costs_matrix)
result_indirect_costs <- krippendorffs.alpha(
indirect_costs_matrix_num,
level = "nominal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2))
summary(result_indirect_costs)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = indirect_costs_matrix_num, level = "nominal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NA NAtable(numb_awards_peryear_matrix_num[, 1])##
## 1 2 3 5 6 6.5 7 8.5 9 10 12 32 75
## 1 2 2 1 2 1 2 1 1 1 1 1 1table(numb_awards_peryear_matrix_num[, 2])##
## 1 2 3 5 6 7 9 10 12 32 75
## 1 2 2 1 2 2 1 1 1 1 1all(numb_awards_peryear_matrix_num[, 1] == numb_awards_peryear_matrix_num[, 2], na.rm = TRUE)## [1] TRUEIndirect costs has 100% agreement, so no need to run the Krippendorff’s Alpha.
Too few cases to run the Kirppendorff’s Alpha
indirect_costs_ratespecify_matrix <- ds %>% select(researcher, grant_id, indirect_costs_ratespecify) %>% pivot_wider(names_from = researcher, values_from = indirect_costs_ratespecify) %>% select(-grant_id) %>% as.matrix()
indirect_costs_ratespecify_matrix_num <- apply(indirect_costs_ratespecify_matrix, 2, function(x) as.numeric(as.factor(x)))
Remove rows with any NA values
indirect_costs_ratespecify_matrix_num_clean <- indirect_costs_ratespecify_matrix_num[complete.cases(indirect_costs_ratespecify_matrix_num),]
Re-run Krippendorff’s alpha
result_indirect_costs_ratespecify <- krippendorffs.alpha( indirect_costs_ratespecify_matrix_num_clean, level = “ratio”, method = “customary”, confint = TRUE, control = list(parallel = TRUE, nodes = 2) )
summary(result_indirect_costs_ratespecify)
indirect_costs_demin_matrix <- ds %>%
select(researcher, grant_id, indirect_costs_demin) %>%
pivot_wider(names_from = researcher, values_from = indirect_costs_demin) %>%
select(-grant_id) %>%
as.matrix()
indirect_costs_demin_vec <- as.vector(indirect_costs_demin_matrix)
indirect_costs_demin_factor <- factor(
indirect_costs_demin_vec,
levels = c("Yes (please specify the rate, e.g., 10%)", "No"),
ordered = FALSE
)
indirect_costs_demin_matrix_num <- matrix(as.numeric(indirect_costs_demin_factor), ncol = 2)
colnames(indirect_costs_demin_matrix_num) <- colnames(indirect_costs_demin_matrix)
result_indirect_costs_demin <- krippendorffs.alpha(
indirect_costs_demin_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_indirect_costs_demin)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = indirect_costs_demin_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.6465 -0.123 1indirect_costs_demin_specify_matrix <- ds %>%
select(researcher, grant_id, indirect_costs_demin_specify) %>%
pivot_wider(names_from = researcher, values_from = indirect_costs_demin_specify) %>%
select(-grant_id) %>%
as.matrix()
# Ensure numeric format (already done in your dataset)
indirect_costs_demin_specify_matrix_num <- apply(indirect_costs_demin_specify_matrix, 2, as.numeric)
result_indirect_costs_demin_specify <- krippendorffs.alpha(
indirect_costs_demin_specify_matrix_num,
level = "interval",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_indirect_costs_demin_specify)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = indirect_costs_demin_specify_matrix_num,
## level = "interval", method = "customary", confint = TRUE,
## control = list(parallel = TRUE, nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.03125 -2.3 1payment_method_matrix <- ds %>%
select(researcher, grant_id, payment_method) %>%
pivot_wider(names_from = researcher, values_from = payment_method) %>%
select(-grant_id) %>%
as.matrix()
payment_method_vec <- as.vector(payment_method_matrix)
payment_method_factor <- factor(
payment_method_vec,
levels = c("Other (please specify)",
"Not specified",
"Paid as a reimbursement",
"Paid in advance"),
ordered = FALSE
)
payment_method_matrix_num <- matrix(as.numeric(payment_method_factor), ncol = 2)
colnames(payment_method_matrix_num) <- colnames(payment_method_matrix)
result_payment_method <- krippendorffs.alpha(
payment_method_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_payment_method)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = payment_method_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.1143 -0.755 0.5224match_required_matrix <- ds %>%
select(researcher, grant_id, match_required) %>%
pivot_wider(names_from = researcher, values_from = match_required) %>%
select(-grant_id) %>%
as.matrix()
match_required_vec <- as.vector(match_required_matrix)
match_required_factor <- factor(
match_required_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
match_required_matrix_num <- matrix(as.numeric(match_required_factor), ncol = 2)
colnames(match_required_matrix_num) <- colnames(match_required_matrix)
result_match_required <- krippendorffs.alpha(
match_required_matrix_num,
level = "nominal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_match_required)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = match_required_matrix_num, level = "nominal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NA NAtable(match_required_matrix_num[, 1])##
## 1 2
## 3 17table(match_required_matrix_num[, 2])##
## 1 2
## 3 17all(match_required_matrix_num[, 1] == match_required_matrix_num[, 2], na.rm = TRUE)## [1] TRUEMatch requirement has 100% agreement, so no need to run the Krippendorff’s Alpha.
multi_year_funding_matrix <- ds %>%
select(researcher, grant_id, multi_year_funding) %>%
pivot_wider(names_from = researcher, values_from = multi_year_funding) %>%
select(-grant_id) %>%
as.matrix()
multi_year_funding_vec <- as.vector(multi_year_funding_matrix)
multi_year_funding_factor <- factor(
multi_year_funding_vec,
levels = c(
"Not specified",
"Multi-year funding is not available, and recipients cannot re-apply for grant funds again in the future",
"Multi-year funding is not available, but recipients can re-apply for grant funds again in the future",
"Multi-year funding is available but not required",
"Multi-year funding is the only available option for recipients"
),
ordered = FALSE
)
multi_year_funding_matrix_num <- matrix(as.numeric(multi_year_funding_factor), ncol = 2)
colnames(multi_year_funding_matrix_num) <- colnames(multi_year_funding_matrix)
result_multi_year_funding <- krippendorffs.alpha(
multi_year_funding_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_multi_year_funding)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = multi_year_funding_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.8035 0.5088 1loi_matrix <- ds %>%
select(researcher, grant_id, loi) %>%
pivot_wider(names_from = researcher, values_from = loi) %>%
select(-grant_id) %>%
as.matrix()
loi_vec <- as.vector(loi_matrix)
loi_factor <- factor(
loi_vec,
levels = c(
"No/Not Mentioned",
"Encouraged but not required",
"Yes"
),
ordered = TRUE
)
loi_matrix_num <- matrix(as.numeric(loi_factor), ncol = 2)
colnames(loi_matrix_num) <- colnames(loi_matrix)
result_loi <- krippendorffs.alpha(
loi_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_loi)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = loi_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.8361 0.5903 1need_matrix <- ds %>%
select(researcher, grant_id, need) %>%
pivot_wider(names_from = researcher, values_from = need) %>%
select(-grant_id) %>%
as.matrix()
need_vec <- as.vector(need_matrix)
need_factor <- factor(
need_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
need_matrix_num <- matrix(as.numeric(need_factor), ncol = 2)
colnames(need_matrix_num) <- colnames(need_matrix)
result_need <- krippendorffs.alpha(
need_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_need)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = need_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.02094 -1.654 1program_matrix <- ds %>%
select(researcher, grant_id, program) %>%
pivot_wider(names_from = researcher, values_from = program) %>%
select(-grant_id) %>%
as.matrix()
program_vec <- as.vector(program_matrix)
program_factor <- factor(
program_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
program_matrix_num <- matrix(as.numeric(program_factor), ncol = 2)
colnames(program_matrix_num) <- colnames(program_matrix)
result_program <- krippendorffs.alpha(
program_matrix_num,
level = "ordinal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_program)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = program_matrix_num, level = "ordinal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NaN NAtable(program_matrix_num[, 1])##
## 3
## 20table(program_matrix_num[, 2])##
## 3
## 20all(program_matrix_num[, 1] == program_matrix_num[, 2]) # Check for perfect agreement## [1] TRUEProgram Narrative has 100% agreement, so no need to run the Krippendorff’s Alpha.
goals_matrix <- ds %>%
select(researcher, grant_id, goals) %>%
pivot_wider(names_from = researcher, values_from = goals) %>%
select(-grant_id) %>%
as.matrix()
# Convert to ordered factor using known levels
goals_vec <- as.vector(goals_matrix)
goals_factor <- factor(
goals_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
goals_matrix_num <- matrix(as.numeric(goals_factor), ncol = 2)
colnames(goals_matrix_num) <- colnames(goals_matrix)
result_goals <- krippendorffs.alpha(
goals_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_goals)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = goals_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0 -2 1logic_model_matrix <- ds %>%
select(researcher, grant_id, logic_model) %>%
pivot_wider(names_from = researcher, values_from = logic_model) %>%
select(-grant_id) %>%
as.matrix()
logic_model_vec <- as.vector(logic_model_matrix)
logic_model_factor <- factor(
logic_model_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
logic_model_matrix_num <- matrix(as.numeric(logic_model_factor), ncol = 2)
colnames(logic_model_matrix_num) <- colnames(logic_model_matrix)
result_logic_model <- krippendorffs.alpha(
logic_model_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_logic_model)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = logic_model_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.371 -0.3839 0.9371org_statement_matrix <- ds %>%
select(researcher, grant_id, org_statement) %>%
pivot_wider(names_from = researcher, values_from = org_statement) %>%
select(-grant_id) %>%
as.matrix()
org_statement_vec <- as.vector(org_statement_matrix)
org_statement_factor <- factor(
org_statement_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
org_statement_matrix_num <- matrix(as.numeric(org_statement_factor), ncol = 2)
colnames(org_statement_matrix_num) <- colnames(org_statement_matrix)
result_org_statement <- krippendorffs.alpha(
org_statement_matrix_num,
level = "ordinal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_org_statement)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = org_statement_matrix_num, level = "ordinal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NaN NAtable(org_statement_matrix_num[, 1])##
## 3
## 20table(org_statement_matrix_num[, 2])##
## 3
## 20all(org_statement_matrix_num[, 1] == org_statement_matrix_num[, 2]) # Check for perfect agreement## [1] TRUEOrganization Statement has 100% agreement, so no need to run the Krippendorff’s Alpha.
budget_matrix <- ds %>%
select(researcher, grant_id, budget) %>%
pivot_wider(names_from = researcher, values_from = budget) %>%
select(-grant_id) %>%
as.matrix()
budget_vec <- as.vector(budget_matrix)
budget_factor <- factor(
budget_vec,
levels = c(
"No/Not Mentioned",
"Encouraged submission of a budget form provided by the granting organization that includes the information found in a program budget",
"Encouraged submission of the applying organization's own documents",
"Required submission of a budget form provided by the granting organization that includes the information found in a program budget",
"Required submission of the applying organization's own documents"
),
ordered = TRUE
)
budget_matrix_num <- matrix(as.numeric(budget_factor), ncol = 2)
colnames(budget_matrix_num) <- colnames(budget_matrix)
result_budget <- krippendorffs.alpha(
budget_matrix_num,
level = "ordinal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_budget)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = budget_matrix_num, level = "ordinal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NaN NAtable(budget_matrix_num[, 1])##
## 4
## 20table(budget_matrix_num[, 2])##
## 4
## 20all(budget_matrix_num[, 1] == budget_matrix_num[, 2], na.rm = TRUE) # Check for perfect agreement## [1] TRUEBudget has 100% agreement, so no need to run the Krippendorff’s Alpha.
budget_narrative_matrix <- ds %>%
select(researcher, grant_id, budget_narrative) %>%
pivot_wider(names_from = researcher, values_from = budget_narrative) %>%
select(-grant_id) %>%
as.matrix()
budget_narrative_vec <- as.vector(budget_narrative_matrix)
budget_narrative_factor <- factor(
budget_narrative_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
budget_narrative_matrix_num <- matrix(as.numeric(budget_narrative_factor), ncol = 2)
colnames(budget_narrative_matrix_num) <- colnames(budget_narrative_matrix)
result_budget_narrative <- krippendorffs.alpha(
budget_narrative_matrix_num,
level = "ordinal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_budget_narrative)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = budget_narrative_matrix_num, level = "ordinal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NaN NAtable(budget_narrative_matrix_num[, 1])##
## 3
## 20table(budget_narrative_matrix_num[, 2])##
## 3
## 20all(budget_narrative_matrix_num[, 1] == budget_narrative_matrix_num[, 2], na.rm = TRUE) # Check for perfect agreement## [1] TRUEBudget narrative has 100% agreement, so no need to run the Krippendorff’s Alpha.
financial_statements_matrix <- ds %>%
select(researcher, grant_id, financial_statements) %>%
pivot_wider(names_from = researcher, values_from = financial_statements) %>%
select(-grant_id) %>%
as.matrix()
financial_statements_vec <- as.vector(financial_statements_matrix)
financial_statements_factor <- factor(
financial_statements_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
financial_statements_matrix_num <- matrix(as.numeric(financial_statements_factor), ncol = 2)
colnames(financial_statements_matrix_num) <- colnames(financial_statements_matrix)
result_financial_statements <- krippendorffs.alpha(
financial_statements_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_financial_statements)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = financial_statements_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.04776 -1.445 1audit_matrix <- ds %>%
select(researcher, grant_id, audit) %>%
pivot_wider(names_from = researcher, values_from = audit) %>%
select(-grant_id) %>%
as.matrix()
audit_vec <- as.vector(audit_matrix)
audit_factor <- factor(
audit_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
audit_matrix_num <- matrix(as.numeric(audit_factor), ncol = 2)
colnames(audit_matrix_num) <- colnames(audit_matrix)
result_audit <- krippendorffs.alpha(
audit_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_audit)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = audit_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.939 0.8169 1evaluation_matrix <- ds %>%
select(researcher, grant_id, evaluation) %>%
pivot_wider(names_from = researcher, values_from = evaluation) %>%
select(-grant_id) %>%
as.matrix()
evaluation_vec <- as.vector(evaluation_matrix)
evaluation_factor <- factor(
evaluation_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
evaluation_matrix_num <- matrix(as.numeric(evaluation_factor), ncol = 2)
colnames(evaluation_matrix_num) <- colnames(evaluation_matrix)
result_evaluation <- krippendorffs.alpha(
evaluation_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_evaluation)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = evaluation_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.08333 -1.167 0.7292sustainability_matrix <- ds %>%
select(researcher, grant_id, sustainability) %>%
pivot_wider(names_from = researcher, values_from = sustainability) %>%
select(-grant_id) %>%
as.matrix()
sustainability_vec <- as.vector(sustainability_matrix)
sustainability_factor <- factor(
sustainability_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
sustainability_matrix_num <- matrix(as.numeric(sustainability_factor), ncol = 2)
colnames(sustainability_matrix_num) <- colnames(sustainability_matrix)
result_sustainability <- krippendorffs.alpha(
sustainability_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_sustainability)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = sustainability_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.2564 -0.3633 0.7934dei_matrix <- ds %>%
select(researcher, grant_id, dei) %>%
pivot_wider(names_from = researcher, values_from = dei) %>%
select(-grant_id) %>%
as.matrix()
dei_vec <- as.vector(dei_matrix)
dei_factor <- factor(
dei_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
dei_matrix_num <- matrix(as.numeric(dei_factor), ncol = 2)
colnames(dei_matrix_num) <- colnames(dei_matrix)
result_dei <- krippendorffs.alpha(
dei_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_dei)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = dei_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.1967 -0.657 0.2643partnerships_matrix <- ds %>%
select(researcher, grant_id, partnerships) %>%
pivot_wider(names_from = researcher, values_from = partnerships) %>%
select(-grant_id) %>%
as.matrix()
partnerships_vec <- as.vector(partnerships_matrix)
partnerships_factor <- factor(
partnerships_vec,
levels = c("Neither requires nor prioritizes partnerships or MOUs",
"Prioritizes partnerships or MOUs",
"Requires partnerships or MOUs"),
ordered = TRUE
)
partnerships_matrix_num <- matrix(as.numeric(partnerships_factor), ncol = 2)
colnames(partnerships_matrix_num) <- colnames(partnerships_matrix)
result_partnerships <- krippendorffs.alpha(
partnerships_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_partnerships)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = partnerships_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.01511 -0.5399 0.4761mou_matrix <- ds %>%
select(researcher, grant_id, mou) %>%
pivot_wider(names_from = researcher, values_from = mou) %>%
select(-grant_id) %>%
as.matrix()
mou_vec <- as.vector(mou_matrix)
mou_factor <- factor(
mou_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
mou_matrix_num <- matrix(as.numeric(mou_factor), ncol = 2)
colnames(mou_matrix_num) <- colnames(mou_matrix)
result_mou <- krippendorffs.alpha(
mou_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_mou)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = mou_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.9174 0.7934 1letter_of_support_matrix <- ds %>%
select(researcher, grant_id, letter_of_support) %>%
pivot_wider(names_from = researcher, values_from = letter_of_support) %>%
select(-grant_id) %>%
as.matrix()
letter_of_support_vec <- as.vector(letter_of_support_matrix)
letter_of_support_factor <- factor(
letter_of_support_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
letter_of_support_matrix_num <- matrix(as.numeric(letter_of_support_factor), ncol = 2)
colnames(letter_of_support_matrix_num) <- colnames(letter_of_support_matrix)
result_letter_of_support <- krippendorffs.alpha(
letter_of_support_matrix_num,
level = "ordinal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_letter_of_support)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = letter_of_support_matrix_num, level = "ordinal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.3746 -0.02566 0.7254number_attachments_matrix <- ds %>%
select(researcher, grant_id, number_attachments) %>%
pivot_wider(names_from = researcher, values_from = number_attachments) %>%
select(-grant_id) %>%
as.matrix()
number_attachments_matrix_num <- apply(number_attachments_matrix, 2, as.numeric)
result_number_attachments <- krippendorffs.alpha(
number_attachments_matrix_num,
level = "interval",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_number_attachments)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = number_attachments_matrix_num, level = "interval",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.4025 -0.8602 0.0277other_attachments_matrix <- ds %>%
select(researcher, grant_id, other_attachments) %>%
pivot_wider(names_from = researcher, values_from = other_attachments) %>%
select(-grant_id) %>%
as.matrix()
other_attachments_vec <- as.vector(other_attachments_matrix)
other_attachments_factor <- factor(
other_attachments_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
other_attachments_matrix_num <- matrix(as.numeric(other_attachments_factor), ncol = 2)
colnames(other_attachments_matrix_num) <- colnames(other_attachments_matrix)
result_other_attachments <- krippendorffs.alpha(
other_attachments_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_other_attachments)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = other_attachments_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.352 -0.768 0.064outcome_reports_matrix <- ds %>%
select(researcher, grant_id, outcome_reports) %>%
pivot_wider(names_from = researcher, values_from = outcome_reports) %>%
select(-grant_id) %>%
as.matrix()
outcome_reports_vec <- as.vector(outcome_reports_matrix)
outcome_reports_factor <- factor(
outcome_reports_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
outcome_reports_matrix_num <- matrix(as.numeric(outcome_reports_factor), ncol = 2)
colnames(outcome_reports_matrix_num) <- colnames(outcome_reports_matrix)
result_outcome_reports <- krippendorffs.alpha(
outcome_reports_matrix_num,
level = "ordinal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_outcome_reports)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = outcome_reports_matrix_num, level = "ordinal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NaN NAtable(outcome_reports_matrix_num[, 1])##
## 3
## 20table(outcome_reports_matrix_num[, 2])##
## 3
## 20all(outcome_reports_matrix_num[, 1] == outcome_reports_matrix_num[, 2], na.rm = TRUE) # Check for perfect agreement## [1] TRUEOutcome reports has 100% agreement, so no need to run the Krippendorff’s Alpha.
outcome_reports_frequency_matrix <- ds %>%
select(researcher, grant_id, outcome_reports_frequency) %>%
pivot_wider(names_from = researcher, values_from = outcome_reports_frequency) %>%
select(-grant_id) %>%
as.matrix()
outcome_reports_frequency_vec <- as.vector(outcome_reports_frequency_matrix)
outcome_reports_frequency_factor <- factor(
outcome_reports_frequency_vec,
levels = c("Other (please specify)",
"Once, after the grant period has concluded",
"Annually",
"Semi-Annually",
"Quarterly",
"Monthly"),
ordered = FALSE
)
outcome_reports_frequency_matrix_num <- matrix(as.numeric(outcome_reports_frequency_factor), ncol = 2)
colnames(outcome_reports_frequency_matrix_num) <- colnames(outcome_reports_frequency_matrix)
result_outcome_reports_frequency <- krippendorffs.alpha(
outcome_reports_frequency_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_outcome_reports_frequency)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = outcome_reports_frequency_matrix_num,
## level = "nominal", method = "customary", confint = TRUE,
## control = list(parallel = TRUE, nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.8687 0.6061 1financial_reports_matrix <- ds %>%
select(researcher, grant_id, financial_reports) %>%
pivot_wider(names_from = researcher, values_from = financial_reports) %>%
select(-grant_id) %>%
as.matrix()
financial_reports_vec <- as.vector(financial_reports_matrix)
financial_reports_factor <- factor(
financial_reports_vec,
levels = c("No/Not Mentioned", "Encouraged", "Required"),
ordered = TRUE
)
financial_reports_matrix_num <- matrix(as.numeric(financial_reports_factor), ncol = 2)
colnames(financial_reports_matrix_num) <- colnames(financial_reports_matrix)
result_financial_reports <- krippendorffs.alpha(
financial_reports_matrix_num,
level = "ordinal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_financial_reports)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = financial_reports_matrix_num, level = "ordinal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NaN NAtable(financial_reports_matrix_num[, 1])##
## 3
## 20table(financial_reports_matrix_num[, 2])##
## 3
## 20all(financial_reports_matrix_num[, 1] == financial_reports_matrix_num[, 2], na.rm = TRUE) # Check for perfect agreement## [1] TRUEFinancial reports has 100% agreement, so no need to run the Krippendorff’s Alpha.
financial_reports_frequency_matrix <- ds %>%
select(researcher, grant_id, financial_reports_frequency) %>%
pivot_wider(names_from = researcher, values_from = financial_reports_frequency) %>%
select(-grant_id) %>%
as.matrix()
financial_reports_frequency_vec <- as.vector(financial_reports_frequency_matrix)
financial_reports_frequency_factor <- factor(
financial_reports_frequency_vec,
levels = c("Other (please specify)",
"Once, after the grant period has concluded",
"Annually",
"Semi-Annually",
"Quarterly",
"Monthly"),
ordered = FALSE
)
financial_reports_frequency_matrix_num <- matrix(as.numeric(financial_reports_frequency_factor), ncol = 2)
colnames(financial_reports_frequency_matrix_num) <- colnames(financial_reports_frequency_matrix)
result_financial_reports_frequency <- krippendorffs.alpha(
financial_reports_frequency_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_financial_reports_frequency)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = financial_reports_frequency_matrix_num,
## level = "nominal", method = "customary", confint = TRUE,
## control = list(parallel = TRUE, nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.7953 0.4882 1scoring_rubric_matrix <- ds %>%
select(researcher, grant_id, scoring_rubric) %>%
pivot_wider(names_from = researcher, values_from = scoring_rubric) %>%
select(-grant_id) %>%
as.matrix()
scoring_rubric_vec <- as.vector(scoring_rubric_matrix)
scoring_rubric_factor <- factor(
scoring_rubric_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
scoring_rubric_matrix_num <- matrix(as.numeric(scoring_rubric_factor), ncol = 2)
colnames(scoring_rubric_matrix_num) <- colnames(scoring_rubric_matrix)
result_scoring_rubric <- krippendorffs.alpha(
scoring_rubric_matrix_num,
level = "nominal",
method = "analytical",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_scoring_rubric)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = scoring_rubric_matrix_num, level = "nominal",
## method = "analytical", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha NaN NaN NaNtable(scoring_rubric_matrix_num[, 1])##
## 1
## 20table(scoring_rubric_matrix_num[, 2])##
## 1
## 20all(scoring_rubric_matrix_num[, 1] == scoring_rubric_matrix_num[, 2], na.rm = TRUE) # Check for perfect agreement## [1] TRUEScoring rubric has 100% agreement, so no need to run the Krippendorff’s Alpha.
Note: Probably need to come up with some additional questions about the scoring rubric depending on the open text responses we have had thus far, then we can remove the open-text box.
webinar_matrix <- ds %>%
select(researcher, grant_id, webinar) %>%
pivot_wider(names_from = researcher, values_from = webinar) %>%
select(-grant_id) %>%
as.matrix()
webinar_vec <- as.vector(webinar_matrix)
webinar_factor <- factor(
webinar_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
webinar_matrix_num <- matrix(as.numeric(webinar_factor), ncol = 2)
colnames(webinar_matrix_num) <- colnames(webinar_matrix)
result_webinar <- krippendorffs.alpha(
webinar_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_webinar)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = webinar_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.2353 -0.3382 0.8088qna_matrix <- ds %>%
select(researcher, grant_id, qna) %>%
pivot_wider(names_from = researcher, values_from = qna) %>%
select(-grant_id) %>%
as.matrix()
qna_vec <- as.vector(qna_matrix)
qna_factor <- factor(
qna_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
qna_matrix_num <- matrix(as.numeric(qna_factor), ncol = 2)
colnames(qna_matrix_num) <- colnames(qna_matrix)
result_qna <- krippendorffs.alpha(
qna_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_qna)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = qna_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.1558 -0.5195 0.6623office_hours_matrix <- ds %>%
select(researcher, grant_id, office_hours) %>%
pivot_wider(names_from = researcher, values_from = office_hours) %>%
select(-grant_id) %>%
as.matrix()
office_hours_vec <- as.vector(office_hours_matrix)
office_hours_factor <- factor(
office_hours_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
office_hours_matrix_num <- matrix(as.numeric(office_hours_factor), ncol = 2)
colnames(office_hours_matrix_num) <- colnames(office_hours_matrix)
result_office_hours <- krippendorffs.alpha(
office_hours_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_office_hours)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = office_hours_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0 -2 1NOTE: This only happened once, so I think we can get rid of this code.
recipient_listing_matrix <- ds %>%
select(researcher, grant_id, recipient_listing) %>%
pivot_wider(names_from = researcher, values_from = recipient_listing) %>%
select(-grant_id) %>%
as.matrix()
recipient_listing_vec <- as.vector(recipient_listing_matrix)
recipient_listing_factor <- factor(
recipient_listing_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
recipient_listing_matrix_num <- matrix(as.numeric(recipient_listing_factor), ncol = 2)
colnames(recipient_listing_matrix_num) <- colnames(recipient_listing_matrix)
result_recipient_listing <- krippendorffs.alpha(
recipient_listing_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_recipient_listing)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = recipient_listing_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 1 1 1table(recipient_listing_matrix_num[, 1])##
## 1 2
## 2 18table(recipient_listing_matrix_num[, 2])##
## 1 2
## 2 18all(recipient_listing_matrix_num[, 1] == recipient_listing_matrix_num[, 2], na.rm = TRUE) # Check for perfect agreement## [1] TRUERecipient listing has 100% agreement, so no need to run the Krippendorff’s Alpha.
application_tips_matrix <- ds %>%
select(researcher, grant_id, application_tips) %>%
pivot_wider(names_from = researcher, values_from = application_tips) %>%
select(-grant_id) %>%
as.matrix()
application_tips_vec <- as.vector(application_tips_matrix)
application_tips_factor <- factor(
application_tips_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
application_tips_matrix_num <- matrix(as.numeric(application_tips_factor), ncol = 2)
colnames(application_tips_matrix_num) <- colnames(application_tips_matrix)
result_application_tips <- krippendorffs.alpha(
application_tips_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_application_tips)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = application_tips_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.02151 -0.5376 0.5806email_matrix <- ds %>%
select(researcher, grant_id, email) %>%
pivot_wider(names_from = researcher, values_from = email) %>%
select(-grant_id) %>%
as.matrix()
email_vec <- as.vector(email_matrix)
email_factor <- factor(
email_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
email_matrix_num <- matrix(as.numeric(email_factor), ncol = 2)
colnames(email_matrix_num) <- colnames(email_matrix)
result_email <- krippendorffs.alpha(
email_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_email)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = email_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0 -2 1phone_matrix <- ds %>%
select(researcher, grant_id, phone) %>%
pivot_wider(names_from = researcher, values_from = phone) %>%
select(-grant_id) %>%
as.matrix()
phone_vec <- as.vector(phone_matrix)
phone_factor <- factor(
phone_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
phone_matrix_num <- matrix(as.numeric(phone_factor), ncol = 2)
colnames(phone_matrix_num) <- colnames(phone_matrix)
result_phone <- krippendorffs.alpha(
phone_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_phone)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = phone_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha 0.3314 -0.3371 1other_assistance_matrix <- ds %>%
select(researcher, grant_id, other_assistance) %>%
pivot_wider(names_from = researcher, values_from = other_assistance) %>%
select(-grant_id) %>%
as.matrix()
other_assistance_vec <- as.vector(other_assistance_matrix)
other_assistance_factor <- factor(
other_assistance_vec,
levels = c("Yes", "No"),
ordered = FALSE
)
other_assistance_matrix_num <- matrix(as.numeric(other_assistance_factor), ncol = 2)
colnames(other_assistance_matrix_num) <- colnames(other_assistance_matrix)
result_other_assistance <- krippendorffs.alpha(
other_assistance_matrix_num,
level = "nominal",
method = "customary",
confint = TRUE,
control = list(parallel = TRUE, nodes = 2)
)
summary(result_other_assistance)##
## Krippendorff's Alpha
##
## Data: 20 units x 2 coders
##
## Call:
##
## krippendorffs.alpha(data = other_assistance_matrix_num, level = "nominal",
## method = "customary", confint = TRUE, control = list(parallel = TRUE,
## nodes = 2))
##
## Control parameters:
##
## parallel TRUE
## nodes 2
## bootit 1000
## type SOCK
##
## Results:
##
## Estimate Lower Upper
## alpha -0.1818 -0.8571 0.4935# Initialize an empty list to collect results
alpha_results <- list()
# Helper function to extract summary info
extract_alpha_summary <- function(result_object, varname) {
# Attempt to compute CI only if boot.sample exists
if (!is.null(result_object$boot.sample)) {
ci_vals <- quantile(result_object$boot.sample, probs = c(0.025, 0.975), na.rm = TRUE)
} else {
ci_vals <- c(NA, NA)
}
list(
variable = varname,
alpha = if (!is.null(result_object$alpha.hat)) result_object$alpha.hat["alpha"] else NA,
low_ci = ci_vals[1],
high_ci = ci_vals[2],
level = if (!is.null(result_object$level)) result_object$level else NA,
method = if (!is.null(result_object$method)) result_object$method else NA
)
}
# Add each result object you computed
#alpha_results[["eligibility"]] <- extract_alpha_summary(result_eligibility, "eligibility") # 100% agreement
#alpha_results[["research_grant"]] <- extract_alpha_summary(result_research_grant, "research_grant")# 100% agreement
alpha_results[["max_award"]] <- extract_alpha_summary(result_max_award, "max_award")
alpha_results[["max_directonly"]] <- extract_alpha_summary(result_max_directonly, "max_directonly")
#alpha_results[["numb_awards_peryear"]] <- extract_alpha_summary(result_numb_awards_peryear, "numb_awards_peryear") # 100% agreement
alpha_results[["total_funding"]] <- extract_alpha_summary(result_total_funding, "total_funding")
#alpha_results[["indirect_costs"]] <- extract_alpha_summary(result_indirect_costs, "indirect_costs") # 100% agreement
#alpha_results[["indirect_costs_ratespecify"]] <- extract_alpha_summary(result_indirect_costs_ratespecify, "indirect_costs_ratespecify") # too few cases to run
alpha_results[["indirect_costs_demin"]] <- extract_alpha_summary(result_indirect_costs_demin, "indirect_costs_demin")
alpha_results[["indirect_costs_demin_specify"]] <- extract_alpha_summary(result_indirect_costs_demin_specify, "indirect_costs_demin_specify")
alpha_results[["payment_method"]] <- extract_alpha_summary(result_payment_method, "payment_method")
#alpha_results[["match_required"]] <- extract_alpha_summary(result_match_required, "match_required") # 100% agreement
alpha_results[["multi_year_funding"]] <- extract_alpha_summary(result_multi_year_funding, "multi_year_funding")
alpha_results[["loi"]] <- extract_alpha_summary(result_loi, "loi")
alpha_results[["need"]] <- extract_alpha_summary(result_need, "need")
#alpha_results[["program"]] <- extract_alpha_summary(result_program, "program") # 100% agreement
alpha_results[["goals"]] <- extract_alpha_summary(result_goals, "goals")
alpha_results[["logic_model"]] <- extract_alpha_summary(result_logic_model, "logic_model")
#alpha_results[["org_statement"]] <- extract_alpha_summary(result_org_statement, "org_statement") # 100% agreement
#alpha_results[["budget"]] <- extract_alpha_summary(result_budget, "budget") # 100% agreement
#alpha_results[["budget_narrative"]] <- extract_alpha_summary(result_budget_narrative, "budget_narrative") # 100% agreement
alpha_results[["financial_statements"]] <- extract_alpha_summary(result_financial_statements, "financial_statements")
alpha_results[["audit"]] <- extract_alpha_summary(result_audit, "audit")
alpha_results[["evaluation"]] <- extract_alpha_summary(result_evaluation, "evaluation")
alpha_results[["sustainability"]] <- extract_alpha_summary(result_sustainability, "sustainability")
alpha_results[["dei"]] <- extract_alpha_summary(result_dei, "dei")
alpha_results[["partnerships"]] <- extract_alpha_summary(result_partnerships, "partnerships")
alpha_results[["mou"]] <- extract_alpha_summary(result_mou, "mou")
alpha_results[["letter_of_support"]] <- extract_alpha_summary(result_letter_of_support, "letter_of_support")
alpha_results[["number_attachments"]] <- extract_alpha_summary(result_number_attachments, "number_attachments")
alpha_results[["other_attachments"]] <- extract_alpha_summary(result_other_attachments, "other_attachments")
#alpha_results[["outcome_reports"]] <- extract_alpha_summary(result_outcome_reports, "outcome_reports") # 100% agreement
alpha_results[["outcome_reports_frequency"]] <- extract_alpha_summary(result_outcome_reports_frequency, "outcome_reports_frequency")
#alpha_results[["financial_reports"]] <- extract_alpha_summary(result_financial_reports, "financial_reports") # 100% agreement
alpha_results[["financial_reports_frequency"]] <- extract_alpha_summary(result_financial_reports_frequency, "financial_reports_frequency")
#alpha_results[["scoring_rubric"]] <- extract_alpha_summary(result_scoring_rubric, "scoring_rubric") # 100% agreement
alpha_results[["webinar"]] <- extract_alpha_summary(result_webinar, "webinar")
alpha_results[["qna"]] <- extract_alpha_summary(result_qna, "qna")
alpha_results[["office_hours"]] <- extract_alpha_summary(result_office_hours, "office_hours")
#alpha_results[["recipient_listing"]] <- extract_alpha_summary(result_recipient_listing, "recipient_listing") # 100% agreement
alpha_results[["application_tips"]] <- extract_alpha_summary(result_application_tips, "application_tips")
alpha_results[["email"]] <- extract_alpha_summary(result_email, "email")
alpha_results[["phone"]] <- extract_alpha_summary(result_phone, "phone")
alpha_results[["other_assistance"]] <- extract_alpha_summary(result_other_assistance, "other_assistance")
# Convert to a dataframe
krippendorff_summary_df <- do.call(rbind, lapply(alpha_results, as.data.frame))
# Reset rownames to get a clean dataframe
rownames(krippendorff_summary_df) <- NULL
krippendorff_summary_df$reliability <- cut(
krippendorff_summary_df$alpha,
breaks = c(-Inf, 0, 0.667, 0.80, 1),
labels = c("Worse than chance", "Low", "Moderate", "High"),
right = FALSE
)
krippendorff_summary_df$reliability <- factor(krippendorff_summary_df$reliability,
levels = c("Worse than chance", "Low", "Moderate", "High"),
ordered = TRUE)
krippendorff_summary_df <- krippendorff_summary_df[, c("variable", "reliability", "alpha", "low_ci", "high_ci", "level", "method")]
# View or export
arrange(krippendorff_summary_df, desc(reliability), desc(alpha))## variable reliability alpha low_ci high_ci level method
## 1 total_funding High 0.99994353 0.99982062 1.00000000 interval customary
## 2 max_award High 0.99987096 0.99967739 1.00000000 interval customary
## 3 audit High 0.93896714 0.81690141 1.00000000 ordinal customary
## 4 mou High 0.91737288 0.79343220 1.00000000 ordinal customary
## 5 outcome_reports_frequency High 0.86868687 0.60606061 1.00000000 nominal customary
## 6 loi High 0.83613445 0.59033613 1.00000000 ordinal customary
## 7 multi_year_funding High 0.80352645 0.50881612 1.00000000 nominal customary
## 8 financial_reports_frequency Moderate 0.79527559 0.48818898 1.00000000 nominal customary
## 9 max_directonly Moderate 0.77629063 0.55258126 1.00000000 nominal customary
## 10 indirect_costs_demin Low 0.64646465 -0.12299465 1.00000000 nominal customary
## 11 letter_of_support Low 0.37459910 -0.02565747 0.72544901 ordinal customary
## 12 logic_model Low 0.37096774 -0.38387097 0.93709677 ordinal customary
## 13 phone Low 0.33142857 -0.33714286 1.00000000 nominal customary
## 14 sustainability Low 0.25635593 -0.36334746 0.79343220 ordinal customary
## 15 webinar Low 0.23529412 -0.33823529 0.80882353 nominal customary
## 16 qna Low 0.15584416 -0.51948052 0.66233766 nominal customary
## 17 application_tips Low 0.02150538 -0.53763441 0.58064516 nominal customary
## 18 goals Low 0.00000000 -2.00000000 1.00000000 ordinal customary
## 19 office_hours Low 0.00000000 -2.00000000 1.00000000 nominal customary
## 20 email Low 0.00000000 -2.00000000 1.00000000 nominal customary
## 21 partnerships Worse than chance -0.01511335 -0.53986146 0.47607053 ordinal customary
## 22 need Worse than chance -0.02094241 -1.65445026 1.00000000 ordinal customary
## 23 indirect_costs_demin_specify Worse than chance -0.03125000 -2.30000000 1.00000000 interval customary
## 24 financial_statements Worse than chance -0.04776119 -1.44477612 1.00000000 ordinal customary
## 25 evaluation Worse than chance -0.08333333 -1.16666667 0.72916667 ordinal customary
## 26 payment_method Worse than chance -0.11428571 -0.75500000 0.52244898 nominal customary
## 27 other_assistance Worse than chance -0.18181818 -0.85714286 0.49350649 nominal customary
## 28 dei Worse than chance -0.19669552 -0.65696302 0.26433910 ordinal customary
## 29 other_attachments Worse than chance -0.35200000 -0.76800000 0.06400000 nominal customary
## 30 number_attachments Worse than chance -0.40251549 -0.86018507 0.02769715 interval customary# write.csv(krippendorff_summary_df, "krippendorff_alpha_summary.csv", row.names = FALSE)
arrange(percent_agreement_df, desc(agreement_cat), desc(PercentAgreement))## Variable PercentAgreement agreement_cat
## eligibility eligibility 100 Very Strong
## research_grant research_grant 100 Very Strong
## indirect_costs indirect_costs 100 Very Strong
## indirect_costs_ratespecify indirect_costs_ratespecify 100 Very Strong
## match_required match_required 100 Very Strong
## program program 100 Very Strong
## org_statement org_statement 100 Very Strong
## budget budget 100 Very Strong
## budget_narrative budget_narrative 100 Very Strong
## outcome_reports outcome_reports 100 Very Strong
## financial_reports financial_reports 100 Very Strong
## scoring_rubric scoring_rubric 100 Very Strong
## recipient_listing recipient_listing 100 Very Strong
## total_funding total_funding 95 Very Strong
## indirect_costs_demin indirect_costs_demin 95 Very Strong
## goals goals 95 Very Strong
## audit audit 95 Very Strong
## outcome_reports_frequency outcome_reports_frequency 95 Very Strong
## office_hours office_hours 95 Very Strong
## email email 95 Very Strong
## max_award max_award 90 Very Strong
## min_award min_award 90 Very Strong
## numb_awards_peryear numb_awards_peryear 90 Very Strong
## indirect_costs_demin_specify indirect_costs_demin_specify 90 Very Strong
## multi_year_funding multi_year_funding 90 Very Strong
## loi loi 90 Very Strong
## need need 90 Very Strong
## mou mou 90 Very Strong
## financial_reports_frequency financial_reports_frequency 90 Very Strong
## max_directonly max_directonly 85 Strong
## financial_statements financial_statements 85 Strong
## phone phone 85 Strong
## logic_model logic_model 80 Strong
## evaluation evaluation 80 Strong
## webinar webinar 80 Strong
## qna qna 75 Moderate
## sustainability sustainability 70 Moderate
## payment_method payment_method 65 Low
## letter_of_support letter_of_support 65 Low
## application_tips application_tips 65 Low
## other_assistance other_assistance 65 Low
## partnerships partnerships 50 Low
## other_attachments other_attachments 35 Low
## dei dei 25 Low
## number_attachments number_attachments 0 LowThe following variables are not highly reliable:
Moderate:
financial_reports_frequency
max_directonly
Low:
indirect_costs_demin
letter_of_support
logic_model
phone
sustainability
webinar
qna
application_tips
goals
office_hours
email
Worse than chance:
partnerships
need
indirect_costs_demin_specify
financial_statements
evaluation
payment_method
other_assistance
dei
other_attachments
number_attachments
Let’s look at each one.
matrix_list_raw[["financial_reports_frequency"]]## Corbin Walls Grant Mobley
## [1,] "Annually" "Annually"
## [2,] "Semi-Annually" "Semi-Annually"
## [3,] "Annually" "Annually"
## [4,] "Annually" "Annually"
## [5,] "Annually" "Annually"
## [6,] "Quarterly" "Quarterly"
## [7,] "Annually" "Annually"
## [8,] "Other (please specify)" "Other (please specify)"
## [9,] "Annually" "Other (please specify)"
## [10,] "Annually" "Annually"
## [11,] "Annually" "Annually"
## [12,] "Semi-Annually" "Annually"
## [13,] "Quarterly" "Quarterly"
## [14,] "Annually" "Annually"
## [15,] "Annually" "Annually"
## [16,] "Annually" "Annually"
## [17,] "Annually" "Annually"
## [18,] "Annually" "Annually"
## [19,] "Quarterly" "Quarterly"
## [20,] "Annually" "Annually"matrix_list_raw[["max_directonly"]]## Corbin Walls
## [1,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [2,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [3,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [4,] "Neither the grant synopsis nor any linked documents clarify whether this maximum includes both direct and indirect costs."
## [5,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [6,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [7,] "Not Applicable as this grant doesn't list a maximum award amount."
## [8,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [9,] "Neither the grant synopsis nor any linked documents clarify whether this maximum includes both direct and indirect costs."
## [10,] "Not Applicable as this grant doesn't list a maximum award amount."
## [11,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [12,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [13,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [14,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [15,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [16,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [17,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [18,] "Not Applicable as this grant doesn't list a maximum award amount."
## [19,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [20,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## Grant Mobley
## [1,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [2,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [3,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [4,] "Not Applicable as this grant doesn't list a maximum award amount."
## [5,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [6,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [7,] "Not Applicable as this grant doesn't list a maximum award amount."
## [8,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [9,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [10,] "Not Applicable as this grant doesn't list a maximum award amount."
## [11,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [12,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [13,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [14,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [15,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [16,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [17,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [18,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."
## [19,] "No, this is the maximum award amount for total costs, including both indirect and direct costs."
## [20,] "Yes, this is the maximum award amount for direct costs, and additional funds for indirect costs will also be provided."matrix_list_raw[["indirect_costs_demin"]]## Corbin Walls Grant Mobley
## [1,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [2,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [3,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [4,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [5,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [6,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [7,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [8,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [9,] "No" "No"
## [10,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [11,] "No" "Yes (please specify the rate, e.g., 10%)"
## [12,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [13,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [14,] NA NA
## [15,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [16,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [17,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [18,] NA NA
## [19,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"
## [20,] "Yes (please specify the rate, e.g., 10%)" "Yes (please specify the rate, e.g., 10%)"matrix_list_raw[["letter_of_support"]]## Corbin Walls Grant Mobley
## [1,] "No/Not Mentioned" "No/Not Mentioned"
## [2,] "Required" "Required"
## [3,] "Required" "No/Not Mentioned"
## [4,] "Required" "No/Not Mentioned"
## [5,] "Required" "No/Not Mentioned"
## [6,] "Required" "No/Not Mentioned"
## [7,] "Required" "No/Not Mentioned"
## [8,] "Required" "Required"
## [9,] "Required" "Required"
## [10,] "Required" "Required"
## [11,] "Encouraged" "No/Not Mentioned"
## [12,] "Required" "Required"
## [13,] "No/Not Mentioned" "No/Not Mentioned"
## [14,] "Required" "Required"
## [15,] "Required" "No/Not Mentioned"
## [16,] "No/Not Mentioned" "No/Not Mentioned"
## [17,] "No/Not Mentioned" "No/Not Mentioned"
## [18,] "No/Not Mentioned" "No/Not Mentioned"
## [19,] "Required" "Required"
## [20,] "No/Not Mentioned" "No/Not Mentioned"matrix_list_raw[["logic_model"]]## Corbin Walls Grant Mobley
## [1,] "No/Not Mentioned" "No/Not Mentioned"
## [2,] "Required" "No/Not Mentioned"
## [3,] "No/Not Mentioned" "No/Not Mentioned"
## [4,] "No/Not Mentioned" "No/Not Mentioned"
## [5,] "No/Not Mentioned" "No/Not Mentioned"
## [6,] "No/Not Mentioned" "No/Not Mentioned"
## [7,] "No/Not Mentioned" "No/Not Mentioned"
## [8,] "Encouraged" "Required"
## [9,] "No/Not Mentioned" "No/Not Mentioned"
## [10,] "No/Not Mentioned" "No/Not Mentioned"
## [11,] "No/Not Mentioned" "No/Not Mentioned"
## [12,] "Encouraged" "Required"
## [13,] "No/Not Mentioned" "No/Not Mentioned"
## [14,] "No/Not Mentioned" "No/Not Mentioned"
## [15,] "No/Not Mentioned" "Required"
## [16,] "No/Not Mentioned" "No/Not Mentioned"
## [17,] "No/Not Mentioned" "No/Not Mentioned"
## [18,] "No/Not Mentioned" "No/Not Mentioned"
## [19,] "No/Not Mentioned" "No/Not Mentioned"
## [20,] "No/Not Mentioned" "No/Not Mentioned"matrix_list_raw[["phone"]]## Corbin Walls Grant Mobley
## [1,] "Yes" "Yes"
## [2,] "Yes" "Yes"
## [3,] "Yes" "Yes"
## [4,] "Yes" "Yes"
## [5,] "Yes" "Yes"
## [6,] "No" "No"
## [7,] "Yes" "Yes"
## [8,] "Yes" "No"
## [9,] "Yes" "No"
## [10,] "Yes" "Yes"
## [11,] "Yes" "Yes"
## [12,] "Yes" "Yes"
## [13,] "Yes" "Yes"
## [14,] "Yes" "Yes"
## [15,] "Yes" "Yes"
## [16,] "Yes" "Yes"
## [17,] "Yes" "Yes"
## [18,] "Yes" "Yes"
## [19,] "Yes" "No"
## [20,] "Yes" "Yes"matrix_list_raw[["sustainability"]]## Corbin Walls Grant Mobley
## [1,] "No/Not Mentioned" "No/Not Mentioned"
## [2,] "Encouraged" "No/Not Mentioned"
## [3,] "No/Not Mentioned" "No/Not Mentioned"
## [4,] "No/Not Mentioned" "No/Not Mentioned"
## [5,] "No/Not Mentioned" "No/Not Mentioned"
## [6,] "No/Not Mentioned" "No/Not Mentioned"
## [7,] "No/Not Mentioned" "No/Not Mentioned"
## [8,] "Required" "Required"
## [9,] "No/Not Mentioned" "No/Not Mentioned"
## [10,] "Required" "No/Not Mentioned"
## [11,] "No/Not Mentioned" "No/Not Mentioned"
## [12,] "No/Not Mentioned" "No/Not Mentioned"
## [13,] "No/Not Mentioned" "No/Not Mentioned"
## [14,] "No/Not Mentioned" "No/Not Mentioned"
## [15,] "Required" "No/Not Mentioned"
## [16,] "Encouraged" "Required"
## [17,] "No/Not Mentioned" "No/Not Mentioned"
## [18,] "Required" "No/Not Mentioned"
## [19,] "Required" "No/Not Mentioned"
## [20,] "No/Not Mentioned" "No/Not Mentioned"matrix_list_raw[["webinar"]]## Corbin Walls Grant Mobley
## [1,] "No" "No"
## [2,] "No" "No"
## [3,] "No" "No"
## [4,] "No" "No"
## [5,] "No" "No"
## [6,] "Yes" "No"
## [7,] "No" "No"
## [8,] "Yes" "No"
## [9,] "Yes" "Yes"
## [10,] "No" "No"
## [11,] "No" "No"
## [12,] "No" "No"
## [13,] "No" "No"
## [14,] "No" "No"
## [15,] "No" "Yes"
## [16,] "No" "No"
## [17,] "Yes" "No"
## [18,] "No" "No"
## [19,] "No" "No"
## [20,] "No" "No"matrix_list_raw[["qna"]]## Corbin Walls Grant Mobley
## [1,] "Yes" "No"
## [2,] "Yes" "No"
## [3,] "No" "No"
## [4,] "No" "No"
## [5,] "No" "No"
## [6,] "No" "No"
## [7,] "No" "No"
## [8,] "Yes" "Yes"
## [9,] "No" "No"
## [10,] "Yes" "No"
## [11,] "No" "No"
## [12,] "Yes" "No"
## [13,] "No" "No"
## [14,] "No" "No"
## [15,] "No" "No"
## [16,] "No" "No"
## [17,] "No" "No"
## [18,] "No" "No"
## [19,] "Yes" "No"
## [20,] "No" "No"matrix_list_raw[["application_tips"]]## Corbin Walls Grant Mobley
## [1,] "Yes" "Yes"
## [2,] "Yes" "Yes"
## [3,] "Yes" "Yes"
## [4,] "Yes" "Yes"
## [5,] "Yes" "Yes"
## [6,] "No" "Yes"
## [7,] "Yes" "Yes"
## [8,] "Yes" "No"
## [9,] "Yes" "No"
## [10,] "No" "Yes"
## [11,] "Yes" "Yes"
## [12,] "Yes" "Yes"
## [13,] "No" "No"
## [14,] "No" "Yes"
## [15,] "Yes" "Yes"
## [16,] "Yes" "Yes"
## [17,] "Yes" "No"
## [18,] "Yes" "Yes"
## [19,] "Yes" "No"
## [20,] "Yes" "Yes"matrix_list_raw[["goals"]]## Corbin Walls Grant Mobley
## [1,] "Required" "Required"
## [2,] "Required" "Required"
## [3,] "Encouraged" "Required"
## [4,] "Required" "Required"
## [5,] "Required" "Required"
## [6,] "Required" "Required"
## [7,] "Required" "Required"
## [8,] "Required" "Required"
## [9,] "Required" "Required"
## [10,] "Required" "Required"
## [11,] "Required" "Required"
## [12,] "Required" "Required"
## [13,] "Required" "Required"
## [14,] "Required" "Required"
## [15,] "Required" "Required"
## [16,] "Required" "Required"
## [17,] "Required" "Required"
## [18,] "Required" "Required"
## [19,] "Required" "Required"
## [20,] "Required" "Required"matrix_list_raw[["office_hours"]]## Corbin Walls Grant Mobley
## [1,] "Yes" "No"
## [2,] "No" "No"
## [3,] "No" "No"
## [4,] "No" "No"
## [5,] "No" "No"
## [6,] "No" "No"
## [7,] "No" "No"
## [8,] "No" "No"
## [9,] "No" "No"
## [10,] "No" "No"
## [11,] "No" "No"
## [12,] "No" "No"
## [13,] "No" "No"
## [14,] "No" "No"
## [15,] "No" "No"
## [16,] "No" "No"
## [17,] "No" "No"
## [18,] "No" "No"
## [19,] "No" "No"
## [20,] "No" "No"matrix_list_raw[["email"]]## Corbin Walls Grant Mobley
## [1,] "Yes" "Yes"
## [2,] "Yes" "Yes"
## [3,] "Yes" "Yes"
## [4,] "Yes" "Yes"
## [5,] "Yes" "Yes"
## [6,] "Yes" "Yes"
## [7,] "Yes" "Yes"
## [8,] "Yes" "Yes"
## [9,] "Yes" "No"
## [10,] "Yes" "Yes"
## [11,] "Yes" "Yes"
## [12,] "Yes" "Yes"
## [13,] "Yes" "Yes"
## [14,] "Yes" "Yes"
## [15,] "Yes" "Yes"
## [16,] "Yes" "Yes"
## [17,] "Yes" "Yes"
## [18,] "Yes" "Yes"
## [19,] "Yes" "Yes"
## [20,] "Yes" "Yes"matrix_list_raw[["partnerships"]]## Corbin Walls
## [1,] "Neither requires nor prioritizes partnerships or MOUs"
## [2,] "Requires partnerships or MOUs"
## [3,] "Neither requires nor prioritizes partnerships or MOUs"
## [4,] "Neither requires nor prioritizes partnerships or MOUs"
## [5,] "Neither requires nor prioritizes partnerships or MOUs"
## [6,] "Requires partnerships or MOUs"
## [7,] "Requires partnerships or MOUs"
## [8,] "Requires partnerships or MOUs"
## [9,] "Neither requires nor prioritizes partnerships or MOUs"
## [10,] "Requires partnerships or MOUs"
## [11,] "Neither requires nor prioritizes partnerships or MOUs"
## [12,] "Requires partnerships or MOUs"
## [13,] "Neither requires nor prioritizes partnerships or MOUs"
## [14,] "Neither requires nor prioritizes partnerships or MOUs"
## [15,] "Requires partnerships or MOUs"
## [16,] "Requires partnerships or MOUs"
## [17,] "Prioritizes partnerships or MOUs"
## [18,] "Prioritizes partnerships or MOUs"
## [19,] "Requires partnerships or MOUs"
## [20,] "Neither requires nor prioritizes partnerships or MOUs"
## Grant Mobley
## [1,] "Neither requires nor prioritizes partnerships or MOUs"
## [2,] "Neither requires nor prioritizes partnerships or MOUs"
## [3,] "Neither requires nor prioritizes partnerships or MOUs"
## [4,] "Neither requires nor prioritizes partnerships or MOUs"
## [5,] "Neither requires nor prioritizes partnerships or MOUs"
## [6,] "Prioritizes partnerships or MOUs"
## [7,] "Neither requires nor prioritizes partnerships or MOUs"
## [8,] "Neither requires nor prioritizes partnerships or MOUs"
## [9,] "Neither requires nor prioritizes partnerships or MOUs"
## [10,] "Neither requires nor prioritizes partnerships or MOUs"
## [11,] "Neither requires nor prioritizes partnerships or MOUs"
## [12,] "Neither requires nor prioritizes partnerships or MOUs"
## [13,] "Neither requires nor prioritizes partnerships or MOUs"
## [14,] "Neither requires nor prioritizes partnerships or MOUs"
## [15,] "Requires partnerships or MOUs"
## [16,] "Neither requires nor prioritizes partnerships or MOUs"
## [17,] "Neither requires nor prioritizes partnerships or MOUs"
## [18,] "Neither requires nor prioritizes partnerships or MOUs"
## [19,] "Neither requires nor prioritizes partnerships or MOUs"
## [20,] "Neither requires nor prioritizes partnerships or MOUs"matrix_list_raw[["need"]]## Corbin Walls Grant Mobley
## [1,] "Required" "Required"
## [2,] "Required" "Required"
## [3,] "Encouraged" "Required"
## [4,] "Required" "Required"
## [5,] "Required" "Required"
## [6,] "Required" "Required"
## [7,] "Required" "Required"
## [8,] "Required" "Required"
## [9,] "Required" "Required"
## [10,] "Required" "Required"
## [11,] "Required" "Required"
## [12,] "Required" "Required"
## [13,] "No/Not Mentioned" "Required"
## [14,] "Required" "Required"
## [15,] "Required" "Required"
## [16,] "Required" "Required"
## [17,] "Required" "Required"
## [18,] "Required" "Required"
## [19,] "Required" "Required"
## [20,] "Required" "Required"matrix_list_raw[["indirect_costs_demin_specify"]]## Corbin Walls Grant Mobley
## [1,] 0.1 0.1
## [2,] 0.1 0.1
## [3,] 0.1 0.1
## [4,] 0.1 0.1
## [5,] 0.1 0.1
## [6,] 0.1 0.1
## [7,] 0.1 0.1
## [8,] 0.1 0.1
## [9,] NA NA
## [10,] 0.4 0.1
## [11,] NA 0.1
## [12,] 0.1 0.1
## [13,] 0.1 0.1
## [14,] NA NA
## [15,] 0.1 0.1
## [16,] 0.1 0.1
## [17,] 0.1 0.1
## [18,] NA NA
## [19,] 0.1 0.1
## [20,] 0.1 0.1matrix_list_raw[["financial_statements"]]## Corbin Walls Grant Mobley
## [1,] "No/Not Mentioned" "No/Not Mentioned"
## [2,] "Required" "No/Not Mentioned"
## [3,] "No/Not Mentioned" "No/Not Mentioned"
## [4,] "No/Not Mentioned" "No/Not Mentioned"
## [5,] "No/Not Mentioned" "No/Not Mentioned"
## [6,] "No/Not Mentioned" "No/Not Mentioned"
## [7,] "No/Not Mentioned" "No/Not Mentioned"
## [8,] "No/Not Mentioned" "No/Not Mentioned"
## [9,] "No/Not Mentioned" "No/Not Mentioned"
## [10,] "No/Not Mentioned" "No/Not Mentioned"
## [11,] "No/Not Mentioned" "No/Not Mentioned"
## [12,] "Required" "No/Not Mentioned"
## [13,] "No/Not Mentioned" "No/Not Mentioned"
## [14,] "No/Not Mentioned" "No/Not Mentioned"
## [15,] "No/Not Mentioned" "No/Not Mentioned"
## [16,] "No/Not Mentioned" "No/Not Mentioned"
## [17,] "No/Not Mentioned" "No/Not Mentioned"
## [18,] "No/Not Mentioned" "No/Not Mentioned"
## [19,] "Encouraged" "No/Not Mentioned"
## [20,] "No/Not Mentioned" "No/Not Mentioned"matrix_list_raw[["evaluation"]]## Corbin Walls Grant Mobley
## [1,] "Required" "Required"
## [2,] "Required" "Required"
## [3,] "Required" "Required"
## [4,] "Required" "Required"
## [5,] "No/Not Mentioned" "Required"
## [6,] "Required" "Required"
## [7,] "No/Not Mentioned" "Required"
## [8,] "Required" "Required"
## [9,] "No/Not Mentioned" "Required"
## [10,] "Required" "Required"
## [11,] "No/Not Mentioned" "Required"
## [12,] "Required" "Required"
## [13,] "Required" "Required"
## [14,] "Required" "Required"
## [15,] "Required" "Required"
## [16,] "Required" "Required"
## [17,] "Required" "Required"
## [18,] "Required" "Required"
## [19,] "Required" "Required"
## [20,] "Required" "Required"matrix_list_raw[["payment_method"]]## Corbin Walls Grant Mobley
## [1,] "Paid in advance" "Paid in advance"
## [2,] "Paid in advance" "Paid in advance"
## [3,] "Paid in advance" "Paid in advance"
## [4,] "Paid in advance" "Paid in advance"
## [5,] "Paid in advance" "Paid in advance"
## [6,] "Not specified" "Paid in advance"
## [7,] "Paid in advance" "Paid in advance"
## [8,] "Other (please specify)" "Paid in advance"
## [9,] "Paid in advance" "Paid in advance"
## [10,] "Paid as a reimbursement" "Paid in advance"
## [11,] "Not specified" "Paid in advance"
## [12,] "Not specified" "Paid in advance"
## [13,] "Other (please specify)" "Paid in advance"
## [14,] "Paid in advance" "Paid in advance"
## [15,] "Not specified" "Paid in advance"
## [16,] "Paid in advance" "Paid in advance"
## [17,] "Paid in advance" "Paid in advance"
## [18,] "Paid in advance" "Paid in advance"
## [19,] "Paid in advance" "Paid in advance"
## [20,] "Paid in advance" "Paid in advance"matrix_list_raw[["other_assistance"]]## Corbin Walls Grant Mobley
## [1,] "Yes" "No"
## [2,] "Yes" "No"
## [3,] "No" "No"
## [4,] "No" "No"
## [5,] "No" "No"
## [6,] "Yes" "No"
## [7,] "No" "No"
## [8,] "Yes" "No"
## [9,] "No" "No"
## [10,] "No" "No"
## [11,] "No" "No"
## [12,] "Yes" "No"
## [13,] "No" "No"
## [14,] "No" "No"
## [15,] "Yes" "No"
## [16,] "No" "No"
## [17,] "No" "No"
## [18,] "No" "No"
## [19,] "Yes" "No"
## [20,] "No" "No"matrix_list_raw[["dei"]]## Corbin Walls Grant Mobley
## [1,] "No/Not Mentioned" "No/Not Mentioned"
## [2,] "Encouraged" "No/Not Mentioned"
## [3,] "Required" "Required"
## [4,] "No/Not Mentioned" "No/Not Mentioned"
## [5,] "Required" "Encouraged"
## [6,] "Encouraged" "No/Not Mentioned"
## [7,] "Required" "Required"
## [8,] "No/Not Mentioned" "No/Not Mentioned"
## [9,] "Required" "No/Not Mentioned"
## [10,] "Required" "No/Not Mentioned"
## [11,] "Encouraged" "No/Not Mentioned"
## [12,] "Required" "No/Not Mentioned"
## [13,] "Encouraged" "No/Not Mentioned"
## [14,] "Required" "No/Not Mentioned"
## [15,] "Required" "No/Not Mentioned"
## [16,] "Encouraged" "No/Not Mentioned"
## [17,] "Required" "No/Not Mentioned"
## [18,] "Required" "No/Not Mentioned"
## [19,] "Required" "No/Not Mentioned"
## [20,] "Encouraged" "No/Not Mentioned"matrix_list_raw[["other_attachments"]]## Corbin Walls Grant Mobley
## [1,] "Yes" "No"
## [2,] "Yes" "Yes"
## [3,] "Yes" "No"
## [4,] "Yes" "No"
## [5,] "Yes" "No"
## [6,] "No" "Yes"
## [7,] "Yes" "Yes"
## [8,] "Yes" "Yes"
## [9,] "Yes" "No"
## [10,] "Yes" "No"
## [11,] "Yes" "No"
## [12,] "Yes" "Yes"
## [13,] "No" "No"
## [14,] "Yes" "No"
## [15,] "Yes" "Yes"
## [16,] "Yes" "No"
## [17,] "Yes" "No"
## [18,] "Yes" "No"
## [19,] "Yes" "Yes"
## [20,] "Yes" "No"matrix_list_raw[["number_attachments"]]## Corbin Walls Grant Mobley
## [1,] 10 0
## [2,] 21 12
## [3,] 13 1
## [4,] 10 0
## [5,] 13 1
## [6,] 13 3
## [7,] 12 2
## [8,] 9 4
## [9,] 12 3
## [10,] 14 0
## [11,] 10 0
## [12,] 15 12
## [13,] 6 0
## [14,] 18 0
## [15,] 10 9
## [16,] 10 0
## [17,] 10 0
## [18,] 10 0
## [19,] 7 4
## [20,] 10 0# Standardize column names for joining
percent_agreement_df_clean <- percent_agreement_df %>%
rename(variable = Variable)
# Combine both dataframes using a full join
combined_reliability_df <- full_join(
percent_agreement_df_clean,
krippendorff_summary_df,
by = "variable"
)
# Arrange for better readability (optional)
combined_reliability_df <- combined_reliability_df %>%
arrange(desc(agreement_cat), desc(reliability), desc(PercentAgreement), desc(alpha))
combined_reliability_df <- combined_reliability_df %>%
mutate(reliability = case_when(
is.na(reliability) ~ "Perfect Agreement",
TRUE ~ reliability))
combined_reliability_df$reliability <- factor(combined_reliability_df$reliability,
levels = c("Worse than chance", "Low", "Moderate", "High", "Perfect Agreement"),
ordered = TRUE)
# View result
print(combined_reliability_df)## variable PercentAgreement agreement_cat reliability alpha low_ci high_ci
## 1 total_funding 95 Very Strong High 0.99994353 0.99982062 1.00000000
## 2 audit 95 Very Strong High 0.93896714 0.81690141 1.00000000
## 3 outcome_reports_frequency 95 Very Strong High 0.86868687 0.60606061 1.00000000
## 4 max_award 90 Very Strong High 0.99987096 0.99967739 1.00000000
## 5 mou 90 Very Strong High 0.91737288 0.79343220 1.00000000
## 6 loi 90 Very Strong High 0.83613445 0.59033613 1.00000000
## 7 multi_year_funding 90 Very Strong High 0.80352645 0.50881612 1.00000000
## 8 financial_reports_frequency 90 Very Strong Moderate 0.79527559 0.48818898 1.00000000
## 9 indirect_costs_demin 95 Very Strong Low 0.64646465 -0.12299465 1.00000000
## 10 goals 95 Very Strong Low 0.00000000 -2.00000000 1.00000000
## 11 office_hours 95 Very Strong Low 0.00000000 -2.00000000 1.00000000
## 12 email 95 Very Strong Low 0.00000000 -2.00000000 1.00000000
## 13 need 90 Very Strong Worse than chance -0.02094241 -1.65445026 1.00000000
## 14 indirect_costs_demin_specify 90 Very Strong Worse than chance -0.03125000 -2.30000000 1.00000000
## 15 eligibility 100 Very Strong Perfect Agreement NA NA NA
## 16 research_grant 100 Very Strong Perfect Agreement NA NA NA
## 17 indirect_costs 100 Very Strong Perfect Agreement NA NA NA
## 18 indirect_costs_ratespecify 100 Very Strong Perfect Agreement NA NA NA
## 19 match_required 100 Very Strong Perfect Agreement NA NA NA
## 20 program 100 Very Strong Perfect Agreement NA NA NA
## 21 org_statement 100 Very Strong Perfect Agreement NA NA NA
## 22 budget 100 Very Strong Perfect Agreement NA NA NA
## 23 budget_narrative 100 Very Strong Perfect Agreement NA NA NA
## 24 outcome_reports 100 Very Strong Perfect Agreement NA NA NA
## 25 financial_reports 100 Very Strong Perfect Agreement NA NA NA
## 26 scoring_rubric 100 Very Strong Perfect Agreement NA NA NA
## 27 recipient_listing 100 Very Strong Perfect Agreement NA NA NA
## 28 min_award 90 Very Strong Perfect Agreement NA NA NA
## 29 numb_awards_peryear 90 Very Strong Perfect Agreement NA NA NA
## 30 max_directonly 85 Strong Moderate 0.77629063 0.55258126 1.00000000
## 31 phone 85 Strong Low 0.33142857 -0.33714286 1.00000000
## 32 logic_model 80 Strong Low 0.37096774 -0.38387097 0.93709677
## 33 webinar 80 Strong Low 0.23529412 -0.33823529 0.80882353
## 34 financial_statements 85 Strong Worse than chance -0.04776119 -1.44477612 1.00000000
## 35 evaluation 80 Strong Worse than chance -0.08333333 -1.16666667 0.72916667
## 36 qna 75 Moderate Low 0.15584416 -0.51948052 0.66233766
## 37 sustainability 70 Moderate Low 0.25635593 -0.36334746 0.79343220
## 38 letter_of_support 65 Low Low 0.37459910 -0.02565747 0.72544901
## 39 application_tips 65 Low Low 0.02150538 -0.53763441 0.58064516
## 40 payment_method 65 Low Worse than chance -0.11428571 -0.75500000 0.52244898
## 41 other_assistance 65 Low Worse than chance -0.18181818 -0.85714286 0.49350649
## 42 partnerships 50 Low Worse than chance -0.01511335 -0.53986146 0.47607053
## 43 other_attachments 35 Low Worse than chance -0.35200000 -0.76800000 0.06400000
## 44 dei 25 Low Worse than chance -0.19669552 -0.65696302 0.26433910
## 45 number_attachments 0 Low Worse than chance -0.40251549 -0.86018507 0.02769715
## level method
## 1 interval customary
## 2 ordinal customary
## 3 nominal customary
## 4 interval customary
## 5 ordinal customary
## 6 ordinal customary
## 7 nominal customary
## 8 nominal customary
## 9 nominal customary
## 10 ordinal customary
## 11 nominal customary
## 12 nominal customary
## 13 ordinal customary
## 14 interval customary
## 15 <NA> <NA>
## 16 <NA> <NA>
## 17 <NA> <NA>
## 18 <NA> <NA>
## 19 <NA> <NA>
## 20 <NA> <NA>
## 21 <NA> <NA>
## 22 <NA> <NA>
## 23 <NA> <NA>
## 24 <NA> <NA>
## 25 <NA> <NA>
## 26 <NA> <NA>
## 27 <NA> <NA>
## 28 <NA> <NA>
## 29 <NA> <NA>
## 30 nominal customary
## 31 nominal customary
## 32 ordinal customary
## 33 nominal customary
## 34 ordinal customary
## 35 ordinal customary
## 36 nominal customary
## 37 ordinal customary
## 38 ordinal customary
## 39 nominal customary
## 40 nominal customary
## 41 nominal customary
## 42 ordinal customary
## 43 nominal customary
## 44 ordinal customary
## 45 interval customarywrite.csv(combined_reliability_df, "/Users/corbinwalls/Library/CloudStorage/OneDrive-american.edu/Projects/The Cost of Compliance/Study 1/Data/Intercoder Reliability/Initial ICR Results.csv")
write.csv(ds, "/Users/corbinwalls/Library/CloudStorage/OneDrive-american.edu/Projects/The Cost of Compliance/Study 1/Data/Intercoder Reliability/Cleaned Grants.Gov Coding data - Initial Intercoder Reliability - June 25, 2025.csv")We should discuss every variable that did not have a Strong percent agreement and either perfect agreement or high reliability.
# List of variables to check
vars_to_check <- c(
"financial_reports_frequency", "indirect_costs_demin", "goals", "office_hours",
"email", "need", "indirect_costs_demin_specify", "max_directonly", "phone",
"logic_model", "webinar", "financial_statements", "evaluation", "qna",
"sustainability", "letter_of_support", "application_tips", "payment_method",
"other_assistance", "partnerships", "other_attachments", "dei", "number_attachments"
)
# Ensure all variables to check are character to prevent pivoting errors
ds_clean <- ds %>%
mutate(across(all_of(vars_to_check), as.character))
# Pivot wider to compare coder values
ds_wide <- ds_clean %>%
select(researcher, grant_id, grant_title, all_of(vars_to_check)) %>%
pivot_longer(cols = all_of(vars_to_check), names_to = "variable", values_to = "value") %>%
pivot_wider(names_from = researcher, values_from = value) %>%
filter(!is.na(`Corbin Walls`), !is.na(`Grant Mobley`), `Corbin Walls` != `Grant Mobley`) %>%
select(variable, grant_id, grant_title, corbin_code = `Corbin Walls`, grant_code = `Grant Mobley`)
# Result: a dataframe of all disagreements
disagreement_df <- ds_wide
disagreement_df <- disagreement_df %>% arrange(variable, grant_id)
# Load the hyperlink file
links <- readxl::read_excel("/Users/corbinwalls/Library/CloudStorage/OneDrive-american.edu/Projects/The Cost of Compliance/Study 1/Data/Intercoder Reliability/Links to Synopses.xlsx")
# Merge with disagreement table
disagreement_df_with_links <- disagreement_df %>%
left_join(links %>% select(grant_id, grant_hyperlink), by = "grant_id")
write.csv(disagreement_df_with_links, "/Users/corbinwalls/Library/CloudStorage/OneDrive-american.edu/Projects/The Cost of Compliance/Study 1/Data/Intercoder Reliability/Coding Disagreements for Review - Initial ICR.csv")