0) install konfound from github repository
library(tidyverse)
library(htmltools)
library(evaluate)
library(githubinstall)
suppressMessages(gh_install_packages("konfound", ref = "newitcv_output", force = TRUE))
## Warning in fread(download_url, sep = "\t", header = FALSE, stringsAsFactors =
## FALSE, : Found and resolved improper quoting out-of-sample. First healed line
## 4848: <<Puriney honfleuR "Evening, honfleuR" by Seurat>>. If the fields are not
## quoted (e.g. field separator does not appear within any field), try quote="" to
## avoid this warning.
## Do you want to install the package (Y/n)?
##
## ── R CMD build ─────────────────────────────────────────────────────────────────
##
checking for file ‘/private/var/folders/52/my2sfsnn1qs78nfvl14zyynh0000gn/T/RtmpzCRpqQ/remotes34e64c590b89/konfound-project-konfound-1b4047d/DESCRIPTION’ ...
✔ checking for file ‘/private/var/folders/52/my2sfsnn1qs78nfvl14zyynh0000gn/T/RtmpzCRpqQ/remotes34e64c590b89/konfound-project-konfound-1b4047d/DESCRIPTION’
##
─ preparing ‘konfound’:
## checking DESCRIPTION meta-information ...
✔ checking DESCRIPTION meta-information
##
─ checking for LF line-endings in source and make files and shell scripts
##
─ checking for empty or unneeded directories
##
─ building ‘konfound_0.4.0.tar.gz’
##
##
library(konfound)
1) Output Language for RIR in Linear Model
1.1) Invalidate, eff_thr = NA
# Case 1: Linear
### invalidate
## eff_thr = NA
pkonfound(est_eff = -0.3, std_err = 0.01, n_obs = 5000, n_covariates = 10, alpha = .05, tails = 2, index = "RIR", nu = 0, model_type = "ols", eff_thr = NA, to_return = "print")
## Robustness of Inference to Replacement (RIR):
## TO INVALIDATE:
##
## RIR = 4673
##
## You entered an estimated effect of -0.3. To invalidate
## the inference of an effect using the threshold of -0.02 for
## statistical significance with alpha = 0.05, 93.465% of the
## (-0.3) estimate would have to be due to bias. This implies
## that to invalidate the inference one would expect to have to
## replace 4673 (93.465%) observations with cases for which the
## treatment effect is 0 (RIR = 4673).
##
## See Frank et al. (2013) for a description of the method.
##
## Citation: Frank, K.A., Maroulis, S., Duong, M., and Kelcey, B. (2013).
## What would it take to change an inference?
## Using Rubin's causal model to interpret the robustness of causal inferences.
## Education, Evaluation and Policy Analysis, 35 437-460.
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
* Example image of citation hyperlink (automatically poped-up
in viewer pane)
- It appears that R might not directly support adding hyperlinks in
the Console pane (where output language is displayed), so I suggest
alternative method.
1.2) Invalidate, eff_thr != NA (user-entered
threshold)
## eff_thr != NA (user entered)
pkonfound(est_eff = -0.3, std_err = 0.01, n_obs = 5000, n_covariates = 10, alpha = .05, tails = 2, index = "RIR", nu = 0, model_type = "ols", eff_thr = 0.2, to_return = "print")
## Robustness of Inference to Replacement (RIR):
## TO INVALIDATE:
##
## RIR = 4673
##
## You entered an effect of -0.3, and specified a threshold
## for inference of 0.2. To invalidate the inference based on your
## estimate, 93.465% of the (-0.3) estimate would have to be due to
## bias. This implies that to invalidate the inference one would
## expect to have to replace 4673 (93.465%) observations with
## cases for which the treatment effect is 0 (RIR = 4673).
##
## See Frank et al. (2013) for a description of the method.
##
## Citation: Frank, K.A., Maroulis, S., Duong, M., and Kelcey, B. (2013).
## What would it take to change an inference?
## Using Rubin's causal model to interpret the robustness of causal inferences.
## Education, Evaluation and Policy Analysis, 35 437-460.
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
1.3) Sustain, eff_thr = NA
### sustain
## eff_thr = NA
pkonfound(est_eff = 0.003, std_err = 0.1, n_obs = 5000, n_covariates = 10, alpha = .05, tails = 2, index = "RIR", nu = 0, model_type = "ols", eff_thr = NA, to_return = "print")
## Robustness of Inference to Replacement (RIR):
## TO SUSTAIN:
##
## RIR = 4923
##
## You entered an estimated effect of 0.003. The threshold value for
## statistical significance is 0.196 (alpha = 0.05). To reach that
## threshold, 98.47% of the (0.003) estimate would have to be due to
## bias. This implies that to sustain an inference one would expect to
## have to replace 4923 (98.47%) observations with effect of 0 with
## cases with effect of 0.196 (RIR = 4923).
##
## See Frank et al. (2013) for a description of the method.
##
## Citation: Frank, K.A., Maroulis, S., Duong, M., and Kelcey, B. (2013).
## What would it take to change an inference?
## Using Rubin's causal model to interpret the robustness of causal inferences.
## Education, Evaluation and Policy Analysis, 35 437-460.
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
1.4) Sustain, eff_thr != NA (user-entered
threshold)
## eff_thr != NA (user entered)
pkonfound(est_eff = 0.09, std_err = 0.1, n_obs = 5000, n_covariates = 10, alpha = .05, tails = 2, index = "RIR", nu = 0, model_type = "ols", eff_thr = 0.2, to_return = "print")
## Robustness of Inference to Replacement (RIR):
## TO SUSTAIN:
##
## RIR = 2705
##
## You entered an effect size of 0.09, and specified a threshold
## for inference of 0.2. To reach that threshold, 54.092% of the
## (0.09) estimate would have to be due to bias. This implies
## that to sustain an inference one would expect to have to replace
## 2705 (54.092%) observations with effect of 0 with cases
## with effect of 0.196 (RIR = 2705).
##
## See Frank et al. (2013) for a description of the method.
##
## Citation: Frank, K.A., Maroulis, S., Duong, M., and Kelcey, B. (2013).
## What would it take to change an inference?
## Using Rubin's causal model to interpret the robustness of causal inferences.
## Education, Evaluation and Policy Analysis, 35 437-460.
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
2) Output Language for RIR in Non-linear Model
2.1) Invalidate, changeSE = TRUE
- In this example, added p-values are <0.000 and 0.051 and RIR_perc
is 16.37%
# Case 2: Non-linear
## Case 2-1: changeSE = TRUE, Invalidate
pkonfound(-0.3, 0.01, 5000, n_covariates = 0, alpha = .05, tails = 2, nu = 0, n_treat = 2500, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 220
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 1157 1343
## Treatment 1344 1156
##
## The reported effect size = -0.300, SE = 0.010, p-value = 0.000
## The SE has been adjusted to 0.057 to generate real number in the
## implied table. Numbers in the table cells have been rounded
## to integers, which may slightly alter the estimated effect from
## the value originally entered.
##
## To invalidate the inference that the effect is greater than 0
## (alpha = 0.050) one would need to replace 220 (16.37%) treatment failure
## cases with cases for which the probability of failure in the
## control group applies (RIR = 220). This is equivalent to transferring
## 118 case from treatment failure to treatment success (Fragility = 118).
## This transfer of cases yields the following table:
##
##
## Table after transfer:
##
## Fail Success
## Control 1157 1343
## Treatment 1226 1274
## (Effect size = -0.111, SE = 0.057, p-value = 0.051)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
2.2) Invalidate, changeSE = FALSE
## Case 2-2: changeSE = FALSE, Invalidate
pkonfound(-0.3, 0.1, 5000, n_covariates = 10, alpha = .05, tails = 2, n_treat = 2500, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 207
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 2246 254
## Treatment 2307 193
##
## The reported effect size = -0.300, and SE = 0.100, p-value = 0.052
## Values have been rounded to the nearest integer. This may cause
## a little change to the estimated effect for the table.
##
## To invalidate the inference that the effect is greater than 0
## (alpha = 0.050) one would need to replace 207 (8.97%) treatment failure
## cases with cases for which the probability of failure in the
## control group applies (RIR = 207). This is equivalent to
## transferring 21 case from treatment failure to treatment success
## (Fragility = 21). This transfer of cases yields the following
## table:
##
## Table after transfer:
##
## Fail Success
## Control 2246 254
## Treatment 2286 214
## (Effect size = -0.189, SE = 0.097, p-value = 0.052)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
2.3) Sustain, changeSE = TRUE (where RIR_perc
exceeds 100%)
## Case 2-3: changeSE = TRUE, Sustain
pkonfound(-0.03, 0.2, 50, n_covariates = 5, alpha = .005, tails = 2, nu = 0, n_treat = 25, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 21
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 12 13
## Treatment 13 12
##
## The reported effect size = -0.030, SE = 0.200, p-value = 0.779
## The SE has been adjusted to 0.566 to generate real number in the
## implied table. Numbers in the table cells have been rounded
## to integers, which may slightly alter the estimated effect from
## the value originally entered.
##
## To reach the threshold that would sustain the inference that the
## effect is greater than 0 (alpha = 0.005) one would need to replace 21
## (175%) treatment success cases with cases for which the probability of
## failure in the control group applies (RIR = 21). This is equivalent
## to transferring 10 case from treatment success to treatment failure
## (Fragility = 10). This transfer of cases yields the following
## table:
##
## Note the RIR exceeds 100%. Generating the transfer of 10 cases would
## require replacing cases with a probability of success of 0.17 of
## which is smaller than the probability of success for the control
## group used to calculate the RIR.
##
## Table after transfer:
##
## Fail Success
## Control 12 13
## Treatment 23 2
## (Effect size = -2.522, SE = 0.839, p-value = 0.004)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
2.4) Sustain, changeSE = FALSE
## Case 2-4: changeSE = FALSE, Sustain
pkonfound(-0.03, 0.2, 500, n_covariates = 10, alpha = .05, tails = 2, nu = 0, n_treat = 250, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 24
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 180 70
## Treatment 182 68
##
## The reported effect size = -0.030, and SE = 0.200, p-value = 0.049
## Values have been rounded to the nearest integer. This may cause
## a little change to the estimated effect for the table.
##
## To reach the threshold that would sustain the inference that the
## effect is greater than 0 (alpha = 0.050) one would need to replace 24
## (35.29%) treatment success cases with cases for which the probability of
## failure in the control group applies (RIR = 24). This is equivalent
## to transferring 17 case from treatment success to treatment failure
## (Fragility = 17). This transfer of cases yields the following
## table:
##
## Table after transfer:
##
## Fail Success
## Control 180 70
## Treatment 199 51
## (Effect size = -0.417, SE = 0.211, p-value = 0.049)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
3) Output Language for RIR in Non-linear Model
(needtworows = T)
- Examining code line by line,
needtworows = TRUE is a
relatively rare case. Additionally, there are constraints that prevent
invalidate_ob = TRUE when
needtworows = TRUE.
- For instance, an extremely small
alpha value (e.g.,
0.0001) combined with a small number of observations
(n_obs, e.g., fewer than 50) is a major factor leading to
needtworows = TRUE.
- However, this scenario also results in a larger
thr_t,
which tends to make invalidate_ob = TRUE in almost all
cases.
- Therefore, even though language outputs are made for the all
possible
needtworows = TRUE scenario, finding a real case
that satisfies this condition might be challenging.
- Similarly, a small
n_obs is closely associated with
minse being greater than user_std_err.
Consequently, it is also difficult to find a real case where
changeSE = FALSE under the needtworows = TRUE
condition.
3.1) needtworows = T, sustain,
changeSE = TRUE
## Case 3: needtworows = T, Sustain, changeSE = T,
pkonfound(0.25, 0.5, 70, n_covariates = 20, alpha = .001, tails = 2, nu = 0, n_treat = 20, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 37
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 27 23
## Treatment 10 10
##
## The reported effect size = 0.250, SE = 0.500, p-value = 0.763
## The SE has been adjusted to 0.530 to generate real number in the
## implied table. Numbers in the table cells have been rounded
## to integers, which may slightly alter the estimated effect from
## the value originally entered.
##
## The inference cannot be sustained merely by switching cases in
## only one treatment condition. Therefore, cases have been switched from
## treatment failure to treatment success and from control success to control failure.
## The final Fragility(= 17) and RIR(= 37) reflect both sets of changes.
## Please compare the after transfer table with the implied table.
##
## Table after transfer:
##
## Fail Success
## Control 35 15
## Treatment 1 19
## (Effect size = 3.792, SE = 1.071, p-value = 0.001)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
3.2) Another case for needtworows = T,
sustain, changeSE = TRUE
## Case 3: needtworows = T, Sustain, changeSE = T,
pkonfound(0.8, 0.2, 50, n_covariates = 20, alpha = .001, tails = 2, nu = 0, n_treat = 20, switch_trm = FALSE, model_type = "logistic", to_return = "print")
## RIR = 19
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 19 11
## Treatment 8 12
##
## The reported effect size = 0.800, SE = 0.200, p-value = 0.120
## The SE has been adjusted to 0.593 to generate real number in the
## implied table. Numbers in the table cells have been rounded
## to integers, which may slightly alter the estimated effect from
## the value originally entered.
##
## The inference cannot be sustained merely by switching cases in
## only one treatment condition. Therefore, cases have been switched from
## control success to control failure and from treatment failure to treatment success.
## The final Fragility(= 12) and RIR(= 19) reflect both sets of changes.
## Please compare the after transfer table with the implied table.
##
## Table after transfer:
##
## Fail Success
## Control 29 1
## Treatment 6 14
## (Effect size = 4.215, SE = 1.128, p-value = 0.001)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
3.3) Part of code of language output for possible case,
needtworows = T & invalidate &
changeSE = TRUE
else {
### when needtworows = T
### changeSE = T
cat(paste0(
sprintf("The inference cannot be invalidated merely by switching cases in"),
sprintf("\nonly one treatment condition. Therefore, cases have been switched from"),
c("\n"), transferway, c(" and from "),
transferway_extra, c("."), c("\n"),
sprintf("The final Fragility(= %d) and RIR(= %d)", final_solution$final_switch, RIR),
c(" reflect both sets of changes. \nPlease compare the after transfer table with the implied table.")
)
)
}
4) Output Language for RIR with 2by2 table
4.1) 2by2 table with print option
(test="fisher")
# 2 by 2 table: printed output
pkonfound(a = 35, b = 17, c = 17, d = 38, alpha = 0.05, switch_trm = T, test = "fisher", replace = "control", "to_return" = "print")
## Background Information:
## This function calculates the number of cases that would have to be replaced
## with no effect cases (RIR) to invalidate an inference made about the association
## between the rows and columns in a 2x2 table.
## One can also interpret this as switches from one cell to another, such as from
## the treatment success cell to the treatment failure cell.
##
## Conclusion:
## To invalidate the inference, you would need to replace 14 treatment success
## cases for which the probability of failure in the control group applies (RIR = 14).
## This is equivalent to transferring 9 cases from treatment success to treatment failure.
## For the User-entered Table, we have an estimated odds ratio of 4.530, with p-value of 0.000:
##
## User-entered Table:
## Fail Success
## Control 35 17
## Treatment 17 38
##
##
## For the Transfer Table, we have an estimated odds ratio of 2.278, with p-value of 0.051:
## Transfer Table:
## Fail Success
## Control 35 17
## Treatment 26 29
##
## RIR:
## RIR = 14
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
4.2) 2by2 table with raw_output option
(test="fisher")
# 2 by 2 table: raw output
pkonfound(a = 35, b = 17, c = 17, d = 38, alpha = 0.05, switch_trm = T, test = "fisher", replace = "control", "to_return" = "raw_output")
## For interpretation, check out to_return = 'print'.
## $RIR_primary
## [1] 14
##
## $RIR_supplemental
## [1] 0
##
## $RIR_perc
## [1] 36.84211
##
## $fragility_primary
## [1] 9
##
## $starting_table
## Fail Success
## Control 35 17
## Treatment 17 38
##
## $final_table
## Fail Success
## Control 35 17
## Treatment 26 29
4.3) 2by2 table with print option
(test="chisq")
# 2 by 2 table: printed output
pkonfound(a = 35, b = 17, c = 17, d = 38, alpha = 0.05, switch_trm = T, test = "chisq", replace = "control", "to_return" = "print")
## Background Information:
## This function calculates the number of cases that would have to be replaced
## with no effect cases (RIR) to invalidate an inference made about the association
## between the rows and columns in a 2x2 table.
## One can also interpret this as switches from one cell to another, such as from
## the treatment success cell to the treatment failure cell.
##
## Conclusion:
## To invalidate the inference, you would need to replace 15 treatment success
## cases for which the probability of failure in the control group applies (RIR = 15).
## This is equivalent to transferring 10 cases from treatment success to treatment failure.
## For the User-entered Table, we have a Pearson's chi square of 14.176, with p-value of 0.000:
##
## User-entered Table:
## Fail Success
## Control 35 17
## Treatment 17 38
##
##
## For the Transfer Table, we have a Pearson's chi square of 3.640, with p-value of 0.056:
## Transfer Table:
## Fail Success
## Control 35 17
## Treatment 27 28
##
## RIR:
## RIR = 15
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
4.4) 2by2 table with raw_output option
(test="chisq")
# 2 by 2 table: raw output
pkonfound(a = 35, b = 17, c = 17, d = 38, alpha = 0.05, switch_trm = T, test = "chisq", replace = "control", "to_return" = "raw_output")
## For interpretation, check out to_return = 'print'.
## $RIR_primary
## [1] 15
##
## $RIR_supplemental
## [1] 0
##
## $RIR_perc
## [1] 39.47368
##
## $fragility_primary
## [1] 10
##
## $starting_table
## Fail Success
## Control 35 17
## Treatment 17 38
##
## $final_table
## Fail Success
## Control 35 17
## Treatment 27 28