0) install konfound from github repository
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr 1.1.4 ✔ readr 2.1.4
## ✔ forcats 1.0.0 ✔ stringr 1.5.1
## ✔ ggplot2 3.4.4 ✔ tibble 3.2.1
## ✔ lubridate 1.9.2 ✔ tidyr 1.3.0
## ✔ purrr 1.0.2
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(evaluate)
library(githubinstall)
gh_install_packages("konfound", ref = "newitcv_output", force = TRUE)
## Warning in fread(download_url, sep = "\t", header = FALSE, stringsAsFactors =
## FALSE, : Found and resolved improper quoting out-of-sample. First healed line
## 4848: <<Puriney honfleuR "Evening, honfleuR" by Seurat>>. If the fields are not
## quoted (e.g. field separator does not appear within any field), try quote="" to
## avoid this warning.
## Suggestion:
## - jrosen48/konfound R package for carrying out sensitivity analysis
## Do you want to install the package (Y/n)?
## Downloading GitHub repo jrosen48/konfound@newitcv_output
##
## ── R CMD build ─────────────────────────────────────────────────────────────────
##
checking for file ‘/private/var/folders/52/my2sfsnn1qs78nfvl14zyynh0000gn/T/RtmpGZgPOX/remotesb6d33db92aaf/konfound-project-konfound-2479a00/DESCRIPTION’ ...
✔ checking for file ‘/private/var/folders/52/my2sfsnn1qs78nfvl14zyynh0000gn/T/RtmpGZgPOX/remotesb6d33db92aaf/konfound-project-konfound-2479a00/DESCRIPTION’
##
─ preparing ‘konfound’:
##
checking DESCRIPTION meta-information ...
✔ checking DESCRIPTION meta-information
##
─ checking for LF line-endings in source and make files and shell scripts
##
─ checking for empty or unneeded directories
##
─ building ‘konfound_0.4.0.tar.gz’
##
##
library(konfound)
## Sensitivity analysis as described in Frank, Maroulis, Duong, and Kelcey (2013) and in Frank (2000).
## For more information visit http://konfound-it.com.
1) Output Language for RIR in Linear Model
1.1) Invalidate, eff_thr = NA
# Case 1: Linear
### invalidate
## eff_thr = NA
pkonfound(est_eff = -0.3, std_err = 0.01, n_obs = 5000, n_covariates = 10, alpha = .05, tails = 2, index = "RIR", nu = 0, model_type = "ols", eff_thr = NA, to_return = "print")
## Robustness of Inference to Replacement (RIR):
## TO INVALIDATE:
##
## RIR = 4673
##
## You entered an estimated effect of -0.3. To invalidate
## the inference of an effect using the threshold of -0.02 for
## statistical significance with alpha = 0.05 ,93.465% of the
## (-0.3) estimate would have to be due to bias. This implies
## that to invalidate the inference one would expect to have to
## replace 4673 observations with cases for which the treatment
## effect is 0 (RIR = 4673).
##
## See Frank et al. (2013) for a description of the method.
##
## Citation: Frank, K.A., Maroulis, S., Duong, M., and Kelcey, B. (2013).
## What would it take to change an inference?
## Using Rubin's causal model to interpret the robustness of causal inferences.
## Education, Evaluation and Policy Analysis, 35 437-460.
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
1.2) Invalidate, eff_thr != NA (user-entered
threshold)
## eff_thr != NA (user entered)
pkonfound(est_eff = -0.3, std_err = 0.01, n_obs = 5000, n_covariates = 10, alpha = .05, tails = 2, index = "RIR", nu = 0, model_type = "ols", eff_thr = 0.2, to_return = "print")
## Warning in test_sensitivity(est_eff = est_eff, std_err = std_err, n_obs = n_obs, : The threshold you specified will be used without
## regard for the standard error and statistical significance
## (assuming sdx = sdy = 1 unless otherwise specified).
## If you seek to account for the standard error,
## specify a non-zero null hypothesis as in the nu argument.
## Robustness of Inference to Replacement (RIR):
## TO INVALIDATE:
##
## RIR = 4673
##
## You entered an effect of -0.3, and specified a threshold
## value of 0.2. To invalidate the inference based on your estimate
## 93.465 % of the (-0.3) estimate would have to be due to bias.
## This implies that to invalidate the inference one would expect
## to have to replace 4673 observations with cases for which
## the treatment effect is 0 (RIR = 4673).
##
## See Frank et al. (2013) for a description of the method.
##
## Citation: Frank, K.A., Maroulis, S., Duong, M., and Kelcey, B. (2013).
## What would it take to change an inference?
## Using Rubin's causal model to interpret the robustness of causal inferences.
## Education, Evaluation and Policy Analysis, 35 437-460.
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
1.3) Sustain, eff_thr = NA
### sustain
## eff_thr = NA
pkonfound(est_eff = 0.003, std_err = 0.1, n_obs = 5000, n_covariates = 10, alpha = .05, tails = 2, index = "RIR", nu = 0, model_type = "ols", eff_thr = NA, to_return = "print")
## Robustness of Inference to Replacement (RIR):
## TO SUSTAIN:
##
## RIR = 4923
##
## You entered an estimated effect of 0.003. The threshold value for
## statistical significance is 0.196 (alpha = 0.05). To reach that
## threshold, 98.47% of the (0.003) estimate would have to be due to
## bias. This implies that to sustain the inference on would expect to
## have to replace 4923 observation with effect of 0 with cases with
## effect of 0.196(RIR = 4923).
##
## See Frank et al. (2013) for a description of the method.
##
## Citation: Frank, K.A., Maroulis, S., Duong, M., and Kelcey, B. (2013).
## What would it take to change an inference?
## Using Rubin's causal model to interpret the robustness of causal inferences.
## Education, Evaluation and Policy Analysis, 35 437-460.
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
1.4) Sustain, eff_thr != NA (user-entered
threshold)
## eff_thr != NA (user entered)
pkonfound(est_eff = 0.09, std_err = 0.1, n_obs = 5000, n_covariates = 10, alpha = .05, tails = 2, index = "RIR", nu = 0, model_type = "ols", eff_thr = 0.2, to_return = "print")
## Warning in test_sensitivity(est_eff = est_eff, std_err = std_err, n_obs = n_obs, : The threshold you specified will be used without
## regard for the standard error and statistical significance
## (assuming sdx = sdy = 1 unless otherwise specified).
## If you seek to account for the standard error,
## specify a non-zero null hypothesis as in the nu argument.
## Robustness of Inference to Replacement (RIR):
## TO SUSTAIN:
##
## RIR = 2705
##
## You entered an effect size of 0.09, and specified a threshold
## value of 0.2. To reach that threshold, 54.092% of the (0.09)
## estimate would have to be due to bias. This implies that
## to sustain the inference one would expect to have to replace
## 2705 observation with effect of 0 with cases with effect
## of 0.196(RIR = 2705).
##
## See Frank et al. (2013) for a description of the method.
##
## Citation: Frank, K.A., Maroulis, S., Duong, M., and Kelcey, B. (2013).
## What would it take to change an inference?
## Using Rubin's causal model to interpret the robustness of causal inferences.
## Education, Evaluation and Policy Analysis, 35 437-460.
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
2) Output Language for RIR in Non-linear Model
2.1) Invalidate, changeSE = TRUE
# Case 2: Non-linear
## Case 2-1: changeSE = TRUE, Invalidate
pkonfound(-0.3, 0.01, 5000, n_covariates = 0, alpha = .05, tails = 2, nu = 0, n_treat = 2500, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 220
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 1157 1343
## Treatment 1344 1156
##
## (The reported effect size = -0.300, SE = 0.057, t-ratio = -5.284
## Values have been rounded to the nearest integer. This may cause
## a little change to the estimated effect for the table. In order
## to generate a usable implied contingency table, we increased
## the standard error to 0.057. (the reported standard error is 0.010))
##
## To invalidate the inference that the effect is greater than 0
## (alpha =0.05) you would need to replace 220 treatment failure
## cases with cases for which the probability of failure in the
## control group applies (RIR = 220).This is equivalent to transferring
## 118 case from treatment failure to treatment success (Fragility = 118).
## This transfer of cases yields the following table:
##
## Table after transfer:
##
## Fail Success
## Control 1157 1343
## Treatment 1226 1274
## (Effect size = -0.111, SE = 0.057, t-ratio = -1.954)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
2.2) Invalidate, changeSE = FALSE
## Case 2-2: changeSE = FALSE, Invalidate
pkonfound(-0.3, 0.1, 5000, n_covariates = 10, alpha = .05, tails = 2, n_treat = 2500, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 207
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 2246 254
## Treatment 2307 193
##
## (The reported effect size = -0.301, SE = 0.100, t-ratio = -3.015
## Values have been rounded to the nearest integer. This may cause
## a little change to the estimated effect for the table.)
##
## To invalidate the inference that the effect is greater than 0
## (alpha =0.05) one would need to replace 207 treatment failure
## cases with cases for which the probability of failure in the
## control group applies (RIR = 207).This is equivalent to
## transferring 21 case from treatment failure to treatment success
## table (Fragility = 21). This transfer of cases yields the following
## table:
##
## Table after transfer:
##
## Fail Success
## Control 2246 254
## Treatment 2286 214
## (Effect size = -0.189, SE = 0.097, t-ratio = -1.940)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
2.3) Sustain, changeSE = TRUE
## Case 2-3: changeSE = TRUE, Sustain
pkonfound(-0.03, 0.2, 50, n_covariates = 5, alpha = .005, tails = 2, nu = 0, n_treat = 25, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 21
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 12 13
## Treatment 13 12
##
## (The reported effect size = -0.160, SE = 0.566, t-ratio = -0.283
## Values have been rounded to the nearest integer. This may cause
## a little change to the estimated effect for the table. In order
## to generate a usable implied contingency table, we increased
## the standard error to 0.566. (the reported standard error is 0.200))
##
## To reach the threshold that would sustain the inference that the
## effect is greater than 0 (alpha =0.01) you would need to replace
## 21 treatment success cases with cases for which the probability of
## failure in the control group applies (RIR = 21).This is equivalent
## to transferring 10 case from treatment success to treatment failure
## table (Fragility = 10). This transfer of cases yields the following
## table:
##
## Table after transfer:
##
## Fail Success
## Control 12 13
## Treatment 23 2
## (Effect size = -2.522, SE = 0.839, t-ratio = -3.007)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
2.4) Sustain, changeSE = FALSE
## Case 2-4: changeSE = FALSE, Sustain
pkonfound(-0.03, 0.2, 500, n_covariates = 10, alpha = .05, tails = 2, nu = 0, n_treat = 250, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 24
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 180 70
## Treatment 182 68
##
## (The reported effect size = -0.040, SE = 0.200, t-ratio = -0.200
## Values have been rounded to the nearest integer. This may cause
## a little change to the estimated effect for the table.)
##
## To reach the threshold that would sustain the inference that the
## effect is greater than 0 (alpha =0.05) one would need to replace
## 24 treatment success cases with cases for which the probability of
## failure in the control group applies (RIR = 24).This is equivalent
## to transferring 17 case from treatment success to treatment failure
## table (Fragility = 17). This transfer of cases yields the following
## table:
##
## Table after transfer:
##
## Fail Success
## Control 180 70
## Treatment 199 51
## (Effect size = -0.417, SE = 0.211, t-ratio = -1.977)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
3) Output Language for RIR in Non-linear Model
(needtworows = T)
- Examining code line by line,
needtworows = TRUE is a
relatively rare case. Additionally, there are constraints that prevent
invalidate_ob = TRUE when
needtworows = TRUE.
- For instance, an extremely small
alpha value (e.g.,
0.0001) combined with a small number of observations
(n_obs, e.g., fewer than 50) is a major factor leading to
needtworows = TRUE.
- However, this scenario also results in a larger
thr_t,
which tends to make invalidate_ob = TRUE in almost all
cases.
- Therefore, even though language outputs are made for the all
possible
needtworows = TRUE scenario, finding a real case
that satisfies this condition might be challenging.
- Similarly, a small
n_obs is closely associated with
minse being greater than user_std_err.
Consequently, it is also difficult to find a real case where
changeSE = FALSE under the needtworows = TRUE
condition.
3.1) needtworows = T, Sustain,
changeSE = TRUE
## Case 3: needtworows = T, Sustain, changeSE = T,
pkonfound(0.25, 0.5, 70, n_covariates = 20, alpha = .001, tails = 2, nu = 0, n_treat = 20, switch_trm = TRUE, model_type = "logistic", to_return = "print")
## RIR = 37
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 27 23
## Treatment 10 10
##
## (The reported effect size = 0.160, SE = 0.530, t-ratio = 0.303
## Values have been rounded to the nearest integer. This may cause
## a little change to the estimated effect for the table. In order
## to generate a usable implied contingency table, we increased
## the standard error to 0.530. (the reported standard error is 0.500))
##
## The inference cannot be sustained merely by switching cases in
## only one treatment condition. Therefore,cases have been switched from
## treatment failure to treatment success and from control success to control failure.
## The final Fragility(= 17) and RIR(= 37) reflects both sets of changes.
## Please compare the after transfer table with the implied table.
##
## Table after transfer:
##
## Fail Success
## Control 35 15
## Treatment 1 19
## (Effect size = 3.792, SE = 1.071, t-ratio = 3.539)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
3.2) Another case for needtworows = T, Sustain,
changeSE = TRUE
## Case 3: needtworows = T, Sustain, changeSE = T,
pkonfound(0.8, 0.2, 50, n_covariates = 20, alpha = .001, tails = 2, nu = 0, n_treat = 20, switch_trm = FALSE, model_type = "logistic", to_return = "print")
## RIR = 19
##
## The table you entered or is implied by your effect size:
##
## Fail Success
## Control 19 11
## Treatment 8 12
##
## (The reported effect size = 0.952, SE = 0.593, t-ratio = 1.605
## Values have been rounded to the nearest integer. This may cause
## a little change to the estimated effect for the table. In order
## to generate a usable implied contingency table, we increased
## the standard error to 0.593. (the reported standard error is 0.200))
##
## The inference cannot be sustained merely by switching cases in
## only one treatment condition. Therefore,cases have been switched from
## control success to control failure and from treatment failure to treatment success.
## The final Fragility(= 12) and RIR(= 19) reflects both sets of changes.
## Please compare the after transfer table with the implied table.
##
## Table after transfer:
##
## Fail Success
## Control 29 1
## Treatment 6 14
## (Effect size = 4.215, SE = 1.128, t-ratio = 3.736)
## For other forms of output, run ?pkonfound and inspect the to_return argument
## For models fit in R, consider use of konfound().
3.3) Part of code of language output for possible case,
needtworows = T & invalidate &
changeSE = TRUE
else {
### when needtworows = T
### changeSE = T
cat(paste0(
sprintf("The inference cannot be invalidated merely by switching cases in"),
sprintf("\nonly one treatment condition. Therefore,cases have been switched from"),
c("\n"), transferway, c(" and from "),
transferway_extra, c("."), c("\n"),
sprintf("The final Fragility(= %d) and RIR(= %d)", final_solution$final_switch, RIR),
c(" reflects both sets of changes. \nPlease compare the after transfer table with the implied table.")
)
)
}