Power Calculations

knitr::opts_chunk$set(echo = TRUE, warning = FALSE, message = FALSE)
source("https://raw.githubusercontent.com/scmcdonald/visualization/main/theme_sarah.R")

library(extraDistr)
library(googlesheets4)
library(tidyverse)
library(truncnorm)
library(reshape2)
library(stringr)
library(knitr)
gs4_auth()
te <- read_sheet("https://docs.google.com/spreadsheets/d/11tPHIUMUe9VIgZWPyAgNKM5ksMdoBB_8hJ0yJ6Q4yxY/edit#gid=112727328", sheet = 2)

Gamma - Poisson Distribution

We will use rgpois from extraDistr. The probability mass function is the same as this description, except that \(\alpha\) and \(\beta\) are switched.

We know that \(\mu = \alpha \beta)\) and \(\sigma^2 = \alpha \beta + \beta^2\alpha\).

So, given \(\mu\) and \(\sigma^2\), we can solve for \(\alpha\) and \(\beta\).

\(\mu = \alpha\beta\)

\(\implies \alpha = \frac{\mu}{\beta}\)

\(\sigma^2 = \alpha\beta + \alpha\beta^2\)

\(\implies \sigma^2 = \mu + \alpha\beta^2\)

\(\implies \sigma^2 - \mu = \alpha\beta^2 = \frac{\mu}{\beta}\beta^2 = \mu \beta\)

\(\implies \beta = \frac{\sigma^2 - \mu }{\mu}\)

and

\(\alpha = \frac{\mu}{\beta}\)

gamma_poisson <- function(mean, sd, n){
  
  gp_out <- list()
  
  beta = (sd^2 - mean)/mean
  alpha = mean/beta
  
  X <- rgpois(n = n, shape = alpha, scale = beta)
  
  return(X)
  
}


beta_fun <- function(mean, sd, n){
  k <- (mean*(1-mean)/(sd^2) - 1)
  alpha_val <- mean*k
  beta_val <- (1-mean)*k
  X <- rbeta(n, alpha_val, beta_val)
  
  return(X)
}

Zeros to NA

zeros_to_NA <- function(vec){
  vec[vec == 0] <- NA
  
  return(vec)
}

samples <- data.frame(
  sample = 1:4, 
  mean = c(8.5218, 11.5010, 13.9309, 18.5348), 
  sd = c(23.9048, 27.8013, 30.6623, 36.6466)
)

DGP

dgp <- function(n, p, sample, base_mean_score, base_sd_score, treatments = c("control", "sms", "game", "video")){

  # step 1: treatment assignment and initiate outcomes
  # step 2: number of posts shared weekly, bi-weekly, monthly
  df <- data.frame(treatment = sample(treatments, n, replace = T, prob = c(p, rep((1-p)/3, 3))), 
                   week1 = NA,
                   week2 = NA,
                   week3 = NA, 
                   week4 =NA, 
                   week1_score = NA, 
                   week2_score = NA, 
                   week3_score = NA, 
                   week4_score= NA,
                   biweek1_score = NA,
                   biweek2_score = NA,
                   monthly_score = NA)
  
  # weekly counts
  df[, c("week1", "week2", "week3", 'week4')] <- replicate(4, 
     gamma_poisson(samples[samples$sample == sample, ]$mean,  samples[samples$sample == sample, ]$sd, n)
  )
  
  # calculate nuumber of posts or biweekly and monthly
  df$biweek1 <- df$week1 + df$week2
  df$biweek2 <- df$week3 + df$week4
  
  df$monthly <- df$biweek1 + df$biweek2
  
  # average across weeks
  
  df$average_weekly <-  rowMeans(df[, c("week1", "week2", "week3", "week4")])
  
  # get TE level
  df$te_level <- as.numeric(cut(df$average_weekly , breaks = c(0, .25, 1, 5, 10, Inf), ordered_result = T))
  df[is.na(df$te_level ), "te_level"] <- 0
  
  # make 0s NA
  df[, c("week1", "week2", "week3", "week4", "biweek1", "biweek2", "monthly")] <- vapply(
    df[, c("week1", "week2", "week3", "week4", "biweek1", "biweek2", "monthly")], 
    zeros_to_NA, numeric(nrow(df)))

  # step 3: Simulate misinfo score weekly, bi-weekly, monthly
  
  te_groups <- as.data.frame(table(treatment = df$treatment, te_level = df$te_level), responseName = "nobs")
  
  te_groups <- te_groups[te_groups$treatment != "control",]
  te_groups <- te_groups[te_groups$te_level != 0,]
  

  
  # control observations
  
  df[df$treatment == "control", c("week1_score", "week2_score", "week3_score", "week4_score", "biweek1_score", "biweek2_score", "monthly_score")] <- 
    lapply( c("week1", "week2", "week3", "week4", "biweek1", "biweek2", "monthly"), function(x){ 
      
    # draw for y
      y <- beta_fun(
        n = sum(df$treatment == "control"), 
        mean= base_mean_score, 
        sd = base_sd_score) 
        # if variable was NA, outcome will be NA

      y[is.na(df[df$treatment == "control", x])] <- NA
      
    return(y)
    
  })
  

    # treatments
  
  for(i in 1:nrow(te_groups)) {
    
    treat =  te_groups[i, "treatment"] 
    level = te_groups[i, "te_level"]
    
    ids <- df$treat == treat & df$te_level == level
    
    df[ids,  c("week1_score", "week2_score", "week3_score", "week4_score", "biweek1_score", "biweek2_score", "monthly_score")] <- 
      vapply( c("week1", "week2", "week3", "week4", "biweek1", "biweek2", "monthly"), function(x){ 
      
    time_id <- str_remove(x, pattern = "[0-9]")
    
    # draw for y
    y <-beta_fun(
      n =te_groups[i, "nobs"], 
      mean= base_mean_score - te[te$te_level ==level  & te$treatment ==treat & te$time == time_id,]$mean, 
      sd = base_sd_score)
    
    # if variable was NA, outcome will be NA
    y[is.na(df[ids, x])] <- NA
    
    return(y)
    
  }, 
  
  numeric(nrow(df[ids, ])))
    
}
  
  # treatments

  return(df)

}

Testing

Hypothesis 1: Control v Game

Hypothesis 2: Control v SMS

Hypothesis 3: Control v Video

trial <- function(n, p, sample, base_mean_score, base_sd_score, treatments = c("control", "sms", "game", "video") ){
 
  # DGP
  df_sim <- dgp(n, p, sample, base_mean_score, base_sd_score, treatments)

  outcomes <-  c("week1_score", "biweek1_score", "monthly_score")



  # PVALUES matrix
  # pvalues is a function of each outcome
  p_values <- lapply(setNames(outcomes, outcomes), FUN = function(outcome){   

     test_df <- df_sim[!is.na(df_sim[[outcome]]), c("treatment",outcome)]

        # T< C
        # this does all comparison at once for an outcome and outputs a matrix
        p_matrix <- pairwise.t.test(x = test_df[[outcome]],
                                    g = test_df$treatment,
                                    p.adjust.method = "none",
                                    pool.sd = F,
                                    alternative = "less")$p.value[, 1]
        
        # output vector
       return(p_matrix)

  })
  

  p_values <-  do.call("cbind", p_values)
  
  # correct within outcome for 3 hypotheses
  bonferroni <- apply(p_values, 2, function(x)p.adjust(x, method = "bonferroni"))
  holm <- apply(p_values, 2, function(x)p.adjust(x, method = "holm"))
  hochberg <- apply(p_values, 2, function(x)p.adjust(x, method = "hochberg"))

  # return matrices of p-values
  return(pvalues = list(none = p_values,
              bonferroni = bonferroni, 
              holm = holm, 
              hochberg = hochberg))
}

Power

# this function runs trial r times and computes the power
power <- function(n, p, sample, base_mean_score, base_sd_score, treatments = c("control", "game", "sms", "video"), r, alpha){

  # replicate trial r times
  results <- replicate(r,  trial(n, p, sample, base_mean_score, base_sd_score, treatments))

  # empty df to hold results
  power_df <- data.frame()
  
  out_dim <- prod(dim(results[1, ][[1]]))
  
  # loop over corrections
  for( i in c("none", "bonferroni", "holm", "hochberg")){

      tbl <- results[i, ]
      
      # makes True False matrices for if each individual values is below alpha = 0.05
      alpha_matrix <- lapply(tbl, function(x) x < alpha)
      
      # sum up how many obs are below alpha, and divide by number of replications
      power_out <- apply(simplify2array(alpha_matrix), c(1,2), sum)/r
    
      # visualization df
      power_df <- rbind(power_df, cbind(melt(power_out,varnames=c("hypothesis", "outcome"),  value.name = "power"), 
                                        correction =rep(i, out_dim), 
                                        N = rep(n, out_dim), 
                                        p_control = rep(p, out_dim), 
                                        sample= rep(sample, out_dim)
                                        ) )
  }
  
  return(power_df)
  
}

power_out <- list()
for(n in c(5000, 8000, 10000, 12000, 15000, 20000, 22000, 25000, 30000)){
  for(p in c(0.25, 0.3, 0.4)){
    for(s in 1:4){
  power_out[[paste(n, p, s)]] <- power(n = n, 
              p =  p, 
              sample = s, 
              base_mean_score =  0.109, 
              base_sd_score = 0.094, r = 100, alpha = .05)
  }
  }
}

power_final <- do.call("rbind", power_out)

write.csv(power_final, "/Users/scmcd/Documents/git/fb_misinfo_interventions/power_calculations/power_v2023.csv", row.names = F)

power_final <- read.csv("/Users/scmcd/Documents/git/fb_misinfo_interventions/power_calculations/power_v2023.csv")

Table

power_final%>%
  pivot_wider(names_from = outcome, values_from = power) %>%
  kable(digits = 3)

hypothesis	correction	N	p_control	sample	week1_score	biweek1_score	monthly
game	none	5000	0.25	1	0.23	0.31	0.06
sms	none	5000	0.25	1	0.72	0.91	0.09
video	none	5000	0.25	1	0.53	0.57	0.08
game	bonferroni	5000	0.25	1	0.12	0.14	0.02
sms	bonferroni	5000	0.25	1	0.53	0.78	0.07
video	bonferroni	5000	0.25	1	0.27	0.39	0.03
game	holm	5000	0.25	1	0.19	0.28	0.02
sms	holm	5000	0.25	1	0.57	0.81	0.07
video	holm	5000	0.25	1	0.36	0.48	0.03
game	hochberg	5000	0.25	1	0.21	0.29	0.02
sms	hochberg	5000	0.25	1	0.58	0.82	0.07
video	hochberg	5000	0.25	1	0.39	0.50	0.03
game	none	5000	0.25	2	0.28	0.37	0.07
sms	none	5000	0.25	2	0.81	0.93	0.05
video	none	5000	0.25	2	0.61	0.72	0.04
game	bonferroni	5000	0.25	2	0.15	0.28	0.02
sms	bonferroni	5000	0.25	2	0.72	0.86	0.04
video	bonferroni	5000	0.25	2	0.42	0.48	0.00
game	holm	5000	0.25	2	0.23	0.33	0.02
sms	holm	5000	0.25	2	0.73	0.87	0.04
video	holm	5000	0.25	2	0.49	0.58	0.01
game	hochberg	5000	0.25	2	0.25	0.35	0.02
sms	hochberg	5000	0.25	2	0.73	0.87	0.04
video	hochberg	5000	0.25	2	0.51	0.60	0.01
game	none	5000	0.25	3	0.26	0.40	0.02
sms	none	5000	0.25	3	0.83	0.98	0.03
video	none	5000	0.25	3	0.58	0.72	0.02
game	bonferroni	5000	0.25	3	0.11	0.21	0.00
sms	bonferroni	5000	0.25	3	0.74	0.92	0.01
video	bonferroni	5000	0.25	3	0.34	0.53	0.01
game	holm	5000	0.25	3	0.22	0.36	0.00
sms	holm	5000	0.25	3	0.75	0.92	0.01
video	holm	5000	0.25	3	0.43	0.59	0.01
game	hochberg	5000	0.25	3	0.22	0.38	0.00
sms	hochberg	5000	0.25	3	0.75	0.92	0.01
video	hochberg	5000	0.25	3	0.43	0.61	0.01
game	none	5000	0.25	4	0.29	0.41	0.05
sms	none	5000	0.25	4	0.84	0.96	0.08
video	none	5000	0.25	4	0.62	0.76	0.02
game	bonferroni	5000	0.25	4	0.18	0.21	0.02
sms	bonferroni	5000	0.25	4	0.71	0.93	0.01
video	bonferroni	5000	0.25	4	0.39	0.60	0.00
game	holm	5000	0.25	4	0.24	0.38	0.02
sms	holm	5000	0.25	4	0.72	0.93	0.02
video	holm	5000	0.25	4	0.49	0.65	0.01
game	hochberg	5000	0.25	4	0.24	0.39	0.02
sms	hochberg	5000	0.25	4	0.74	0.93	0.02
video	hochberg	5000	0.25	4	0.51	0.66	0.01
game	none	5000	0.30	1	0.28	0.38	0.07
sms	none	5000	0.30	1	0.83	0.95	0.03
video	none	5000	0.30	1	0.48	0.68	0.06
game	bonferroni	5000	0.30	1	0.11	0.29	0.01
sms	bonferroni	5000	0.30	1	0.67	0.86	0.01
video	bonferroni	5000	0.30	1	0.38	0.55	0.02
game	holm	5000	0.30	1	0.19	0.33	0.02
sms	holm	5000	0.30	1	0.69	0.87	0.01
video	holm	5000	0.30	1	0.43	0.63	0.03
game	hochberg	5000	0.30	1	0.20	0.34	0.02
sms	hochberg	5000	0.30	1	0.71	0.87	0.01
video	hochberg	5000	0.30	1	0.45	0.64	0.03
game	none	5000	0.30	2	0.29	0.38	0.04
sms	none	5000	0.30	2	0.89	0.94	0.04
video	none	5000	0.30	2	0.60	0.76	0.04
game	bonferroni	5000	0.30	2	0.15	0.20	0.01
sms	bonferroni	5000	0.30	2	0.77	0.82	0.02
video	bonferroni	5000	0.30	2	0.50	0.53	0.00
game	holm	5000	0.30	2	0.25	0.33	0.01
sms	holm	5000	0.30	2	0.81	0.84	0.02
video	holm	5000	0.30	2	0.53	0.62	0.00
game	hochberg	5000	0.30	2	0.25	0.37	0.01
sms	hochberg	5000	0.30	2	0.81	0.87	0.02
video	hochberg	5000	0.30	2	0.53	0.65	0.00
game	none	5000	0.30	3	0.37	0.48	0.07
sms	none	5000	0.30	3	0.91	0.98	0.09
video	none	5000	0.30	3	0.69	0.74	0.06
game	bonferroni	5000	0.30	3	0.16	0.30	0.02
sms	bonferroni	5000	0.30	3	0.81	0.94	0.04
video	bonferroni	5000	0.30	3	0.52	0.58	0.03
game	holm	5000	0.30	3	0.30	0.42	0.02
sms	holm	5000	0.30	3	0.82	0.94	0.04
video	holm	5000	0.30	3	0.57	0.67	0.03
game	hochberg	5000	0.30	3	0.34	0.45	0.03
sms	hochberg	5000	0.30	3	0.84	0.95	0.05
video	hochberg	5000	0.30	3	0.59	0.69	0.03
game	none	5000	0.30	4	0.30	0.47	0.06
sms	none	5000	0.30	4	0.88	0.95	0.04
video	none	5000	0.30	4	0.59	0.79	0.09
game	bonferroni	5000	0.30	4	0.18	0.34	0.02
sms	bonferroni	5000	0.30	4	0.74	0.89	0.02
video	bonferroni	5000	0.30	4	0.41	0.61	0.02
game	holm	5000	0.30	4	0.29	0.46	0.02
sms	holm	5000	0.30	4	0.75	0.89	0.02
video	holm	5000	0.30	4	0.49	0.68	0.02
game	hochberg	5000	0.30	4	0.29	0.47	0.02
sms	hochberg	5000	0.30	4	0.76	0.89	0.02
video	hochberg	5000	0.30	4	0.50	0.69	0.02
game	none	5000	0.40	1	0.33	0.31	0.05
sms	none	5000	0.40	1	0.76	0.86	0.06
video	none	5000	0.40	1	0.48	0.53	0.05
game	bonferroni	5000	0.40	1	0.18	0.19	0.01
sms	bonferroni	5000	0.40	1	0.61	0.79	0.03
video	bonferroni	5000	0.40	1	0.31	0.37	0.02
game	holm	5000	0.40	1	0.26	0.26	0.01
sms	holm	5000	0.40	1	0.63	0.82	0.03
video	holm	5000	0.40	1	0.37	0.44	0.02
game	hochberg	5000	0.40	1	0.29	0.27	0.01
sms	hochberg	5000	0.40	1	0.65	0.82	0.03
video	hochberg	5000	0.40	1	0.41	0.45	0.02
game	none	5000	0.40	2	0.34	0.47	0.04
sms	none	5000	0.40	2	0.89	0.98	0.04
video	none	5000	0.40	2	0.64	0.67	0.04
game	bonferroni	5000	0.40	2	0.24	0.31	0.02
sms	bonferroni	5000	0.40	2	0.76	0.91	0.00
video	bonferroni	5000	0.40	2	0.48	0.56	0.02
game	holm	5000	0.40	2	0.29	0.44	0.02
sms	holm	5000	0.40	2	0.78	0.94	0.00
video	holm	5000	0.40	2	0.56	0.63	0.02
game	hochberg	5000	0.40	2	0.31	0.44	0.02
sms	hochberg	5000	0.40	2	0.80	0.94	0.00
video	hochberg	5000	0.40	2	0.57	0.63	0.02
game	none	5000	0.40	3	0.30	0.42	0.05
sms	none	5000	0.40	3	0.91	0.99	0.09
video	none	5000	0.40	3	0.67	0.75	0.03
game	bonferroni	5000	0.40	3	0.15	0.22	0.01
sms	bonferroni	5000	0.40	3	0.75	0.91	0.02
video	bonferroni	5000	0.40	3	0.49	0.49	0.01
game	holm	5000	0.40	3	0.25	0.36	0.01
sms	holm	5000	0.40	3	0.76	0.91	0.02
video	holm	5000	0.40	3	0.52	0.58	0.01
game	hochberg	5000	0.40	3	0.29	0.38	0.02
sms	hochberg	5000	0.40	3	0.77	0.91	0.03
video	hochberg	5000	0.40	3	0.56	0.60	0.01
game	none	5000	0.40	4	0.38	0.37	0.06
sms	none	5000	0.40	4	0.93	1.00	0.07
video	none	5000	0.40	4	0.60	0.79	0.03
game	bonferroni	5000	0.40	4	0.22	0.20	0.02
sms	bonferroni	5000	0.40	4	0.83	0.97	0.01
video	bonferroni	5000	0.40	4	0.47	0.59	0.01
game	holm	5000	0.40	4	0.32	0.35	0.02
sms	holm	5000	0.40	4	0.84	0.98	0.01
video	holm	5000	0.40	4	0.50	0.70	0.01
game	hochberg	5000	0.40	4	0.36	0.36	0.03
sms	hochberg	5000	0.40	4	0.85	0.99	0.02
video	hochberg	5000	0.40	4	0.54	0.72	0.01
game	none	8000	0.25	1	0.27	0.52	0.02
sms	none	8000	0.25	1	0.84	0.99	0.05
video	none	8000	0.25	1	0.62	0.79	0.05
game	bonferroni	8000	0.25	1	0.11	0.30	0.02
sms	bonferroni	8000	0.25	1	0.66	0.96	0.01
video	bonferroni	8000	0.25	1	0.44	0.59	0.02
game	holm	8000	0.25	1	0.21	0.47	0.02
sms	holm	8000	0.25	1	0.69	0.96	0.01
video	holm	8000	0.25	1	0.52	0.68	0.02
game	hochberg	8000	0.25	1	0.23	0.50	0.02
sms	hochberg	8000	0.25	1	0.70	0.96	0.01
video	hochberg	8000	0.25	1	0.54	0.71	0.02
game	none	8000	0.25	2	0.35	0.49	0.11
sms	none	8000	0.25	2	0.96	0.99	0.06
video	none	8000	0.25	2	0.76	0.91	0.06
game	bonferroni	8000	0.25	2	0.23	0.32	0.05
sms	bonferroni	8000	0.25	2	0.93	0.96	0.02
video	bonferroni	8000	0.25	2	0.57	0.78	0.00
game	holm	8000	0.25	2	0.34	0.49	0.06
sms	holm	8000	0.25	2	0.93	0.96	0.02
video	holm	8000	0.25	2	0.66	0.83	0.01
game	hochberg	8000	0.25	2	0.35	0.49	0.06
sms	hochberg	8000	0.25	2	0.93	0.96	0.03
video	hochberg	8000	0.25	2	0.67	0.83	0.02
game	none	8000	0.25	3	0.50	0.48	0.06
sms	none	8000	0.25	3	0.98	1.00	0.06
video	none	8000	0.25	3	0.81	0.85	0.05
game	bonferroni	8000	0.25	3	0.32	0.33	0.00
sms	bonferroni	8000	0.25	3	0.89	0.97	0.04
video	bonferroni	8000	0.25	3	0.62	0.75	0.01
game	holm	8000	0.25	3	0.47	0.46	0.01
sms	holm	8000	0.25	3	0.93	0.97	0.04
video	holm	8000	0.25	3	0.70	0.81	0.01
game	hochberg	8000	0.25	3	0.48	0.46	0.01
sms	hochberg	8000	0.25	3	0.93	0.97	0.04
video	hochberg	8000	0.25	3	0.71	0.81	0.01
game	none	8000	0.25	4	0.52	0.58	0.06
sms	none	8000	0.25	4	1.00	0.99	0.06
video	none	8000	0.25	4	0.83	0.91	0.06
game	bonferroni	8000	0.25	4	0.33	0.38	0.02
sms	bonferroni	8000	0.25	4	0.97	0.98	0.01
video	bonferroni	8000	0.25	4	0.65	0.73	0.01
game	holm	8000	0.25	4	0.50	0.57	0.02
sms	holm	8000	0.25	4	0.99	0.98	0.01
video	holm	8000	0.25	4	0.73	0.81	0.02
game	hochberg	8000	0.25	4	0.50	0.57	0.03
sms	hochberg	8000	0.25	4	0.99	0.98	0.02
video	hochberg	8000	0.25	4	0.73	0.81	0.02
game	none	8000	0.30	1	0.40	0.53	0.05
sms	none	8000	0.30	1	0.96	1.00	0.07
video	none	8000	0.30	1	0.71	0.83	0.06
game	bonferroni	8000	0.30	1	0.20	0.29	0.01
sms	bonferroni	8000	0.30	1	0.84	0.99	0.03
video	bonferroni	8000	0.30	1	0.57	0.67	0.02
game	holm	8000	0.30	1	0.36	0.48	0.01
sms	holm	8000	0.30	1	0.89	0.99	0.03
video	holm	8000	0.30	1	0.63	0.76	0.02
game	hochberg	8000	0.30	1	0.37	0.48	0.01
sms	hochberg	8000	0.30	1	0.89	0.99	0.03
video	hochberg	8000	0.30	1	0.64	0.76	0.02
game	none	8000	0.30	2	0.44	0.59	0.04
sms	none	8000	0.30	2	0.96	1.00	0.01
video	none	8000	0.30	2	0.69	0.87	0.05
game	bonferroni	8000	0.30	2	0.30	0.41	0.01
sms	bonferroni	8000	0.30	2	0.88	0.99	0.00
video	bonferroni	8000	0.30	2	0.56	0.75	0.01
game	holm	8000	0.30	2	0.43	0.55	0.01
sms	holm	8000	0.30	2	0.89	1.00	0.00
video	holm	8000	0.30	2	0.67	0.84	0.01
game	hochberg	8000	0.30	2	0.43	0.56	0.01
sms	hochberg	8000	0.30	2	0.89	1.00	0.00
video	hochberg	8000	0.30	2	0.67	0.85	0.01
game	none	8000	0.30	3	0.45	0.62	0.08
sms	none	8000	0.30	3	1.00	1.00	0.05
video	none	8000	0.30	3	0.81	0.89	0.04
game	bonferroni	8000	0.30	3	0.28	0.46	0.01
sms	bonferroni	8000	0.30	3	0.99	0.98	0.04
video	bonferroni	8000	0.30	3	0.63	0.76	0.02
game	holm	8000	0.30	3	0.40	0.61	0.02
sms	holm	8000	0.30	3	1.00	0.98	0.04
video	holm	8000	0.30	3	0.72	0.83	0.02
game	hochberg	8000	0.30	3	0.42	0.61	0.02
sms	hochberg	8000	0.30	3	1.00	0.98	0.04
video	hochberg	8000	0.30	3	0.74	0.83	0.02
game	none	8000	0.30	4	0.45	0.60	0.00
sms	none	8000	0.30	4	0.98	1.00	0.03
video	none	8000	0.30	4	0.81	0.92	0.06
game	bonferroni	8000	0.30	4	0.26	0.39	0.00
sms	bonferroni	8000	0.30	4	0.92	0.99	0.00
video	bonferroni	8000	0.30	4	0.62	0.72	0.01
game	holm	8000	0.30	4	0.42	0.57	0.00
sms	holm	8000	0.30	4	0.93	0.99	0.00
video	holm	8000	0.30	4	0.71	0.81	0.01
game	hochberg	8000	0.30	4	0.43	0.59	0.00
sms	hochberg	8000	0.30	4	0.94	0.99	0.00
video	hochberg	8000	0.30	4	0.71	0.83	0.01
game	none	8000	0.40	1	0.32	0.43	0.05
sms	none	8000	0.40	1	0.92	0.97	0.06
video	none	8000	0.40	1	0.60	0.85	0.07
game	bonferroni	8000	0.40	1	0.17	0.26	0.00
sms	bonferroni	8000	0.40	1	0.73	0.92	0.01
video	bonferroni	8000	0.40	1	0.43	0.68	0.01
game	holm	8000	0.40	1	0.26	0.40	0.00
sms	holm	8000	0.40	1	0.77	0.93	0.01
video	holm	8000	0.40	1	0.51	0.74	0.01
game	hochberg	8000	0.40	1	0.26	0.41	0.01
sms	hochberg	8000	0.40	1	0.78	0.94	0.02
video	hochberg	8000	0.40	1	0.52	0.76	0.01
game	none	8000	0.40	2	0.44	0.42	0.06
sms	none	8000	0.40	2	0.99	0.98	0.05
video	none	8000	0.40	2	0.74	0.83	0.04
game	bonferroni	8000	0.40	2	0.26	0.27	0.01
sms	bonferroni	8000	0.40	2	0.98	0.97	0.03
video	bonferroni	8000	0.40	2	0.58	0.65	0.01
game	holm	8000	0.40	2	0.42	0.41	0.01
sms	holm	8000	0.40	2	0.98	0.97	0.03
video	holm	8000	0.40	2	0.69	0.74	0.01
game	hochberg	8000	0.40	2	0.42	0.42	0.01
sms	hochberg	8000	0.40	2	0.98	0.97	0.03
video	hochberg	8000	0.40	2	0.69	0.75	0.01
game	none	8000	0.40	3	0.49	0.58	0.07
sms	none	8000	0.40	3	0.99	0.99	0.02
video	none	8000	0.40	3	0.87	0.90	0.04
game	bonferroni	8000	0.40	3	0.30	0.40	0.00
sms	bonferroni	8000	0.40	3	0.92	0.98	0.00
video	bonferroni	8000	0.40	3	0.66	0.81	0.01
game	holm	8000	0.40	3	0.44	0.57	0.00
sms	holm	8000	0.40	3	0.94	0.98	0.00
video	holm	8000	0.40	3	0.76	0.88	0.01
game	hochberg	8000	0.40	3	0.48	0.57	0.00
sms	hochberg	8000	0.40	3	0.96	0.98	0.00
video	hochberg	8000	0.40	3	0.79	0.88	0.01
game	none	8000	0.40	4	0.45	0.60	0.04
sms	none	8000	0.40	4	0.99	1.00	0.04
video	none	8000	0.40	4	0.86	0.89	0.02
game	bonferroni	8000	0.40	4	0.30	0.40	0.00
sms	bonferroni	8000	0.40	4	0.97	1.00	0.02
video	bonferroni	8000	0.40	4	0.70	0.78	0.00
game	holm	8000	0.40	4	0.44	0.59	0.00
sms	holm	8000	0.40	4	0.98	1.00	0.02
video	holm	8000	0.40	4	0.79	0.83	0.00
game	hochberg	8000	0.40	4	0.44	0.59	0.00
sms	hochberg	8000	0.40	4	0.98	1.00	0.02
video	hochberg	8000	0.40	4	0.79	0.83	0.00
game	none	10000	0.25	1	0.44	0.62	0.04
sms	none	10000	0.25	1	0.95	1.00	0.04
video	none	10000	0.25	1	0.69	0.90	0.09
game	bonferroni	10000	0.25	1	0.26	0.34	0.03
sms	bonferroni	10000	0.25	1	0.93	0.96	0.02
video	bonferroni	10000	0.25	1	0.56	0.78	0.04
game	holm	10000	0.25	1	0.42	0.58	0.04
sms	holm	10000	0.25	1	0.93	0.97	0.02
video	holm	10000	0.25	1	0.61	0.82	0.05
game	hochberg	10000	0.25	1	0.42	0.59	0.04
sms	hochberg	10000	0.25	1	0.93	0.97	0.02
video	hochberg	10000	0.25	1	0.61	0.83	0.05
game	none	10000	0.25	2	0.45	0.72	0.05
sms	none	10000	0.25	2	0.98	1.00	0.06
video	none	10000	0.25	2	0.77	0.96	0.04
game	bonferroni	10000	0.25	2	0.30	0.46	0.00
sms	bonferroni	10000	0.25	2	0.95	1.00	0.01
video	bonferroni	10000	0.25	2	0.64	0.86	0.03
game	holm	10000	0.25	2	0.44	0.71	0.01
sms	holm	10000	0.25	2	0.95	1.00	0.01
video	holm	10000	0.25	2	0.72	0.90	0.03
game	hochberg	10000	0.25	2	0.44	0.71	0.01
sms	hochberg	10000	0.25	2	0.95	1.00	0.01
video	hochberg	10000	0.25	2	0.72	0.90	0.03
game	none	10000	0.25	3	0.51	0.58	0.07
sms	none	10000	0.25	3	0.98	1.00	0.05
video	none	10000	0.25	3	0.81	0.98	0.06
game	bonferroni	10000	0.25	3	0.30	0.41	0.02
sms	bonferroni	10000	0.25	3	0.95	1.00	0.01
video	bonferroni	10000	0.25	3	0.69	0.83	0.03
game	holm	10000	0.25	3	0.47	0.58	0.02
sms	holm	10000	0.25	3	0.96	1.00	0.01
video	holm	10000	0.25	3	0.74	0.91	0.03
game	hochberg	10000	0.25	3	0.48	0.58	0.02
sms	hochberg	10000	0.25	3	0.96	1.00	0.01
video	hochberg	10000	0.25	3	0.75	0.91	0.03
game	none	10000	0.25	4	0.58	0.64	0.08
sms	none	10000	0.25	4	1.00	1.00	0.05
video	none	10000	0.25	4	0.86	0.97	0.07
game	bonferroni	10000	0.25	4	0.41	0.49	0.03
sms	bonferroni	10000	0.25	4	0.96	1.00	0.01
video	bonferroni	10000	0.25	4	0.72	0.88	0.03
game	holm	10000	0.25	4	0.56	0.64	0.04
sms	holm	10000	0.25	4	0.97	1.00	0.01
video	holm	10000	0.25	4	0.78	0.93	0.05
game	hochberg	10000	0.25	4	0.57	0.64	0.04
sms	hochberg	10000	0.25	4	0.97	1.00	0.01
video	hochberg	10000	0.25	4	0.79	0.93	0.05
game	none	10000	0.30	1	0.34	0.57	0.06
sms	none	10000	0.30	1	0.95	0.99	0.03
video	none	10000	0.30	1	0.66	0.89	0.12
game	bonferroni	10000	0.30	1	0.15	0.43	0.01
sms	bonferroni	10000	0.30	1	0.86	0.98	0.02
video	bonferroni	10000	0.30	1	0.53	0.77	0.02
game	holm	10000	0.30	1	0.28	0.53	0.01
sms	holm	10000	0.30	1	0.87	0.98	0.02
video	holm	10000	0.30	1	0.57	0.81	0.02
game	hochberg	10000	0.30	1	0.31	0.54	0.02
sms	hochberg	10000	0.30	1	0.88	0.98	0.02
video	hochberg	10000	0.30	1	0.60	0.82	0.03
game	none	10000	0.30	2	0.52	0.69	0.08
sms	none	10000	0.30	2	0.99	1.00	0.06
video	none	10000	0.30	2	0.81	0.91	0.06
game	bonferroni	10000	0.30	2	0.33	0.40	0.03
sms	bonferroni	10000	0.30	2	0.97	1.00	0.02
video	bonferroni	10000	0.30	2	0.69	0.81	0.00
game	holm	10000	0.30	2	0.52	0.66	0.03
sms	holm	10000	0.30	2	0.98	1.00	0.02
video	holm	10000	0.30	2	0.78	0.89	0.02
game	hochberg	10000	0.30	2	0.52	0.68	0.03
sms	hochberg	10000	0.30	2	0.98	1.00	0.02
video	hochberg	10000	0.30	2	0.78	0.91	0.02
game	none	10000	0.30	3	0.52	0.68	0.05
sms	none	10000	0.30	3	0.99	1.00	0.09
video	none	10000	0.30	3	0.83	0.97	0.05
game	bonferroni	10000	0.30	3	0.34	0.46	0.02
sms	bonferroni	10000	0.30	3	0.98	1.00	0.00
video	bonferroni	10000	0.30	3	0.69	0.89	0.01
game	holm	10000	0.30	3	0.49	0.67	0.02
sms	holm	10000	0.30	3	0.98	1.00	0.00
video	holm	10000	0.30	3	0.80	0.95	0.01
game	hochberg	10000	0.30	3	0.50	0.67	0.02
sms	hochberg	10000	0.30	3	0.98	1.00	0.01
video	hochberg	10000	0.30	3	0.81	0.95	0.02
game	none	10000	0.30	4	0.54	0.59	0.10
sms	none	10000	0.30	4	1.00	1.00	0.06
video	none	10000	0.30	4	0.89	0.97	0.06
game	bonferroni	10000	0.30	4	0.42	0.43	0.02
sms	bonferroni	10000	0.30	4	0.99	1.00	0.02
video	bonferroni	10000	0.30	4	0.73	0.91	0.01
game	holm	10000	0.30	4	0.53	0.59	0.02
sms	holm	10000	0.30	4	0.99	1.00	0.02
video	holm	10000	0.30	4	0.83	0.94	0.02
game	hochberg	10000	0.30	4	0.53	0.59	0.02
sms	hochberg	10000	0.30	4	0.99	1.00	0.02
video	hochberg	10000	0.30	4	0.83	0.94	0.02
game	none	10000	0.40	1	0.37	0.54	0.08
sms	none	10000	0.40	1	0.97	1.00	0.05
video	none	10000	0.40	1	0.72	0.89	0.03
game	bonferroni	10000	0.40	1	0.24	0.41	0.02
sms	bonferroni	10000	0.40	1	0.93	1.00	0.02
video	bonferroni	10000	0.40	1	0.60	0.80	0.01
game	holm	10000	0.40	1	0.34	0.54	0.02
sms	holm	10000	0.40	1	0.93	1.00	0.02
video	holm	10000	0.40	1	0.68	0.87	0.01
game	hochberg	10000	0.40	1	0.34	0.54	0.02
sms	hochberg	10000	0.40	1	0.93	1.00	0.02
video	hochberg	10000	0.40	1	0.68	0.87	0.01
game	none	10000	0.40	2	0.41	0.57	0.03
sms	none	10000	0.40	2	0.96	1.00	0.03
video	none	10000	0.40	2	0.87	0.94	0.07
game	bonferroni	10000	0.40	2	0.22	0.40	0.00
sms	bonferroni	10000	0.40	2	0.95	1.00	0.01
video	bonferroni	10000	0.40	2	0.70	0.79	0.02
game	holm	10000	0.40	2	0.39	0.56	0.01
sms	holm	10000	0.40	2	0.96	1.00	0.01
video	holm	10000	0.40	2	0.78	0.88	0.02
game	hochberg	10000	0.40	2	0.39	0.57	0.02
sms	hochberg	10000	0.40	2	0.96	1.00	0.02
video	hochberg	10000	0.40	2	0.78	0.89	0.03
game	none	10000	0.40	3	0.48	0.64	0.03
sms	none	10000	0.40	3	1.00	1.00	0.04
video	none	10000	0.40	3	0.86	0.95	0.02
game	bonferroni	10000	0.40	3	0.31	0.35	0.01
sms	bonferroni	10000	0.40	3	1.00	1.00	0.02
video	bonferroni	10000	0.40	3	0.73	0.83	0.01
game	holm	10000	0.40	3	0.46	0.61	0.01
sms	holm	10000	0.40	3	1.00	1.00	0.02
video	holm	10000	0.40	3	0.83	0.91	0.01
game	hochberg	10000	0.40	3	0.47	0.63	0.01
sms	hochberg	10000	0.40	3	1.00	1.00	0.02
video	hochberg	10000	0.40	3	0.84	0.93	0.01
game	none	10000	0.40	4	0.60	0.73	0.10
sms	none	10000	0.40	4	0.99	1.00	0.09
video	none	10000	0.40	4	0.88	0.97	0.08
game	bonferroni	10000	0.40	4	0.30	0.55	0.04
sms	bonferroni	10000	0.40	4	0.95	1.00	0.04
video	bonferroni	10000	0.40	4	0.80	0.91	0.03
game	holm	10000	0.40	4	0.59	0.73	0.04
sms	holm	10000	0.40	4	0.95	1.00	0.04
video	holm	10000	0.40	4	0.82	0.97	0.04
game	hochberg	10000	0.40	4	0.59	0.73	0.04
sms	hochberg	10000	0.40	4	0.95	1.00	0.04
video	hochberg	10000	0.40	4	0.82	0.97	0.04
game	none	12000	0.25	1	0.46	0.63	0.06
sms	none	12000	0.25	1	0.96	1.00	0.05
video	none	12000	0.25	1	0.76	0.95	0.05
game	bonferroni	12000	0.25	1	0.28	0.43	0.01
sms	bonferroni	12000	0.25	1	0.91	1.00	0.01
video	bonferroni	12000	0.25	1	0.61	0.88	0.02
game	holm	12000	0.25	1	0.42	0.61	0.01
sms	holm	12000	0.25	1	0.91	1.00	0.02
video	holm	12000	0.25	1	0.68	0.94	0.02
game	hochberg	12000	0.25	1	0.44	0.61	0.01
sms	hochberg	12000	0.25	1	0.91	1.00	0.02
video	hochberg	12000	0.25	1	0.70	0.94	0.02
game	none	12000	0.25	2	0.55	0.61	0.04
sms	none	12000	0.25	2	0.99	1.00	0.06
video	none	12000	0.25	2	0.91	0.95	0.05
game	bonferroni	12000	0.25	2	0.34	0.40	0.01
sms	bonferroni	12000	0.25	2	0.97	1.00	0.01
video	bonferroni	12000	0.25	2	0.76	0.88	0.01
game	holm	12000	0.25	2	0.54	0.60	0.01
sms	holm	12000	0.25	2	0.98	1.00	0.01
video	holm	12000	0.25	2	0.83	0.91	0.01
game	hochberg	12000	0.25	2	0.55	0.60	0.01
sms	hochberg	12000	0.25	2	0.98	1.00	0.02
video	hochberg	12000	0.25	2	0.84	0.91	0.02
game	none	12000	0.25	3	0.56	0.75	0.03
sms	none	12000	0.25	3	0.99	1.00	0.11
video	none	12000	0.25	3	0.90	0.96	0.07
game	bonferroni	12000	0.25	3	0.40	0.54	0.01
sms	bonferroni	12000	0.25	3	0.98	1.00	0.02
video	bonferroni	12000	0.25	3	0.79	0.89	0.03
game	holm	12000	0.25	3	0.53	0.75	0.02
sms	holm	12000	0.25	3	0.98	1.00	0.02
video	holm	12000	0.25	3	0.82	0.93	0.04
game	hochberg	12000	0.25	3	0.54	0.75	0.02
sms	hochberg	12000	0.25	3	0.98	1.00	0.02
video	hochberg	12000	0.25	3	0.83	0.93	0.04
game	none	12000	0.25	4	0.64	0.65	0.05
sms	none	12000	0.25	4	1.00	1.00	0.03
video	none	12000	0.25	4	0.96	0.97	0.03
game	bonferroni	12000	0.25	4	0.50	0.44	0.00
sms	bonferroni	12000	0.25	4	1.00	1.00	0.01
video	bonferroni	12000	0.25	4	0.88	0.94	0.00
game	holm	12000	0.25	4	0.64	0.64	0.00
sms	holm	12000	0.25	4	1.00	1.00	0.01
video	holm	12000	0.25	4	0.92	0.96	0.00
game	hochberg	12000	0.25	4	0.64	0.65	0.00
sms	hochberg	12000	0.25	4	1.00	1.00	0.01
video	hochberg	12000	0.25	4	0.92	0.97	0.00
game	none	12000	0.30	1	0.50	0.49	0.02
sms	none	12000	0.30	1	0.99	1.00	0.03
video	none	12000	0.30	1	0.76	0.89	0.04
game	bonferroni	12000	0.30	1	0.30	0.34	0.01
sms	bonferroni	12000	0.30	1	0.92	0.99	0.00
video	bonferroni	12000	0.30	1	0.59	0.74	0.00
game	holm	12000	0.30	1	0.45	0.47	0.01
sms	holm	12000	0.30	1	0.92	0.99	0.00
video	holm	12000	0.30	1	0.68	0.84	0.00
game	hochberg	12000	0.30	1	0.49	0.49	0.01
sms	hochberg	12000	0.30	1	0.94	0.99	0.00
video	hochberg	12000	0.30	1	0.72	0.86	0.00
game	none	12000	0.30	2	0.52	0.70	0.02
sms	none	12000	0.30	2	0.99	1.00	0.06
video	none	12000	0.30	2	0.91	0.96	0.05
game	bonferroni	12000	0.30	2	0.30	0.50	0.00
sms	bonferroni	12000	0.30	2	0.99	1.00	0.00
video	bonferroni	12000	0.30	2	0.80	0.89	0.00
game	holm	12000	0.30	2	0.51	0.69	0.00
sms	holm	12000	0.30	2	0.99	1.00	0.00
video	holm	12000	0.30	2	0.86	0.90	0.00
game	hochberg	12000	0.30	2	0.51	0.70	0.00
sms	hochberg	12000	0.30	2	0.99	1.00	0.00
video	hochberg	12000	0.30	2	0.86	0.91	0.00
game	none	12000	0.30	3	0.57	0.65	0.06
sms	none	12000	0.30	3	1.00	1.00	0.05
video	none	12000	0.30	3	0.90	0.96	0.04
game	bonferroni	12000	0.30	3	0.39	0.42	0.03
sms	bonferroni	12000	0.30	3	0.98	1.00	0.02
video	bonferroni	12000	0.30	3	0.84	0.92	0.02
game	holm	12000	0.30	3	0.56	0.64	0.04
sms	holm	12000	0.30	3	0.98	1.00	0.02
video	holm	12000	0.30	3	0.86	0.94	0.02
game	hochberg	12000	0.30	3	0.57	0.65	0.04
sms	hochberg	12000	0.30	3	0.98	1.00	0.02
video	hochberg	12000	0.30	3	0.87	0.95	0.02
game	none	12000	0.30	4	0.60	0.75	0.06
sms	none	12000	0.30	4	1.00	1.00	0.01
video	none	12000	0.30	4	0.93	0.97	0.10
game	bonferroni	12000	0.30	4	0.47	0.61	0.02
sms	bonferroni	12000	0.30	4	1.00	1.00	0.00
video	bonferroni	12000	0.30	4	0.86	0.92	0.01
game	holm	12000	0.30	4	0.60	0.75	0.02
sms	holm	12000	0.30	4	1.00	1.00	0.01
video	holm	12000	0.30	4	0.89	0.94	0.01
game	hochberg	12000	0.30	4	0.60	0.75	0.02
sms	hochberg	12000	0.30	4	1.00	1.00	0.01
video	hochberg	12000	0.30	4	0.89	0.94	0.01
game	none	12000	0.40	1	0.53	0.65	0.06
sms	none	12000	0.40	1	1.00	1.00	0.08
video	none	12000	0.40	1	0.83	0.92	0.08
game	bonferroni	12000	0.40	1	0.33	0.47	0.02
sms	bonferroni	12000	0.40	1	0.97	1.00	0.03
video	bonferroni	12000	0.40	1	0.74	0.84	0.02
game	holm	12000	0.40	1	0.49	0.64	0.02
sms	holm	12000	0.40	1	0.97	1.00	0.03
video	holm	12000	0.40	1	0.78	0.89	0.02
game	hochberg	12000	0.40	1	0.50	0.64	0.02
sms	hochberg	12000	0.40	1	0.98	1.00	0.03
video	hochberg	12000	0.40	1	0.78	0.89	0.02
game	none	12000	0.40	2	0.54	0.68	0.03
sms	none	12000	0.40	2	0.98	1.00	0.03
video	none	12000	0.40	2	0.86	0.96	0.03
game	bonferroni	12000	0.40	2	0.33	0.50	0.01
sms	bonferroni	12000	0.40	2	0.94	1.00	0.01
video	bonferroni	12000	0.40	2	0.75	0.91	0.01
game	holm	12000	0.40	2	0.50	0.67	0.01
sms	holm	12000	0.40	2	0.94	1.00	0.01
video	holm	12000	0.40	2	0.78	0.94	0.01
game	hochberg	12000	0.40	2	0.54	0.67	0.01
sms	hochberg	12000	0.40	2	0.96	1.00	0.01
video	hochberg	12000	0.40	2	0.82	0.94	0.01
game	none	12000	0.40	3	0.59	0.79	0.04
sms	none	12000	0.40	3	1.00	1.00	0.04
video	none	12000	0.40	3	0.91	0.97	0.01
game	bonferroni	12000	0.40	3	0.40	0.61	0.03
sms	bonferroni	12000	0.40	3	1.00	1.00	0.02
video	bonferroni	12000	0.40	3	0.83	0.89	0.00
game	holm	12000	0.40	3	0.58	0.79	0.03
sms	holm	12000	0.40	3	1.00	1.00	0.02
video	holm	12000	0.40	3	0.88	0.97	0.00
game	hochberg	12000	0.40	3	0.58	0.79	0.03
sms	hochberg	12000	0.40	3	1.00	1.00	0.02
video	hochberg	12000	0.40	3	0.88	0.97	0.00
game	none	12000	0.40	4	0.59	0.69	0.07
sms	none	12000	0.40	4	1.00	1.00	0.03
video	none	12000	0.40	4	1.00	0.99	0.05
game	bonferroni	12000	0.40	4	0.43	0.49	0.03
sms	bonferroni	12000	0.40	4	0.99	1.00	0.02
video	bonferroni	12000	0.40	4	0.92	0.97	0.01
game	holm	12000	0.40	4	0.59	0.68	0.04
sms	holm	12000	0.40	4	0.99	1.00	0.02
video	holm	12000	0.40	4	0.94	0.97	0.01
game	hochberg	12000	0.40	4	0.59	0.68	0.04
sms	hochberg	12000	0.40	4	0.99	1.00	0.02
video	hochberg	12000	0.40	4	0.94	0.97	0.01
game	none	15000	0.25	1	0.57	0.66	0.03
sms	none	15000	0.25	1	0.99	1.00	0.05
video	none	15000	0.25	1	0.94	0.98	0.07
game	bonferroni	15000	0.25	1	0.37	0.48	0.02
sms	bonferroni	15000	0.25	1	0.99	0.99	0.01
video	bonferroni	15000	0.25	1	0.85	0.94	0.03
game	holm	15000	0.25	1	0.55	0.66	0.02
sms	holm	15000	0.25	1	0.99	0.99	0.01
video	holm	15000	0.25	1	0.87	0.95	0.03
game	hochberg	15000	0.25	1	0.56	0.66	0.02
sms	hochberg	15000	0.25	1	0.99	0.99	0.01
video	hochberg	15000	0.25	1	0.88	0.95	0.03
game	none	15000	0.25	2	0.58	0.71	0.04
sms	none	15000	0.25	2	1.00	1.00	0.06
video	none	15000	0.25	2	0.89	0.95	0.08
game	bonferroni	15000	0.25	2	0.43	0.55	0.02
sms	bonferroni	15000	0.25	2	1.00	1.00	0.03
video	bonferroni	15000	0.25	2	0.81	0.91	0.01
game	holm	15000	0.25	2	0.58	0.71	0.04
sms	holm	15000	0.25	2	1.00	1.00	0.03
video	holm	15000	0.25	2	0.88	0.94	0.03
game	hochberg	15000	0.25	2	0.58	0.71	0.04
sms	hochberg	15000	0.25	2	1.00	1.00	0.03
video	hochberg	15000	0.25	2	0.88	0.94	0.03
game	none	15000	0.25	3	0.66	0.73	0.04
sms	none	15000	0.25	3	1.00	1.00	0.05
video	none	15000	0.25	3	0.98	0.99	0.06
game	bonferroni	15000	0.25	3	0.48	0.58	0.02
sms	bonferroni	15000	0.25	3	1.00	1.00	0.03
video	bonferroni	15000	0.25	3	0.91	0.95	0.02
game	holm	15000	0.25	3	0.66	0.72	0.03
sms	holm	15000	0.25	3	1.00	1.00	0.03
video	holm	15000	0.25	3	0.94	0.97	0.02
game	hochberg	15000	0.25	3	0.66	0.73	0.03
sms	hochberg	15000	0.25	3	1.00	1.00	0.03
video	hochberg	15000	0.25	3	0.94	0.98	0.02
game	none	15000	0.25	4	0.65	0.75	0.01
sms	none	15000	0.25	4	1.00	1.00	0.05
video	none	15000	0.25	4	0.97	0.96	0.04
game	bonferroni	15000	0.25	4	0.44	0.60	0.01
sms	bonferroni	15000	0.25	4	1.00	1.00	0.02
video	bonferroni	15000	0.25	4	0.93	0.91	0.02
game	holm	15000	0.25	4	0.65	0.75	0.01
sms	holm	15000	0.25	4	1.00	1.00	0.03
video	holm	15000	0.25	4	0.95	0.94	0.02
game	hochberg	15000	0.25	4	0.65	0.75	0.01
sms	hochberg	15000	0.25	4	1.00	1.00	0.03
video	hochberg	15000	0.25	4	0.95	0.94	0.02
game	none	15000	0.30	1	0.54	0.57	0.02
sms	none	15000	0.30	1	0.98	1.00	0.09
video	none	15000	0.30	1	0.90	0.97	0.03
game	bonferroni	15000	0.30	1	0.38	0.50	0.00
sms	bonferroni	15000	0.30	1	0.95	1.00	0.02
video	bonferroni	15000	0.30	1	0.79	0.90	0.01
game	holm	15000	0.30	1	0.52	0.57	0.00
sms	holm	15000	0.30	1	0.96	1.00	0.02
video	holm	15000	0.30	1	0.85	0.94	0.01
game	hochberg	15000	0.30	1	0.53	0.57	0.00
sms	hochberg	15000	0.30	1	0.96	1.00	0.02
video	hochberg	15000	0.30	1	0.86	0.94	0.01
game	none	15000	0.30	2	0.63	0.73	0.05
sms	none	15000	0.30	2	0.99	1.00	0.07
video	none	15000	0.30	2	0.91	0.98	0.04
game	bonferroni	15000	0.30	2	0.41	0.55	0.01
sms	bonferroni	15000	0.30	2	0.99	1.00	0.02
video	bonferroni	15000	0.30	2	0.83	0.98	0.01
game	holm	15000	0.30	2	0.61	0.73	0.01
sms	holm	15000	0.30	2	0.99	1.00	0.02
video	holm	15000	0.30	2	0.86	0.98	0.01
game	hochberg	15000	0.30	2	0.62	0.73	0.02
sms	hochberg	15000	0.30	2	0.99	1.00	0.03
video	hochberg	15000	0.30	2	0.87	0.98	0.01
game	none	15000	0.30	3	0.62	0.84	0.06
sms	none	15000	0.30	3	1.00	1.00	0.05
video	none	15000	0.30	3	0.96	0.99	0.08
game	bonferroni	15000	0.30	3	0.52	0.67	0.01
sms	bonferroni	15000	0.30	3	0.99	1.00	0.01
video	bonferroni	15000	0.30	3	0.91	0.98	0.04
game	holm	15000	0.30	3	0.62	0.83	0.03
sms	holm	15000	0.30	3	0.99	1.00	0.01
video	holm	15000	0.30	3	0.94	0.98	0.04
game	hochberg	15000	0.30	3	0.62	0.84	0.04
sms	hochberg	15000	0.30	3	0.99	1.00	0.02
video	hochberg	15000	0.30	3	0.94	0.99	0.04
game	none	15000	0.30	4	0.76	0.77	0.06
sms	none	15000	0.30	4	1.00	1.00	0.04
video	none	15000	0.30	4	0.99	0.99	0.08
game	bonferroni	15000	0.30	4	0.62	0.61	0.00
sms	bonferroni	15000	0.30	4	1.00	1.00	0.01
video	bonferroni	15000	0.30	4	0.94	0.94	0.03
game	holm	15000	0.30	4	0.76	0.77	0.00
sms	holm	15000	0.30	4	1.00	1.00	0.01
video	holm	15000	0.30	4	0.97	0.98	0.03
game	hochberg	15000	0.30	4	0.76	0.77	0.00
sms	hochberg	15000	0.30	4	1.00	1.00	0.02
video	hochberg	15000	0.30	4	0.97	0.98	0.04
game	none	15000	0.40	1	0.54	0.66	0.06
sms	none	15000	0.40	1	0.99	1.00	0.06
video	none	15000	0.40	1	0.90	0.97	0.05
game	bonferroni	15000	0.40	1	0.39	0.48	0.01
sms	bonferroni	15000	0.40	1	0.99	1.00	0.02
video	bonferroni	15000	0.40	1	0.78	0.93	0.01
game	holm	15000	0.40	1	0.51	0.66	0.01
sms	holm	15000	0.40	1	0.99	1.00	0.02
video	holm	15000	0.40	1	0.86	0.95	0.01
game	hochberg	15000	0.40	1	0.51	0.66	0.01
sms	hochberg	15000	0.40	1	0.99	1.00	0.02
video	hochberg	15000	0.40	1	0.86	0.95	0.01
game	none	15000	0.40	2	0.55	0.76	0.05
sms	none	15000	0.40	2	1.00	1.00	0.05
video	none	15000	0.40	2	0.93	0.98	0.06
game	bonferroni	15000	0.40	2	0.34	0.61	0.03
sms	bonferroni	15000	0.40	2	1.00	1.00	0.01
video	bonferroni	15000	0.40	2	0.82	0.96	0.04
game	holm	15000	0.40	2	0.53	0.76	0.03
sms	holm	15000	0.40	2	1.00	1.00	0.02
video	holm	15000	0.40	2	0.85	0.98	0.04
game	hochberg	15000	0.40	2	0.55	0.76	0.03
sms	hochberg	15000	0.40	2	1.00	1.00	0.02
video	hochberg	15000	0.40	2	0.87	0.98	0.04
game	none	15000	0.40	3	0.59	0.84	0.02
sms	none	15000	0.40	3	1.00	1.00	0.04
video	none	15000	0.40	3	0.96	1.00	0.02
game	bonferroni	15000	0.40	3	0.42	0.69	0.00
sms	bonferroni	15000	0.40	3	0.99	1.00	0.03
video	bonferroni	15000	0.40	3	0.88	0.99	0.00
game	holm	15000	0.40	3	0.59	0.84	0.00
sms	holm	15000	0.40	3	1.00	1.00	0.03
video	holm	15000	0.40	3	0.94	1.00	0.00
game	hochberg	15000	0.40	3	0.59	0.84	0.00
sms	hochberg	15000	0.40	3	1.00	1.00	0.03
video	hochberg	15000	0.40	3	0.94	1.00	0.00
game	none	15000	0.40	4	0.70	0.84	0.07
sms	none	15000	0.40	4	1.00	1.00	0.05
video	none	15000	0.40	4	0.94	0.97	0.05
game	bonferroni	15000	0.40	4	0.54	0.71	0.02
sms	bonferroni	15000	0.40	4	1.00	1.00	0.02
video	bonferroni	15000	0.40	4	0.91	0.96	0.01
game	holm	15000	0.40	4	0.69	0.84	0.02
sms	holm	15000	0.40	4	1.00	1.00	0.02
video	holm	15000	0.40	4	0.94	0.97	0.01
game	hochberg	15000	0.40	4	0.69	0.84	0.02
sms	hochberg	15000	0.40	4	1.00	1.00	0.02
video	hochberg	15000	0.40	4	0.94	0.97	0.01
game	none	20000	0.25	1	0.66	0.75	0.04
sms	none	20000	0.25	1	1.00	1.00	0.06
video	none	20000	0.25	1	0.98	0.97	0.10
game	bonferroni	20000	0.25	1	0.43	0.54	0.02
sms	bonferroni	20000	0.25	1	0.99	1.00	0.05
video	bonferroni	20000	0.25	1	0.85	0.96	0.03
game	holm	20000	0.25	1	0.66	0.74	0.02
sms	holm	20000	0.25	1	0.99	1.00	0.05
video	holm	20000	0.25	1	0.94	0.97	0.04
game	hochberg	20000	0.25	1	0.66	0.74	0.02
sms	hochberg	20000	0.25	1	0.99	1.00	0.05
video	hochberg	20000	0.25	1	0.94	0.97	0.04
game	none	20000	0.25	2	0.73	0.85	0.04
sms	none	20000	0.25	2	1.00	1.00	0.09
video	none	20000	0.25	2	0.99	1.00	0.04
game	bonferroni	20000	0.25	2	0.52	0.73	0.02
sms	bonferroni	20000	0.25	2	1.00	1.00	0.02
video	bonferroni	20000	0.25	2	0.94	0.99	0.01
game	holm	20000	0.25	2	0.72	0.85	0.02
sms	holm	20000	0.25	2	1.00	1.00	0.02
video	holm	20000	0.25	2	0.99	1.00	0.01
game	hochberg	20000	0.25	2	0.72	0.85	0.02
sms	hochberg	20000	0.25	2	1.00	1.00	0.02
video	hochberg	20000	0.25	2	0.99	1.00	0.01
game	none	20000	0.25	3	0.73	0.87	0.02
sms	none	20000	0.25	3	1.00	1.00	0.01
video	none	20000	0.25	3	0.98	1.00	0.03
game	bonferroni	20000	0.25	3	0.53	0.75	0.01
sms	bonferroni	20000	0.25	3	1.00	1.00	0.01
video	bonferroni	20000	0.25	3	0.93	0.99	0.01
game	holm	20000	0.25	3	0.72	0.87	0.01
sms	holm	20000	0.25	3	1.00	1.00	0.01
video	holm	20000	0.25	3	0.98	1.00	0.01
game	hochberg	20000	0.25	3	0.72	0.87	0.01
sms	hochberg	20000	0.25	3	1.00	1.00	0.01
video	hochberg	20000	0.25	3	0.98	1.00	0.01
game	none	20000	0.25	4	0.72	0.93	0.04
sms	none	20000	0.25	4	1.00	1.00	0.07
video	none	20000	0.25	4	1.00	1.00	0.07
game	bonferroni	20000	0.25	4	0.59	0.78	0.00
sms	bonferroni	20000	0.25	4	1.00	1.00	0.01
video	bonferroni	20000	0.25	4	0.99	1.00	0.01
game	holm	20000	0.25	4	0.72	0.93	0.00
sms	holm	20000	0.25	4	1.00	1.00	0.01
video	holm	20000	0.25	4	1.00	1.00	0.01
game	hochberg	20000	0.25	4	0.72	0.93	0.01
sms	hochberg	20000	0.25	4	1.00	1.00	0.02
video	hochberg	20000	0.25	4	1.00	1.00	0.02
game	none	20000	0.30	1	0.69	0.75	0.08
sms	none	20000	0.30	1	1.00	1.00	0.12
video	none	20000	0.30	1	0.93	1.00	0.04
game	bonferroni	20000	0.30	1	0.45	0.61	0.02
sms	bonferroni	20000	0.30	1	1.00	1.00	0.04
video	bonferroni	20000	0.30	1	0.87	0.98	0.01
game	holm	20000	0.30	1	0.68	0.75	0.03
sms	holm	20000	0.30	1	1.00	1.00	0.05
video	holm	20000	0.30	1	0.92	1.00	0.02
game	hochberg	20000	0.30	1	0.68	0.75	0.03
sms	hochberg	20000	0.30	1	1.00	1.00	0.05
video	hochberg	20000	0.30	1	0.92	1.00	0.02
game	none	20000	0.30	2	0.81	0.92	0.03
sms	none	20000	0.30	2	1.00	1.00	0.05
video	none	20000	0.30	2	0.98	0.99	0.09
game	bonferroni	20000	0.30	2	0.55	0.77	0.02
sms	bonferroni	20000	0.30	2	1.00	1.00	0.03
video	bonferroni	20000	0.30	2	0.95	0.98	0.02
game	holm	20000	0.30	2	0.80	0.91	0.03
sms	holm	20000	0.30	2	1.00	1.00	0.03
video	holm	20000	0.30	2	0.97	0.99	0.04
game	hochberg	20000	0.30	2	0.81	0.91	0.03
sms	hochberg	20000	0.30	2	1.00	1.00	0.03
video	hochberg	20000	0.30	2	0.98	0.99	0.04
game	none	20000	0.30	3	0.82	0.89	0.08
sms	none	20000	0.30	3	1.00	1.00	0.07
video	none	20000	0.30	3	0.99	1.00	0.09
game	bonferroni	20000	0.30	3	0.71	0.82	0.03
sms	bonferroni	20000	0.30	3	1.00	1.00	0.03
video	bonferroni	20000	0.30	3	0.97	1.00	0.04
game	holm	20000	0.30	3	0.82	0.89	0.03
sms	holm	20000	0.30	3	1.00	1.00	0.03
video	holm	20000	0.30	3	0.99	1.00	0.05
game	hochberg	20000	0.30	3	0.82	0.89	0.03
sms	hochberg	20000	0.30	3	1.00	1.00	0.03
video	hochberg	20000	0.30	3	0.99	1.00	0.05
game	none	20000	0.30	4	0.75	0.90	0.08
sms	none	20000	0.30	4	1.00	1.00	0.01
video	none	20000	0.30	4	0.97	1.00	0.06
game	bonferroni	20000	0.30	4	0.62	0.82	0.02
sms	bonferroni	20000	0.30	4	1.00	1.00	0.01
video	bonferroni	20000	0.30	4	0.95	1.00	0.02
game	holm	20000	0.30	4	0.75	0.90	0.02
sms	holm	20000	0.30	4	1.00	1.00	0.01
video	holm	20000	0.30	4	0.96	1.00	0.02
game	hochberg	20000	0.30	4	0.75	0.90	0.02
sms	hochberg	20000	0.30	4	1.00	1.00	0.01
video	hochberg	20000	0.30	4	0.96	1.00	0.02
game	none	20000	0.40	1	0.59	0.87	0.04
sms	none	20000	0.40	1	1.00	1.00	0.03
video	none	20000	0.40	1	0.91	1.00	0.04
game	bonferroni	20000	0.40	1	0.35	0.73	0.01
sms	bonferroni	20000	0.40	1	1.00	1.00	0.01
video	bonferroni	20000	0.40	1	0.80	0.97	0.00
game	holm	20000	0.40	1	0.55	0.87	0.01
sms	holm	20000	0.40	1	1.00	1.00	0.01
video	holm	20000	0.40	1	0.86	0.97	0.00
game	hochberg	20000	0.40	1	0.56	0.87	0.01
sms	hochberg	20000	0.40	1	1.00	1.00	0.01
video	hochberg	20000	0.40	1	0.87	0.97	0.00
game	none	20000	0.40	2	0.74	0.79	0.04
sms	none	20000	0.40	2	1.00	1.00	0.03
video	none	20000	0.40	2	0.94	1.00	0.05
game	bonferroni	20000	0.40	2	0.59	0.74	0.02
sms	bonferroni	20000	0.40	2	1.00	1.00	0.02
video	bonferroni	20000	0.40	2	0.90	0.99	0.04
game	holm	20000	0.40	2	0.73	0.79	0.02
sms	holm	20000	0.40	2	1.00	1.00	0.02
video	holm	20000	0.40	2	0.93	1.00	0.04
game	hochberg	20000	0.40	2	0.73	0.79	0.02
sms	hochberg	20000	0.40	2	1.00	1.00	0.02
video	hochberg	20000	0.40	2	0.93	1.00	0.04
game	none	20000	0.40	3	0.79	0.90	0.11
sms	none	20000	0.40	3	1.00	1.00	0.04
video	none	20000	0.40	3	0.99	1.00	0.08
game	bonferroni	20000	0.40	3	0.66	0.71	0.09
sms	bonferroni	20000	0.40	3	1.00	1.00	0.02
video	bonferroni	20000	0.40	3	0.97	0.99	0.03
game	holm	20000	0.40	3	0.79	0.89	0.09
sms	holm	20000	0.40	3	1.00	1.00	0.02
video	holm	20000	0.40	3	0.99	0.99	0.03
game	hochberg	20000	0.40	3	0.79	0.90	0.09
sms	hochberg	20000	0.40	3	1.00	1.00	0.02
video	hochberg	20000	0.40	3	0.99	1.00	0.03
game	none	20000	0.40	4	0.81	0.88	0.07
sms	none	20000	0.40	4	1.00	1.00	0.08
video	none	20000	0.40	4	0.97	1.00	0.05
game	bonferroni	20000	0.40	4	0.68	0.77	0.03
sms	bonferroni	20000	0.40	4	1.00	1.00	0.04
video	bonferroni	20000	0.40	4	0.96	0.99	0.01
game	holm	20000	0.40	4	0.81	0.87	0.03
sms	holm	20000	0.40	4	1.00	1.00	0.04
video	holm	20000	0.40	4	0.97	0.99	0.01
game	hochberg	20000	0.40	4	0.81	0.88	0.03
sms	hochberg	20000	0.40	4	1.00	1.00	0.04
video	hochberg	20000	0.40	4	0.97	1.00	0.01
game	none	22000	0.25	1	0.69	0.82	0.06
sms	none	22000	0.25	1	1.00	1.00	0.05
video	none	22000	0.25	1	0.95	0.99	0.05
game	bonferroni	22000	0.25	1	0.47	0.71	0.04
sms	bonferroni	22000	0.25	1	1.00	1.00	0.02
video	bonferroni	22000	0.25	1	0.89	0.97	0.02
game	holm	22000	0.25	1	0.69	0.82	0.04
sms	holm	22000	0.25	1	1.00	1.00	0.03
video	holm	22000	0.25	1	0.92	0.98	0.03
game	hochberg	22000	0.25	1	0.69	0.82	0.04
sms	hochberg	22000	0.25	1	1.00	1.00	0.03
video	hochberg	22000	0.25	1	0.92	0.98	0.03
game	none	22000	0.25	2	0.76	0.90	0.03
sms	none	22000	0.25	2	1.00	1.00	0.07
video	none	22000	0.25	2	0.98	1.00	0.06
game	bonferroni	22000	0.25	2	0.64	0.78	0.00
sms	bonferroni	22000	0.25	2	1.00	1.00	0.04
video	bonferroni	22000	0.25	2	0.95	0.99	0.00
game	holm	22000	0.25	2	0.75	0.90	0.00
sms	holm	22000	0.25	2	1.00	1.00	0.04
video	holm	22000	0.25	2	0.98	1.00	0.00
game	hochberg	22000	0.25	2	0.75	0.90	0.01
sms	hochberg	22000	0.25	2	1.00	1.00	0.04
video	hochberg	22000	0.25	2	0.98	1.00	0.01
game	none	22000	0.25	3	0.80	0.92	0.08
sms	none	22000	0.25	3	1.00	1.00	0.05
video	none	22000	0.25	3	1.00	0.99	0.06
game	bonferroni	22000	0.25	3	0.66	0.82	0.02
sms	bonferroni	22000	0.25	3	1.00	1.00	0.03
video	bonferroni	22000	0.25	3	0.98	0.99	0.00
game	holm	22000	0.25	3	0.80	0.92	0.02
sms	holm	22000	0.25	3	1.00	1.00	0.03
video	holm	22000	0.25	3	1.00	0.99	0.00
game	hochberg	22000	0.25	3	0.80	0.92	0.02
sms	hochberg	22000	0.25	3	1.00	1.00	0.04
video	hochberg	22000	0.25	3	1.00	0.99	0.01
game	none	22000	0.25	4	0.81	0.88	0.04
sms	none	22000	0.25	4	1.00	1.00	0.04
video	none	22000	0.25	4	0.98	1.00	0.03
game	bonferroni	22000	0.25	4	0.64	0.83	0.03
sms	bonferroni	22000	0.25	4	1.00	1.00	0.01
video	bonferroni	22000	0.25	4	0.94	0.99	0.01
game	holm	22000	0.25	4	0.80	0.88	0.03
sms	holm	22000	0.25	4	1.00	1.00	0.01
video	holm	22000	0.25	4	0.98	0.99	0.01
game	hochberg	22000	0.25	4	0.80	0.88	0.03
sms	hochberg	22000	0.25	4	1.00	1.00	0.02
video	hochberg	22000	0.25	4	0.98	0.99	0.02
game	none	22000	0.30	1	0.74	0.83	0.01
sms	none	22000	0.30	1	1.00	1.00	0.02
video	none	22000	0.30	1	0.97	0.99	0.04
game	bonferroni	22000	0.30	1	0.51	0.70	0.00
sms	bonferroni	22000	0.30	1	1.00	1.00	0.01
video	bonferroni	22000	0.30	1	0.90	0.96	0.02
game	holm	22000	0.30	1	0.73	0.83	0.00
sms	holm	22000	0.30	1	1.00	1.00	0.01
video	holm	22000	0.30	1	0.96	0.98	0.02
game	hochberg	22000	0.30	1	0.73	0.83	0.00
sms	hochberg	22000	0.30	1	1.00	1.00	0.01
video	hochberg	22000	0.30	1	0.96	0.98	0.02
game	none	22000	0.30	2	0.80	0.88	0.05
sms	none	22000	0.30	2	1.00	1.00	0.05
video	none	22000	0.30	2	1.00	1.00	0.02
game	bonferroni	22000	0.30	2	0.58	0.79	0.02
sms	bonferroni	22000	0.30	2	1.00	1.00	0.02
video	bonferroni	22000	0.30	2	0.95	0.99	0.00
game	holm	22000	0.30	2	0.79	0.88	0.02
sms	holm	22000	0.30	2	1.00	1.00	0.02
video	holm	22000	0.30	2	0.98	1.00	0.00
game	hochberg	22000	0.30	2	0.80	0.88	0.02
sms	hochberg	22000	0.30	2	1.00	1.00	0.02
video	hochberg	22000	0.30	2	0.99	1.00	0.00
game	none	22000	0.30	3	0.85	0.92	0.03
sms	none	22000	0.30	3	1.00	1.00	0.07
video	none	22000	0.30	3	0.99	1.00	0.03
game	bonferroni	22000	0.30	3	0.67	0.79	0.01
sms	bonferroni	22000	0.30	3	1.00	1.00	0.03
video	bonferroni	22000	0.30	3	0.97	0.99	0.01
game	holm	22000	0.30	3	0.84	0.92	0.01
sms	holm	22000	0.30	3	1.00	1.00	0.03
video	holm	22000	0.30	3	0.98	1.00	0.01
game	hochberg	22000	0.30	3	0.84	0.92	0.02
sms	hochberg	22000	0.30	3	1.00	1.00	0.04
video	hochberg	22000	0.30	3	0.98	1.00	0.01
game	none	22000	0.30	4	0.79	0.95	0.03
sms	none	22000	0.30	4	1.00	1.00	0.02
video	none	22000	0.30	4	0.99	1.00	0.02
game	bonferroni	22000	0.30	4	0.66	0.86	0.01
sms	bonferroni	22000	0.30	4	1.00	1.00	0.00
video	bonferroni	22000	0.30	4	0.97	1.00	0.00
game	holm	22000	0.30	4	0.78	0.95	0.01
sms	holm	22000	0.30	4	1.00	1.00	0.00
video	holm	22000	0.30	4	0.99	1.00	0.00
game	hochberg	22000	0.30	4	0.78	0.95	0.01
sms	hochberg	22000	0.30	4	1.00	1.00	0.00
video	hochberg	22000	0.30	4	0.99	1.00	0.00
game	none	22000	0.40	1	0.68	0.86	0.04
sms	none	22000	0.40	1	1.00	1.00	0.07
video	none	22000	0.40	1	0.95	0.99	0.06
game	bonferroni	22000	0.40	1	0.47	0.74	0.03
sms	bonferroni	22000	0.40	1	1.00	1.00	0.02
video	bonferroni	22000	0.40	1	0.87	0.99	0.02
game	holm	22000	0.40	1	0.68	0.86	0.03
sms	holm	22000	0.40	1	1.00	1.00	0.03
video	holm	22000	0.40	1	0.93	0.99	0.03
game	hochberg	22000	0.40	1	0.68	0.86	0.03
sms	hochberg	22000	0.40	1	1.00	1.00	0.03
video	hochberg	22000	0.40	1	0.93	0.99	0.03
game	none	22000	0.40	2	0.77	0.89	0.02
sms	none	22000	0.40	2	1.00	1.00	0.05
video	none	22000	0.40	2	0.98	1.00	0.06
game	bonferroni	22000	0.40	2	0.65	0.77	0.01
sms	bonferroni	22000	0.40	2	0.99	1.00	0.02
video	bonferroni	22000	0.40	2	0.96	1.00	0.02
game	holm	22000	0.40	2	0.77	0.89	0.01
sms	holm	22000	0.40	2	0.99	1.00	0.02
video	holm	22000	0.40	2	0.97	1.00	0.02
game	hochberg	22000	0.40	2	0.77	0.89	0.01
sms	hochberg	22000	0.40	2	0.99	1.00	0.02
video	hochberg	22000	0.40	2	0.97	1.00	0.02
game	none	22000	0.40	3	0.83	0.93	0.04
sms	none	22000	0.40	3	1.00	1.00	0.06
video	none	22000	0.40	3	1.00	1.00	0.03
game	bonferroni	22000	0.40	3	0.63	0.81	0.02
sms	bonferroni	22000	0.40	3	1.00	1.00	0.02
video	bonferroni	22000	0.40	3	0.99	1.00	0.01
game	holm	22000	0.40	3	0.83	0.93	0.02
sms	holm	22000	0.40	3	1.00	1.00	0.02
video	holm	22000	0.40	3	1.00	1.00	0.01
game	hochberg	22000	0.40	3	0.83	0.93	0.02
sms	hochberg	22000	0.40	3	1.00	1.00	0.02
video	hochberg	22000	0.40	3	1.00	1.00	0.01
game	none	22000	0.40	4	0.88	0.95	0.06
sms	none	22000	0.40	4	1.00	1.00	0.04
video	none	22000	0.40	4	0.99	1.00	0.02
game	bonferroni	22000	0.40	4	0.77	0.83	0.01
sms	bonferroni	22000	0.40	4	1.00	1.00	0.02
video	bonferroni	22000	0.40	4	0.98	1.00	0.00
game	holm	22000	0.40	4	0.88	0.95	0.01
sms	holm	22000	0.40	4	1.00	1.00	0.02
video	holm	22000	0.40	4	0.99	1.00	0.00
game	hochberg	22000	0.40	4	0.88	0.95	0.01
sms	hochberg	22000	0.40	4	1.00	1.00	0.02
video	hochberg	22000	0.40	4	0.99	1.00	0.00
game	none	25000	0.25	1	0.64	0.86	0.05
sms	none	25000	0.25	1	1.00	1.00	0.06
video	none	25000	0.25	1	0.97	1.00	0.04
game	bonferroni	25000	0.25	1	0.52	0.69	0.02
sms	bonferroni	25000	0.25	1	1.00	1.00	0.02
video	bonferroni	25000	0.25	1	0.96	0.99	0.02
game	holm	25000	0.25	1	0.64	0.86	0.02
sms	holm	25000	0.25	1	1.00	1.00	0.02
video	holm	25000	0.25	1	0.96	0.99	0.02
game	hochberg	25000	0.25	1	0.64	0.86	0.02
sms	hochberg	25000	0.25	1	1.00	1.00	0.02
video	hochberg	25000	0.25	1	0.96	0.99	0.02
game	none	25000	0.25	2	0.78	0.89	0.03
sms	none	25000	0.25	2	0.99	1.00	0.02
video	none	25000	0.25	2	1.00	1.00	0.04
game	bonferroni	25000	0.25	2	0.63	0.82	0.02
sms	bonferroni	25000	0.25	2	0.99	1.00	0.01
video	bonferroni	25000	0.25	2	0.92	0.99	0.01
game	holm	25000	0.25	2	0.78	0.89	0.02
sms	holm	25000	0.25	2	0.99	1.00	0.01
video	holm	25000	0.25	2	0.95	1.00	0.01
game	hochberg	25000	0.25	2	0.78	0.89	0.02
sms	hochberg	25000	0.25	2	0.99	1.00	0.01
video	hochberg	25000	0.25	2	0.95	1.00	0.01
game	none	25000	0.25	3	0.79	0.95	0.04
sms	none	25000	0.25	3	1.00	1.00	0.09
video	none	25000	0.25	3	1.00	1.00	0.03
game	bonferroni	25000	0.25	3	0.65	0.80	0.02
sms	bonferroni	25000	0.25	3	1.00	1.00	0.02
video	bonferroni	25000	0.25	3	0.98	1.00	0.01
game	holm	25000	0.25	3	0.79	0.95	0.02
sms	holm	25000	0.25	3	1.00	1.00	0.02
video	holm	25000	0.25	3	0.99	1.00	0.01
game	hochberg	25000	0.25	3	0.79	0.95	0.02
sms	hochberg	25000	0.25	3	1.00	1.00	0.02
video	hochberg	25000	0.25	3	0.99	1.00	0.01
game	none	25000	0.25	4	0.87	0.91	0.05
sms	none	25000	0.25	4	1.00	1.00	0.06
video	none	25000	0.25	4	0.99	1.00	0.04
game	bonferroni	25000	0.25	4	0.71	0.79	0.02
sms	bonferroni	25000	0.25	4	1.00	1.00	0.04
video	bonferroni	25000	0.25	4	0.98	1.00	0.03
game	holm	25000	0.25	4	0.87	0.91	0.02
sms	holm	25000	0.25	4	1.00	1.00	0.04
video	holm	25000	0.25	4	0.99	1.00	0.03
game	hochberg	25000	0.25	4	0.87	0.91	0.02
sms	hochberg	25000	0.25	4	1.00	1.00	0.04
video	hochberg	25000	0.25	4	0.99	1.00	0.03
game	none	25000	0.30	1	0.81	0.86	0.03
sms	none	25000	0.30	1	1.00	1.00	0.06
video	none	25000	0.30	1	0.98	1.00	0.05
game	bonferroni	25000	0.30	1	0.65	0.72	0.01
sms	bonferroni	25000	0.30	1	1.00	1.00	0.02
video	bonferroni	25000	0.30	1	0.96	1.00	0.00
game	holm	25000	0.30	1	0.79	0.86	0.01
sms	holm	25000	0.30	1	1.00	1.00	0.02
video	holm	25000	0.30	1	0.97	1.00	0.00
game	hochberg	25000	0.30	1	0.80	0.86	0.01
sms	hochberg	25000	0.30	1	1.00	1.00	0.02
video	hochberg	25000	0.30	1	0.98	1.00	0.00
game	none	25000	0.30	2	0.86	0.94	0.03
sms	none	25000	0.30	2	1.00	1.00	0.01
video	none	25000	0.30	2	0.98	1.00	0.03
game	bonferroni	25000	0.30	2	0.67	0.83	0.02
sms	bonferroni	25000	0.30	2	1.00	1.00	0.00
video	bonferroni	25000	0.30	2	0.97	1.00	0.01
game	holm	25000	0.30	2	0.86	0.94	0.02
sms	holm	25000	0.30	2	1.00	1.00	0.00
video	holm	25000	0.30	2	0.97	1.00	0.01
game	hochberg	25000	0.30	2	0.86	0.94	0.02
sms	hochberg	25000	0.30	2	1.00	1.00	0.00
video	hochberg	25000	0.30	2	0.97	1.00	0.01
game	none	25000	0.30	3	0.86	0.94	0.04
sms	none	25000	0.30	3	1.00	1.00	0.04
video	none	25000	0.30	3	1.00	1.00	0.03
game	bonferroni	25000	0.30	3	0.69	0.85	0.01
sms	bonferroni	25000	0.30	3	1.00	1.00	0.02
video	bonferroni	25000	0.30	3	0.98	1.00	0.02
game	holm	25000	0.30	3	0.85	0.94	0.01
sms	holm	25000	0.30	3	1.00	1.00	0.02
video	holm	25000	0.30	3	0.98	1.00	0.02
game	hochberg	25000	0.30	3	0.86	0.94	0.01
sms	hochberg	25000	0.30	3	1.00	1.00	0.02
video	hochberg	25000	0.30	3	0.99	1.00	0.02
game	none	25000	0.30	4	0.89	0.97	0.03
sms	none	25000	0.30	4	1.00	1.00	0.06
video	none	25000	0.30	4	1.00	1.00	0.05
game	bonferroni	25000	0.30	4	0.82	0.94	0.01
sms	bonferroni	25000	0.30	4	1.00	1.00	0.03
video	bonferroni	25000	0.30	4	0.99	1.00	0.02
game	holm	25000	0.30	4	0.89	0.97	0.01
sms	holm	25000	0.30	4	1.00	1.00	0.03
video	holm	25000	0.30	4	0.99	1.00	0.02
game	hochberg	25000	0.30	4	0.89	0.97	0.01
sms	hochberg	25000	0.30	4	1.00	1.00	0.03
video	hochberg	25000	0.30	4	0.99	1.00	0.02
game	none	25000	0.40	1	0.74	0.93	0.08
sms	none	25000	0.40	1	1.00	1.00	0.07
video	none	25000	0.40	1	0.98	1.00	0.07
game	bonferroni	25000	0.40	1	0.56	0.82	0.03
sms	bonferroni	25000	0.40	1	1.00	1.00	0.02
video	bonferroni	25000	0.40	1	0.92	0.99	0.02
game	holm	25000	0.40	1	0.74	0.93	0.03
sms	holm	25000	0.40	1	1.00	1.00	0.02
video	holm	25000	0.40	1	0.96	1.00	0.02
game	hochberg	25000	0.40	1	0.74	0.93	0.03
sms	hochberg	25000	0.40	1	1.00	1.00	0.02
video	hochberg	25000	0.40	1	0.96	1.00	0.02
game	none	25000	0.40	2	0.83	0.94	0.08
sms	none	25000	0.40	2	1.00	1.00	0.04
video	none	25000	0.40	2	0.98	1.00	0.06
game	bonferroni	25000	0.40	2	0.73	0.85	0.03
sms	bonferroni	25000	0.40	2	1.00	1.00	0.02
video	bonferroni	25000	0.40	2	0.97	0.98	0.01
game	holm	25000	0.40	2	0.83	0.94	0.03
sms	holm	25000	0.40	2	1.00	1.00	0.03
video	holm	25000	0.40	2	0.98	1.00	0.01
game	hochberg	25000	0.40	2	0.83	0.94	0.04
sms	hochberg	25000	0.40	2	1.00	1.00	0.03
video	hochberg	25000	0.40	2	0.98	1.00	0.02
game	none	25000	0.40	3	0.85	0.96	0.06
sms	none	25000	0.40	3	1.00	1.00	0.08
video	none	25000	0.40	3	0.98	1.00	0.06
game	bonferroni	25000	0.40	3	0.72	0.87	0.01
sms	bonferroni	25000	0.40	3	1.00	1.00	0.03
video	bonferroni	25000	0.40	3	0.98	1.00	0.02
game	holm	25000	0.40	3	0.85	0.96	0.01
sms	holm	25000	0.40	3	1.00	1.00	0.03
video	holm	25000	0.40	3	0.98	1.00	0.02
game	hochberg	25000	0.40	3	0.85	0.96	0.01
sms	hochberg	25000	0.40	3	1.00	1.00	0.03
video	hochberg	25000	0.40	3	0.98	1.00	0.02
game	none	25000	0.40	4	0.89	0.97	0.03
sms	none	25000	0.40	4	1.00	1.00	0.02
video	none	25000	0.40	4	1.00	1.00	0.02
game	bonferroni	25000	0.40	4	0.76	0.94	0.01
sms	bonferroni	25000	0.40	4	1.00	1.00	0.01
video	bonferroni	25000	0.40	4	0.98	1.00	0.00
game	holm	25000	0.40	4	0.89	0.97	0.01
sms	holm	25000	0.40	4	1.00	1.00	0.01
video	holm	25000	0.40	4	1.00	1.00	0.00
game	hochberg	25000	0.40	4	0.89	0.97	0.01
sms	hochberg	25000	0.40	4	1.00	1.00	0.01
video	hochberg	25000	0.40	4	1.00	1.00	0.00
game	none	30000	0.25	1	0.78	0.93	0.03
sms	none	30000	0.25	1	1.00	1.00	0.04
video	none	30000	0.25	1	1.00	1.00	0.06
game	bonferroni	30000	0.25	1	0.64	0.82	0.02
sms	bonferroni	30000	0.25	1	1.00	1.00	0.03
video	bonferroni	30000	0.25	1	0.97	1.00	0.00
game	holm	30000	0.25	1	0.78	0.93	0.03
sms	holm	30000	0.25	1	1.00	1.00	0.03
video	holm	30000	0.25	1	1.00	1.00	0.01
game	hochberg	30000	0.25	1	0.78	0.93	0.03
sms	hochberg	30000	0.25	1	1.00	1.00	0.03
video	hochberg	30000	0.25	1	1.00	1.00	0.01
game	none	30000	0.25	2	0.90	0.96	0.07
sms	none	30000	0.25	2	1.00	1.00	0.05
video	none	30000	0.25	2	1.00	0.99	0.04
game	bonferroni	30000	0.25	2	0.70	0.86	0.02
sms	bonferroni	30000	0.25	2	1.00	1.00	0.02
video	bonferroni	30000	0.25	2	0.99	0.99	0.02
game	holm	30000	0.25	2	0.90	0.96	0.02
sms	holm	30000	0.25	2	1.00	1.00	0.02
video	holm	30000	0.25	2	0.99	0.99	0.03
game	hochberg	30000	0.25	2	0.90	0.96	0.02
sms	hochberg	30000	0.25	2	1.00	1.00	0.02
video	hochberg	30000	0.25	2	0.99	0.99	0.03
game	none	30000	0.25	3	0.87	0.96	0.03
sms	none	30000	0.25	3	1.00	1.00	0.03
video	none	30000	0.25	3	1.00	1.00	0.06
game	bonferroni	30000	0.25	3	0.82	0.92	0.02
sms	bonferroni	30000	0.25	3	1.00	1.00	0.00
video	bonferroni	30000	0.25	3	1.00	1.00	0.01
game	holm	30000	0.25	3	0.87	0.96	0.02
sms	holm	30000	0.25	3	1.00	1.00	0.00
video	holm	30000	0.25	3	1.00	1.00	0.01
game	hochberg	30000	0.25	3	0.87	0.96	0.02
sms	hochberg	30000	0.25	3	1.00	1.00	0.00
video	hochberg	30000	0.25	3	1.00	1.00	0.01
game	none	30000	0.25	4	0.84	0.98	0.05
sms	none	30000	0.25	4	1.00	1.00	0.02
video	none	30000	0.25	4	1.00	1.00	0.06
game	bonferroni	30000	0.25	4	0.72	0.93	0.01
sms	bonferroni	30000	0.25	4	1.00	1.00	0.02
video	bonferroni	30000	0.25	4	0.99	1.00	0.04
game	holm	30000	0.25	4	0.84	0.98	0.02
sms	holm	30000	0.25	4	1.00	1.00	0.02
video	holm	30000	0.25	4	1.00	1.00	0.04
game	hochberg	30000	0.25	4	0.84	0.98	0.02
sms	hochberg	30000	0.25	4	1.00	1.00	0.02
video	hochberg	30000	0.25	4	1.00	1.00	0.04
game	none	30000	0.30	1	0.85	0.94	0.08
sms	none	30000	0.30	1	1.00	1.00	0.08
video	none	30000	0.30	1	1.00	1.00	0.05
game	bonferroni	30000	0.30	1	0.74	0.84	0.00
sms	bonferroni	30000	0.30	1	1.00	1.00	0.01
video	bonferroni	30000	0.30	1	0.98	1.00	0.01
game	holm	30000	0.30	1	0.84	0.94	0.00
sms	holm	30000	0.30	1	1.00	1.00	0.01
video	holm	30000	0.30	1	0.99	1.00	0.01
game	hochberg	30000	0.30	1	0.85	0.94	0.00
sms	hochberg	30000	0.30	1	1.00	1.00	0.01
video	hochberg	30000	0.30	1	1.00	1.00	0.01
game	none	30000	0.30	2	0.92	0.97	0.05
sms	none	30000	0.30	2	1.00	1.00	0.05
video	none	30000	0.30	2	1.00	1.00	0.04
game	bonferroni	30000	0.30	2	0.80	0.88	0.00
sms	bonferroni	30000	0.30	2	1.00	1.00	0.02
video	bonferroni	30000	0.30	2	1.00	1.00	0.04
game	holm	30000	0.30	2	0.92	0.97	0.00
sms	holm	30000	0.30	2	1.00	1.00	0.03
video	holm	30000	0.30	2	1.00	1.00	0.04
game	hochberg	30000	0.30	2	0.92	0.97	0.00
sms	hochberg	30000	0.30	2	1.00	1.00	0.03
video	hochberg	30000	0.30	2	1.00	1.00	0.04
game	none	30000	0.30	3	0.95	0.97	0.05
sms	none	30000	0.30	3	1.00	1.00	0.03
video	none	30000	0.30	3	1.00	1.00	0.06
game	bonferroni	30000	0.30	3	0.84	0.91	0.04
sms	bonferroni	30000	0.30	3	1.00	1.00	0.02
video	bonferroni	30000	0.30	3	1.00	1.00	0.03
game	holm	30000	0.30	3	0.95	0.97	0.04
sms	holm	30000	0.30	3	1.00	1.00	0.02
video	holm	30000	0.30	3	1.00	1.00	0.04
game	hochberg	30000	0.30	3	0.95	0.97	0.04
sms	hochberg	30000	0.30	3	1.00	1.00	0.02
video	hochberg	30000	0.30	3	1.00	1.00	0.04
game	none	30000	0.30	4	0.96	0.98	0.05
sms	none	30000	0.30	4	1.00	1.00	0.06
video	none	30000	0.30	4	1.00	1.00	0.05
game	bonferroni	30000	0.30	4	0.83	0.91	0.02
sms	bonferroni	30000	0.30	4	1.00	1.00	0.04
video	bonferroni	30000	0.30	4	0.99	1.00	0.02
game	holm	30000	0.30	4	0.96	0.98	0.02
sms	holm	30000	0.30	4	1.00	1.00	0.04
video	holm	30000	0.30	4	1.00	1.00	0.02
game	hochberg	30000	0.30	4	0.96	0.98	0.02
sms	hochberg	30000	0.30	4	1.00	1.00	0.04
video	hochberg	30000	0.30	4	1.00	1.00	0.02
game	none	30000	0.40	1	0.78	0.91	0.08
sms	none	30000	0.40	1	1.00	1.00	0.06
video	none	30000	0.40	1	0.99	1.00	0.08
game	bonferroni	30000	0.40	1	0.62	0.83	0.04
sms	bonferroni	30000	0.40	1	1.00	1.00	0.00
video	bonferroni	30000	0.40	1	0.97	0.99	0.01
game	holm	30000	0.40	1	0.78	0.91	0.04
sms	holm	30000	0.40	1	1.00	1.00	0.00
video	holm	30000	0.40	1	0.99	1.00	0.02
game	hochberg	30000	0.40	1	0.78	0.91	0.04
sms	hochberg	30000	0.40	1	1.00	1.00	0.00
video	hochberg	30000	0.40	1	0.99	1.00	0.02
game	none	30000	0.40	2	0.82	0.95	0.04
sms	none	30000	0.40	2	1.00	1.00	0.07
video	none	30000	0.40	2	1.00	1.00	0.13
game	bonferroni	30000	0.40	2	0.79	0.84	0.01
sms	bonferroni	30000	0.40	2	1.00	1.00	0.02
video	bonferroni	30000	0.40	2	1.00	1.00	0.02
game	holm	30000	0.40	2	0.82	0.95	0.01
sms	holm	30000	0.40	2	1.00	1.00	0.02
video	holm	30000	0.40	2	1.00	1.00	0.02
game	hochberg	30000	0.40	2	0.82	0.95	0.01
sms	hochberg	30000	0.40	2	1.00	1.00	0.02
video	hochberg	30000	0.40	2	1.00	1.00	0.02
game	none	30000	0.40	3	0.91	0.98	0.07
sms	none	30000	0.40	3	1.00	1.00	0.06
video	none	30000	0.40	3	1.00	1.00	0.08
game	bonferroni	30000	0.40	3	0.73	0.93	0.03
sms	bonferroni	30000	0.40	3	1.00	1.00	0.02
video	bonferroni	30000	0.40	3	1.00	1.00	0.03
game	holm	30000	0.40	3	0.91	0.98	0.03
sms	holm	30000	0.40	3	1.00	1.00	0.02
video	holm	30000	0.40	3	1.00	1.00	0.04
game	hochberg	30000	0.40	3	0.91	0.98	0.04
sms	hochberg	30000	0.40	3	1.00	1.00	0.02
video	hochberg	30000	0.40	3	1.00	1.00	0.05
game	none	30000	0.40	4	0.92	0.99	0.08
sms	none	30000	0.40	4	1.00	1.00	0.01
video	none	30000	0.40	4	1.00	1.00	0.04
game	bonferroni	30000	0.40	4	0.80	0.98	0.02
sms	bonferroni	30000	0.40	4	1.00	1.00	0.01
video	bonferroni	30000	0.40	4	1.00	1.00	0.01
game	holm	30000	0.40	4	0.92	0.99	0.02
sms	holm	30000	0.40	4	1.00	1.00	0.01
video	holm	30000	0.40	4	1.00	1.00	0.01
game	hochberg	30000	0.40	4	0.92	0.99	0.02
sms	hochberg	30000	0.40	4	1.00	1.00	0.01
video	hochberg	30000	0.40	4	1.00	1.00	0.01

Figures

for(o in c("week1_score", "biweek1_score", "monthly_score")){
  cat(paste("## Outcome:", o, "{.tabset}\n\n\n"))
  for(p in c(0.25, 0.3, 0.4)){
      cat(paste("### Probability:", p, "{.tabset}\n\n\n"))

    for(s in 1:4){
        cat(paste("#### Sample:", s, "\n\n\n"))

      plot_df <- power_final %>%
        filter(outcome == o & correction == "holm" & p_control == p &  sample == s) 
      
      gg <- ggplot(plot_df, aes(x = N, y = power, group = str_to_title(hypothesis), color = str_to_title(hypothesis)) ) +
    geom_hline(yintercept = 0.8, 
               linetype = "dashed", 
               color = "grey40") +
    geom_line() + 
    scale_y_continuous(limits = c(0, 1), 
                       breaks = seq(0, 1, 0.2), 
                       expand = c(0, NA)) + 
    scale_x_continuous(limits = c(5000, 30000), 
                       breaks = seq(5000, 30000, 5000), 
                       labels = scales::comma) + 
    theme_sarah() + 
    labs(x = "Sample Size", y = "Power", 
         color = "Hypothesis",
         title = paste("Power by Sample Size"), 
         subtitle = paste("For p = ", p, ", Sample = ", s, ", Outcome = ",o, "\n", sep = "")) +
    theme(legend.position = c(.85,0.2), 
          legend.title = element_text(size = 16, face = "bold"), 
          legend.text = element_text(size =16), 
          legend.key = element_rect(fill = "white")) 
      print(gg)
      cat("\n\n\n")
    }
          cat("\n\n\n")

  }
        cat("\n\n\n")

}

Outcome: week1_score

Probability: 0.25

Sample: 1

Sample: 2

Sample: 3

Sample: 4

Probability: 0.3

Sample: 1

Sample: 2

Sample: 3

Sample: 4

Probability: 0.4

Sample: 1

Sample: 2

Sample: 3

Sample: 4

Outcome: biweek1_score

Probability: 0.25

Sample: 1

Sample: 2

Sample: 3

Sample: 4

Probability: 0.3

Sample: 1

Sample: 2

Sample: 3

Sample: 4

Probability: 0.4

Sample: 1

Sample: 2

Sample: 3

Sample: 4

Outcome: monthly_score

Probability: 0.25

Sample: 1

Sample: 2

Sample: 3

Sample: 4

Probability: 0.3

Sample: 1

Sample: 2

Sample: 3

Sample: 4

Probability: 0.4

Sample: 1

Sample: 2

Sample: 3

Sample: 4