Rejection rates for SB, EBA2 and EBAHalf(1.5)

knitr::opts_chunk$set(echo = T, cache=F, message=F, warning = FALSE)
library(tidyverse); library(kableExtra)
rr.df <- readRDS("rrdf.rds")

Small model: Dimension 10

ddim <- 10
condition <- filter(rr.df, grepl("half2|eba2|sb", test) & dim==ddim & n < 4000000)
condition$test2 <- str_extract(condition$test, "^[a-z2]+")
condition$test2 <- str_replace(condition$test2, "eba$", "ebahalf")

ggplot(condition, aes(dist, rr, color=test2, group=test))+geom_point()+geom_line()+
  facet_grid(n~gamma+chisq, scales="free")+
  geom_abline(slope=0, intercept=0.05)+
  theme(axis.text.x = element_text(angle = 90))

Medium model: Dimension 20

ddim <- 20
condition <- filter(rr.df, grepl("half2|eba2|sb", test) & dim==ddim & n < 4000000)
condition$test2 <- str_extract(condition$test, "^[a-z2]+")
condition$test2 <- str_replace(condition$test2, "eba$", "ebahalf")

ggplot(condition, aes(dist, rr, color=test2, group=test))+geom_point()+geom_line()+
  facet_grid(n~gamma+chisq, scales="free")+
  geom_abline(slope=0, intercept=0.05)+
  theme(axis.text.x = element_text(angle = 90))

Large model: Dimension 40

ddim <- 40
condition <- filter(rr.df, grepl("half2|eba2|sb", test) & dim==ddim & n < 4000000)
condition$test2 <- str_extract(condition$test, "^[a-z2]+")
condition$test2 <- str_replace(condition$test2, "eba$", "ebahalf")

ggplot(condition, aes(dist, rr, color=test2, group=test))+geom_point()+geom_line()+
  facet_grid(n~gamma+chisq, scales="free")+
  geom_abline(slope=0, intercept=0.05)+
  theme(axis.text.x = element_text(angle = 90))

Top 4 Winners RMSE

## measure performance
for(dd in c(10,20,40)){
  for(nn in unique(rr.df$n) ){
    cat("dim=", dd, "n=", nn, "\n \n")
    rr.df %>% filter(n ==nn & dim==dd) %>%
      group_by(test)%>%
      summarise(n(),rmse=sqrt(mean((rr-0.05)^2)),abs=median(abs(rr-.05))) %>% 
      arrange(rmse) %>%  
      head(4) %>% kable() %>% print()
  }
}

dim= 10 n= 200

test	n()	rmse	abs
eba_half4_ub_trad	7	0.0035702	0.0030000
eba_half4_rls	7	0.0035702	0.0023333
eba_half2_ub_rls	7	0.0046904	0.0040000
eba_half6_ub_trad	7	0.0055248	0.0033333

dim= 10 n= 400

test	n()	rmse	abs
eba_half2_rls	7	0.0057113	0.0050000
eba_half2_ub_trad	7	0.0062425	0.0053333
eba_half2_ub_rls	7	0.0064501	0.0030000
eba_half4_trad	7	0.0066964	0.0063333

dim= 10 n= 800

test	n()	rmse	abs
eba_half4_ub_rls	7	0.0059881	0.0046667
eba_half6_rls	7	0.0059987	0.0056667
eba_half4_ub_trad	7	0.0061734	0.0053333
eba_half4_rls	7	0.0064427	0.0056667

dim= 10 n= 1500

test	n()	rmse	abs
eba_half2_rls	7	0.0064832	0.0046667
eba_half2_trad	7	0.0065961	0.0066667
sb_ub_trad	7	0.0066153	0.0020000
sb_trad	7	0.0068359	0.0020000

dim= 10 n= 3000

test	n()	rmse	abs
eba2_trad	7	0.0037310	0.0006667
eba_half6_ub_trad	7	0.0038131	0.0023333
eba2_rls	7	0.0038276	0.0023333
eba2_ub_trad	7	0.0039521	0.0006667

dim= 10 n= 10000

test	n()	rmse	abs
eba4_ub_rls	7	0.0026125	0.0023333
eba4_rls	7	0.0026156	0.0016667
eba4_ub_trad	7	0.0026337	0.0023333
eba4_trad	7	0.0026487	0.0020000

dim= 10 n= 1e+05

test	n()	rmse	abs
ss_rls	7	0.0036143	0.003
ss_ub_rls	7	0.0036143	0.003
ss_trad	7	0.0037097	0.003
ss_ub_trad	7	0.0037097	0.003

dim= 20 n= 200

test	n()	rmse	abs
eba_half6_rls	7	0.0106986	0.0106667
eba_half4_rls	7	0.0136271	0.0110000
eba_half4_ub_rls	7	0.0141079	0.0166667
eba2_ub_trad	7	0.0145149	0.0100000

dim= 20 n= 400

test	n()	rmse	abs
eba_half4_rls	7	0.0110920	0.0053333
eba_half6_rls	7	0.0124862	0.0146667
eba_half4_ub_rls	7	0.0128594	0.0140000
eba_half2_ub_rls	7	0.0154087	0.0093333

dim= 20 n= 800

test	n()	rmse	abs
eba_half4_rls	7	0.0086069	0.0066667
eba_half4_ub_rls	7	0.0098818	0.0090000
eba_half6_rls	7	0.0099825	0.0076667
eba_half2_ub_rls	7	0.0107851	0.0080000

dim= 20 n= 1500

test	n()	rmse	abs
eba_half6_rls	7	0.0126973	0.0066667
eba_half4_ub_rls	7	0.0128742	0.0076667
eba_half4_rls	7	0.0130238	0.0050000
eba_half6_ub_rls	7	0.0130408	0.0096667

dim= 20 n= 3000

test	n()	rmse	abs
eba_half6_ub_rls	7	0.0132168	0.0096667
eba_half6_rls	7	0.0132192	0.0083333
eba_half4_rls	7	0.0136172	0.0063333
eba_half4_ub_rls	7	0.0137350	0.0070000

dim= 20 n= 10000

test	n()	rmse	abs
eba2_ub_rls	7	0.0121479	0.0056667
eba2_rls	7	0.0125300	0.0056667
eba4_trad	7	0.0126529	0.0080000
eba4_ub_trad	7	0.0126848	0.0083333

dim= 20 n= 1e+05

test	n()	rmse	abs
ss_rls	7	0.0044454	0.0033333
ss_ub_rls	7	0.0044454	0.0033333
ss_trad	7	0.0044934	0.0026667
ss_ub_trad	7	0.0044934	0.0026667

dim= 40 n= 200

test	n()	rmse	abs
eba4_ub_trad	7	0.0278012	0.0300000
eba6_trad	7	0.0325135	0.0386667
eba_half6_ub_rls	7	0.0330252	0.0350000
eba2_ub_rls	7	0.0348161	0.0310000

dim= 40 n= 400

test	n()	rmse	abs
eba_half4_rls	7	0.0185485	0.0166667
eba_half6_rls	7	0.0190588	0.0186667
eba_half4_ub_rls	7	0.0210517	0.0263333
eba2_rls	7	0.0240264	0.0280000

dim= 40 n= 800

test	n()	rmse	abs
eba_half4_rls	7	0.0123847	0.0100000
eba_half2_ub_rls	7	0.0125616	0.0110000
eba_half2_rls	7	0.0156022	0.0103333
eba_half4_ub_rls	7	0.0180366	0.0156667

dim= 40 n= 1500

test	n()	rmse	abs
eba_half4_rls	7	0.0126961	0.0116667
eba_half2_ub_rls	7	0.0134843	0.0070000
eba_half2_rls	7	0.0154252	0.0073333
eba_half4_ub_rls	7	0.0155818	0.0153333

dim= 40 n= 3000

test	n()	rmse	abs
eba_half2_ub_rls	7	0.0115126	0.0056667
eba_half2_rls	7	0.0119742	0.0043333
eba_half4_rls	7	0.0122966	0.0116667
eba_half4_ub_rls	7	0.0129590	0.0130000

dim= 40 n= 10000

test	n()	rmse	abs
eba_half4_ub_rls	7	0.0076376	0.0033333
eba_half6_rls	7	0.0077162	0.0030000
eba_half4_rls	7	0.0077654	0.0030000
eba_half6_ub_rls	7	0.0078911	0.0036667

dim= 40 n= 1e+05

test	n()	rmse	abs
ss_ub_rls	7	0.0066165	0.0066667
ss_rls	7	0.0067212	0.0066667
ss_trad	7	0.0067987	0.0070000
ss_ub_trad	7	0.0068533	0.0070000