optim_out_d234

## function to output design with lowest fmin for scale= scale0 and n = end
fl <- function(scale0, end){
  dat <- out[out[,2] == scale0, end]
  if(length(dat) < 6){return("NA")}
  op <- c("dist_beta", "maximin", "LHS", "random", "lhsbeta1", "lhsbeta2")[rank(dat) == 1]
  return(op)
}

## paired t-test
pairt <- function(design1, design2, clm){
  out$design <- as.character(out$design)
  sp1 <- out[out$design == design1, clm + 2]
  sp2 <- out[out$design == design2, clm + 2]
  return(t.test(log(sp1), log(sp2), alternative = "less", paired = T)$p.value)
}
#pairt("maximin", "dist_beta",30)

2-D

load("/home/boyazhang/repos/unifdist/code/optim_ei_2d_prl.RData")
out <- out[,c(c(ncol(out), ncol(out)-1), 1:(ncol(out)-2))]
scales <- unique(out[,2])

par(mfcol = c(2,2))
boxplot(log(out[,17]) ~out$design, main = paste("2d, ninit = 8, 10*sqrt(m), no-update, fmin after", 15, "searches"))
boxplot(log(out[,27]) ~out$design, main = paste("2d, ninit = 8, 10*sqrt(m), no-update, fmin after", 25, "searches"))
boxplot(log(out[,72]) ~out$design, main = paste("2d, ninit = 8, 10*sqrt(m), no-update, fmin after", 70, "searches"))
boxplot(log(out[,102]) ~out$design, main = paste("2d, ninit = 8, 10*sqrt(m), no-update, fmin after", 100, "searches"))

## lowest designs for all scales after 30, 50, 100, 200 searches
search_num <- c(15,25,70,100)
tb <- data.frame(dist_beta = rep(NA, 4),  maximin= rep(NA, 4),  LHS= rep(NA, 4),  random= rep(NA, 4),  lhsbeta1= rep(NA, 4),  lhsbeta2= rep(NA, 4))
tb <- cbind(search_num, tb)
bestdsn <- rep(NA, length(scales))

for( j in 1:length(search_num)){
  for( i in 1:length(scales)){
    bestdsn[i] <- fl(scales[i], search_num[j])}
  tb[j, 2:7] <- table(bestdsn)
}

# ## paired-t test pval table
# pvtb <- list()
# for( j in 1:4){
#   ntb <- tb[j,-1]
#   pvtb[[j]] <- data.frame(dist_beta = NA,  maximin= NA,  LHS= NA,  random= NA,  lhsbeta1= NA,  lhsbeta2= NA)
#   names(pvtb[[j]]) <- names(sort(ntb, decreasing = T))
#   for( i in 1:5)
#   { d1 <-  names(ntb)[rank(ntb, ties.method = "first") == 7-i]
#     d2 <-  names(ntb)[rank(ntb, ties.method = "first") == 6-i]
#     pvtb[[j]][,i] <- pairt(d1, d2, tb[j,1])
#   }
# }

p-val matrix

2d: 15 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.9979211	0.8028183	0.0498150	1.0000000	1.0000000
maximin	0.0020789	NaN	0.0161530	0.0000027	0.9967392	0.9998952
LHS	0.1971817	0.9838470	NaN	0.0047756	0.9999997	1.0000000
random	0.9501850	0.9999973	0.9952244	NaN	1.0000000	1.0000000
lhsbeta1	0.0000000	0.0032608	0.0000003	0.0000000	NaN	0.8857774
lhsbeta2	0.0000000	0.0001048	0.0000000	0.0000000	0.1142226	NaN

2d: 25 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.0000008	0.0000047	0.0000092	0.9998745	0.9999696
maximin	0.9999992	NaN	0.7779433	0.7870843	1.0000000	1.0000000
LHS	0.9999953	0.2220567	NaN	0.5111085	1.0000000	1.0000000
random	0.9999908	0.2129157	0.4888915	NaN	1.0000000	1.0000000
lhsbeta1	0.0001255	0.0000000	0.0000000	0.0000000	NaN	0.6148437
lhsbeta2	0.0000304	0.0000000	0.0000000	0.0000000	0.3851563	NaN

2d: 70 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.00e+00	0.0000010	0.0000002	1.000000	1.000000
maximin	1.0000000	NaN	0.9999838	0.9999565	1.000000	1.000000
LHS	0.9999990	1.62e-05	NaN	0.3753832	1.000000	1.000000
random	0.9999998	4.35e-05	0.6246168	NaN	1.000000	1.000000
lhsbeta1	0.0000000	0.00e+00	0.0000000	0.0000000	NaN	0.001279
lhsbeta2	0.0000000	0.00e+00	0.0000000	0.0000000	0.998721	NaN

2d: 100 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.00e+00	0.0000120	0.0000001	1.0000000	0.9999997
maximin	1.0000000	NaN	0.9999984	0.9999532	1.0000000	1.0000000
LHS	0.9999880	1.60e-06	NaN	0.1913908	1.0000000	1.0000000
random	0.9999999	4.68e-05	0.8086092	NaN	1.0000000	1.0000000
lhsbeta1	0.0000000	0.00e+00	0.0000000	0.0000000	NaN	0.0155375
lhsbeta2	0.0000003	0.00e+00	0.0000000	0.0000000	0.9844625	NaN

3-D

load("/home/boyazhang/repos/unifdist/code/optim_ei_3d.RData")
out <- out[,c(c(ncol(out), ncol(out)-1), 1:(ncol(out)-2))]
scales <- unique(out[,2])

par(mfcol = c(2,2))
boxplot(log(out[,22]) ~out$design, main = paste("3d, ninit = 16, 10*sqrt(m), no-update, fmin after", 20, "searches"))
boxplot(log(out[,52]) ~out$design, main = paste("3d, ninit = 16, 10*sqrt(m), no-update, fmin after", 50, "searches"))
boxplot(log(out[,102]) ~out$design, main = paste("3d, ninit = 16, 10*sqrt(m), no-update, fmin after", 100, "searches"))
boxplot(log(out[,202]) ~out$design, main = paste("3d, ninit = 16, 10*sqrt(m), no-update, fmin after", 200, "searches"))

## lowest designs for all scales after 30, 50, 100, 200 searches
search_num <- c(20,50,100,200)
tb <- data.frame(dist_beta = rep(NA, 4),  maximin= rep(NA, 4),  LHS= rep(NA, 4),  random= rep(NA, 4),  lhsbeta1= rep(NA, 4),  lhsbeta2= rep(NA, 4))
tb <- cbind(search_num, tb)
bestdsn <- rep(NA, length(scales))

for( j in 1:length(search_num)){
  for( i in 1:length(scales)){
    bestdsn[i] <- fl(scales[i], search_num[j])}
  tb[j, 2:7] <- table(bestdsn)
}

## paired-t test pval table
pvtb <- list()
for( j in 1:4){
  ntb <- tb[j,-1]
  pvtb[[j]] <- data.frame(dist_beta = NA,  maximin= NA,  LHS= NA,  random= NA,  lhsbeta1= NA,  lhsbeta2= NA)
  names(pvtb[[j]]) <- names(sort(ntb, decreasing = T))
  for( i in 1:5)
  { d1 <-  names(ntb)[rank(ntb, ties.method = "first") == 7-i]
    d2 <-  names(ntb)[rank(ntb, ties.method = "first") == 6-i]
    pvtb[[j]][,i] <- pairt(d1, d2, tb[j,1])
  }
}

p-val matrix

3d: 20 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.9996420	1.0000000	0.9999862	0.5803819	0.8650862
maximin	0.0003580	NaN	0.9761562	0.6892534	0.0002305	0.0036311
LHS	0.0000000	0.0238438	NaN	0.0595703	0.0000000	0.0000008
random	0.0000138	0.3107466	0.9404297	NaN	0.0000205	0.0007262
lhsbeta1	0.4196181	0.9997695	1.0000000	0.9999795	NaN	0.8204560
lhsbeta2	0.1349138	0.9963689	0.9999992	0.9992738	0.1795440	NaN

3d: 50 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.0004755	0.0252589	0.0200167	0.9998462	0.9575303
maximin	0.9995245	NaN	0.9199095	0.8848266	1.0000000	0.9999997
LHS	0.9747411	0.0800905	NaN	0.4193443	1.0000000	0.9998596
random	0.9799833	0.1151734	0.5806557	NaN	1.0000000	0.9998988
lhsbeta1	0.0001538	0.0000000	0.0000000	0.0000000	NaN	0.0210505
lhsbeta2	0.0424697	0.0000003	0.0001404	0.0001012	0.9789495	NaN

3d: 100 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.00e+00	0.0005158	0.0000505	1.0000000	0.9999821
maximin	1.0000000	NaN	0.9999614	0.9994970	1.0000000	1.0000000
LHS	0.9994842	3.86e-05	NaN	0.2353101	1.0000000	1.0000000
random	0.9999495	5.03e-04	0.7646899	NaN	1.0000000	1.0000000
lhsbeta1	0.0000000	0.00e+00	0.0000000	0.0000000	NaN	0.0030752
lhsbeta2	0.0000179	0.00e+00	0.0000000	0.0000000	0.9969248	NaN

3d: 200 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0e+00	0.0000008	0.0000000	1.0000000	0.9986775
maximin	1.0000000	NaN	1.0000000	0.9999998	1.0000000	1.0000000
LHS	0.9999992	0e+00	NaN	0.1354787	1.0000000	1.0000000
random	1.0000000	2e-07	0.8645213	NaN	1.0000000	1.0000000
lhsbeta1	0.0000000	0e+00	0.0000000	0.0000000	NaN	0.0018074
lhsbeta2	0.0013225	0e+00	0.0000000	0.0000000	0.9981926	NaN

4-D

load("/home/boyazhang/repos/unifdist/code/optim_ei_4d_prl_test.RData")
par(mfcol = c(2,2))
scales <- unique(out[,2])
boxplot(log(out[,52]) ~out$design, main = paste("4d, ninit = 32, 10*sqrt(m), no-update, fmin after", 50, "searches"))
boxplot(log(out[,202]) ~out$design, main = paste("4d, ninit = 32, 10*sqrt(m), no-update, fmin after", 200, "searches"))
boxplot(log(out[,502]) ~out$design, main = paste("4d, ninit = 32, 10*sqrt(m), no-update, fmin after", 500, "searches"))
boxplot(log(out[,702]) ~out$design, main = paste("4d, ninit = 32, 10*sqrt(m), no-update, fmin after", 700, "searches"))

# ## remove scales with uncomplete output
# naind <- c()
# for( i in 1:length(scales)){
#   if(fl(scales[i], 100) == "NA"){
#     naind <- c(naind, i)
#   }
# }
# for(i in naind){
#   out <- out[out[,2] != scales[i],]
# }
# scales <- scales[-naind]

## lowest designs for all scales after 30, 50, 100, 200 searches
search_num <- c(50,200,500,700)
tb <- data.frame(dist_beta = rep(NA, 4),  maximin= rep(NA, 4),  LHS= rep(NA, 4),  random= rep(NA, 4),  lhsbeta1= rep(NA, 4),  lhsbeta2= rep(NA, 4))
tb <- cbind(search_num, tb)
bestdsn <- rep(NA, length(scales))

for( j in 1:length(search_num)){
  for( i in 1:length(scales)){
    bestdsn[i] <- fl(scales[i], search_num[j])}
  tb[j, 2:7] <- table(bestdsn)
}

## paired-t test pval table
pvtb <- list()
for( j in 1:4){
  ntb <- tb[j,-1]
  pvtb[[j]] <- data.frame(dist_beta = NA,  maximin= NA,  LHS= NA,  random= NA,  lhsbeta1= NA,  lhsbeta2= NA)
  names(pvtb[[j]]) <- names(sort(ntb, decreasing = T))
  for( i in 1:5)
  { d1 <-  names(ntb)[rank(ntb, ties.method = "first") == 7-i]
    d2 <-  names(ntb)[rank(ntb, ties.method = "first") == 6-i]
    pvtb[[j]][,i] <- pairt(d1, d2, tb[j,1])
  }
}

p-val matrix

4d: 50 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	1.0000000	1.0000000	1.0000000	1.0000000	1.0000000
maximin	0	NaN	0.8076274	0.9930352	0.6752545	0.6953501
LHS	0	0.1923726	NaN	0.9721547	0.3252330	0.3485853
random	0	0.0069648	0.0278453	NaN	0.0108353	0.0144140
lhsbeta1	0	0.3247455	0.6747670	0.9891647	NaN	0.5203524
lhsbeta2	0	0.3046499	0.6514147	0.9855860	0.4796476	NaN

4d: 200 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.0043229	0.8331157	0.9993764	0.9999731	0.9999955
maximin	0.9956771	NaN	0.9998459	1.0000000	1.0000000	1.0000000
LHS	0.1668843	0.0001541	NaN	0.9913833	0.9992273	0.9996821
random	0.0006236	0.0000000	0.0086167	NaN	0.7530273	0.8341066
lhsbeta1	0.0000269	0.0000000	0.0007727	0.2469727	NaN	0.6182122
lhsbeta2	0.0000045	0.0000000	0.0003179	0.1658934	0.3817878	NaN

4d: 500 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	0.0002372	0.9611947	0.9436150	0.9998045	0.9981271
maximin	0.9997628	NaN	0.9999999	0.9999997	1.0000000	1.0000000
LHS	0.0388053	0.0000001	NaN	0.4363040	0.9654461	0.8616354
random	0.0563850	0.0000003	0.5636960	NaN	0.9737361	0.8904576
lhsbeta1	0.0001955	0.0000000	0.0345539	0.0262639	NaN	0.2489401
lhsbeta2	0.0018729	0.0000000	0.1383646	0.1095424	0.7510599	NaN

4d: 700 searches

	dist_beta	maximin	LHS	random	lhsbeta1	lhsbeta2
dist_beta	NaN	1.02e-05	0.8411475	0.3941812	0.9958072	0.9741775
maximin	0.9999898	NaN	0.9999999	0.9999684	1.0000000	1.0000000
LHS	0.1588525	1.00e-07	NaN	0.1005475	0.9555382	0.8340870
random	0.6058188	3.16e-05	0.8994525	NaN	0.9983797	0.9857086
lhsbeta1	0.0041928	0.00e+00	0.0444618	0.0016203	NaN	0.2460469
lhsbeta2	0.0258225	0.00e+00	0.1659130	0.0142914	0.7539531	NaN