ARCENE Benchmarks

1 FRESA.CAD ARCENE Benchmark

1.1 ARCENE

trainLabeled <- read.delim("C:/Users/Jose Tamez/Dropbox/Documents/FRESACAD/FresaPaper/FresaPaper/Arcene/ARCENE/trainSet.txt")
validLabeled <- read.delim("C:/Users/Jose Tamez/Dropbox/Documents/FRESACAD/FresaPaper/FresaPaper/Arcene/ARCENE/arcene_valid.txt")

trainLabeled$Labels <-  1*(trainLabeled$Labels > 0)
validLabeled$Labels <-  1*(validLabeled$Labels > 0)

arcene <- rbind(trainLabeled,validLabeled)
arcene <- as.data.frame(arcene)


ArceneFRESA <- FRESA.Model(formula = Labels ~ 1,data = arcene,repeats = 10)

1.2 Benchmark

cp <- CVBenchmark(theData = arcene, theOutcome = "Labels", reps = 50, topIncluded = 200)

elapcol <- names(cp$times[[1]]) == "elapsed"
cputimes <- list(Fresa = mean(cp$times$fresatime[ elapcol ]),LASSO = mean(cp$times$LASSOtime[ elapcol ]),RF = mean(cp$times$RFtime[ elapcol ]),SVM = mean(cp$times$SVMtime[ elapcol ]),FeaLect=mean(cp$times$FeaLecttime[ elapcol ]))

featsize <- list(Fresa = mean(cp$featSize$FRESASize),LASSO = mean(cp$featSize$LASSOSize),Univ = mean(cp$featSize$UNIVSize))

1.2.1 Arcene Results

#The Times
pander::pander(cputimes)

• Fresa: 80.32
• LASSO: 1.62
• RF: 86.21
• SVM: 9.107
• FeaLect: 180.4

pander::pander(featsize)

• Fresa: 45.62
• LASSO: 30.12
• Univ: 2897

plotBEREvolution(cp,100,main="Balanced Error Rate", location="topright")

bp <- barPlotCiError(as.matrix(cp$errorciTable),metricname="Balanced Error",thesets=thesets,themethod=theMethod,main="Balanced Error",offsets = c(0.5,0.05),args.legend = list(x = "topright"))

pander::pander(bp$barMatrix,caption = "Balanced Error Rate",round = 3)

Balanced Error Rate

	Default Classifier	Filtered SVM Classifier
B:SWiMS	0.235	0.207
LASSO	0.241	0.217
RF	0.226	0.179
SVM	0.297	0.22

pander::pander(bp$ciTable,caption = "Balanced Error Rate with 95%CI",round = 3)

Balanced Error Rate with 95%CI

	Balanced Error	lower	upper
Default Classifier	0.235	0.155	0.33
Default Classifier	0.241	0.161	0.337
Default Classifier	0.226	0.148	0.32
Default Classifier	0.297	0.214	0.392
Filtered SVM Classifier	0.207	0.132	0.301
Filtered SVM Classifier	0.217	0.141	0.31
Filtered SVM Classifier	0.179	0.109	0.269
Filtered SVM Classifier	0.22	0.145	0.313

bp <- barPlotCiError(as.matrix(cp$accciTable),metricname="Accuracy",thesets=thesets,themethod=theMethod,main="Accuracy",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "Accuracy",round = 3)

Accuracy

	Default Classifier	Filtered SVM Classifier
B:SWiMS	0.77	0.795
LASSO	0.765	0.79
RF	0.78	0.82
SVM	0.72	0.79

pander::pander(bp$ciTable,caption = "Accuracy with 95%CI",round = 3)

Accuracy with 95%CI

	Accuracy	lower	upper
Default Classifier	0.77	0.705	0.826
Default Classifier	0.765	0.7	0.822
Default Classifier	0.78	0.716	0.835
Default Classifier	0.72	0.652	0.781
Filtered SVM Classifier	0.795	0.732	0.849
Filtered SVM Classifier	0.79	0.727	0.844
Filtered SVM Classifier	0.82	0.76	0.871
Filtered SVM Classifier	0.79	0.727	0.844

bp <- barPlotCiError(as.matrix(cp$aucTable),metricname="ROC AUC",thesets=thesets,themethod=theMethod,main="ROC AUC",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "ROC AUC",round = 3)

ROC AUC

	Default Classifier	Filtered SVM Classifier
B:SWiMS	0.803	0.812
LASSO	0.771	0.749
RF	0.774	0.874
SVM	0.814	0.814

pander::pander(bp$ciTable,caption = "ROC AUC with 95%CI",round = 3)

ROC AUC with 95%CI

	ROC AUC	lower	upper
Default Classifier	0.803	0.689	0.868
Default Classifier	0.771	0.688	0.848
Default Classifier	0.774	0.655	0.85
Default Classifier	0.814	0.748	0.885
Filtered SVM Classifier	0.812	0.699	0.875
Filtered SVM Classifier	0.749	0.657	0.817
Filtered SVM Classifier	0.874	0.802	0.916
Filtered SVM Classifier	0.814	0.748	0.869

1.3 Analysis of selected features

gain <- length(ArceneFRESA$BSWiMS.models$formula.list)/10
gplots::heatmap.2(gain*ArceneFRESA$BSWiMS.models$bagging$formulaNetwork[1:30,1:30],trace = "none",mar = c(10,10),main = "B:SWiMS Formula Network")

pander::pander(summary(ArceneFRESA$BSWiMS.model,caption = "arcene",round = 3))

• coefficients:

  Table continues below

  	Estimate	lower	OR	upper	u.Accuracy	r.Accuracy
  V4352	-0.003229	0.995	0.9968	0.9986	0.62	0.837
  V584	0.000225	1	1	1	0.65	0.825
  V8726	-0.0005385	0.9992	0.9995	0.9997	0.625	0.875
  V729	-0.003277	0.9957	0.9967	0.9978	0.57	0.8011
  V4038	0.0002351	1	1	1	0.68	0.835
  V8247	-0.0002315	0.9997	0.9998	0.9999	0.565	0.855
  V8310	0.0006965	1	1.001	1.001	0.54	0.825
  V5005	0.001066	1	1.001	1.002	0.695	0.84
  V9215	-0.002501	0.9965	0.9975	0.9985	0.58	0.8056
  V38	-0.001583	0.9976	0.9984	0.9993	0.56	0.875
  V241	-0.0001381	0.9998	0.9999	0.9999	0.585	0.775
  V7748	-0.003899	0.9945	0.9961	0.9977	0.66	0.8085
  V911	0.0001814	1	1	1	0.625	0.81
  V3365	-0.002025	0.9972	0.998	0.9987	0.685	0.8
  V9818	-0.002981	0.9958	0.997	0.9983	0.57	0.81
  V9275	-0.003258	0.9955	0.9967	0.998	0.565	0.803
  V4584	-0.002772	0.9961	0.9972	0.9984	0.51	0.8206
  V3778	-0.0007018	0.9989	0.9993	0.9997	0.56	0.8375
  V8313	-0.04837	0.9291	0.9528	0.9771	0.56	0.86
  V8245	0.001022	1.001	1.001	1.001	0.58	0.8583
  V8650	0.0003933	1	1	1.001	0.6	0.825
  V312	-0.001696	0.9978	0.9983	0.9988	0.68	0.794
  V4194	-0.0004284	0.9993	0.9996	0.9998	0.655	0.79
  V381	0.0003679	1	1	1	0.59	0.835
  V3161	0.002163	1.001	1.002	1.003	0.585	0.8387
  V3189	-0.0002182	0.9997	0.9998	0.9999	0.635	0.845
  V1367	0.0002186	1	1	1	0.63	0.9
  V698	-0.002749	0.996	0.9973	0.9985	0.65	0.7922
  V8368	-0.001374	0.9981	0.9986	0.9991	0.695	0.7933
  V9458	-3.394e-05	0.9999	1	1	0.56	0.83
  V6584	-0.003209	0.9956	0.9968	0.998	0.535	0.81
  V9395	-0.00331	0.9953	0.9967	0.9981	0.555	0.7965
  V1831	-0.002748	0.9963	0.9973	0.9983	0.59	0.8031
  V7284	5.35e-05	1	1	1	0.59	0.83
  V2309	-0.001113	0.9984	0.9989	0.9994	0.695	0.8222
  V3082	-0.0001587	0.9998	0.9998	0.9999	0.64	0.805
  V1289	0.0001543	1	1	1	0.555	0.795
  V4564	-0.001596	0.9976	0.9984	0.9992	0.56	0.845
  V2657	-0.0003675	0.9994	0.9996	0.9998	0.525	0.8525
  V1476	-0.003677	0.9946	0.9963	0.998	0.655	0.8156
  V3987	0.000181	1	1	1	0.63	0.785
  V8604	4.41e-05	1	1	1	0.57	0.755
  V2397	-0.03404	0.951	0.9665	0.9823	0.56	0.835
  V1975	-0.001697	0.9976	0.9983	0.9991	0.68	0.7856
  V1202	0.0004567	1	1	1.001	0.57	0.84
  V6959	-0.001401	0.998	0.9986	0.9992	0.58	0.8061
  V3170	-0.004454	0.9933	0.9956	0.9979	0.56	0.7944
  V4960	-0.001636	0.9977	0.9984	0.9991	0.7	0.8381
  V6114	-0.0001802	0.9997	0.9998	0.9999	0.54	0.78
  V2266	0.001195	1.001	1.001	1.002	0.56	0.795
  V6111	-0.001175	0.9983	0.9988	0.9994	0.54	0.85
  V5808	0.0001905	1	1	1	0.67	0.845
  V4595	0.0002854	1	1	1	0.64	0.835
  V6688	-0.002287	0.9966	0.9977	0.9988	0.555	0.8061
  V5321	-0.006271	0.9904	0.9937	0.9971	0.56	0.8405
  V5672	0.001798	1.001	1.002	1.003	0.595	0.8331
  V1184	-0.0007371	0.9988	0.9993	0.9997	0.555	0.8221
  V9180	0.0001237	1	1	1	0.605	0.81
  V5761	-0.002246	0.997	0.9978	0.9985	0.56	0.7775
  V1248	-0.00291	0.9957	0.9971	0.9985	0.58	0.8167
  V4290	-0.003206	0.9955	0.9968	0.9981	0.665	0.8006
  V1173	-0.03915	0.9408	0.9616	0.9829	0.56	0.82
  V7857	-0.002204	0.9968	0.9978	0.9988	0.675	0.7744
  V3206	-0.0005144	0.9992	0.9995	0.9998	0.65	0.86
  V4900	-0.001441	0.9978	0.9986	0.9993	0.65	0.82
  V5400	-0.002778	0.9962	0.9972	0.9982	0.635	0.7983
  V4198	-0.0009394	0.9987	0.9991	0.9994	0.635	0.83
  V4805	0.0004915	1	1	1.001	0.56	0.835
  V9123	-0.0007115	0.999	0.9993	0.9996	0.56	0.82
  V4183	-0.001121	0.9983	0.9989	0.9995	0.675	0.79
  V86	-0.003111	0.9956	0.9969	0.9982	0.675	0.7831
  V5389	0.0001349	1	1	1	0.56	0.775
  V3708	-0.002668	0.9962	0.9973	0.9985	0.54	0.8275
  V414	-0.0008771	0.9987	0.9991	0.9995	0.695	0.7972
  V4326	-0.001937	0.9974	0.9981	0.9987	0.565	0.78
  V6163	-0.001681	0.9976	0.9983	0.9991	0.49	0.8025
  V9617	-0.0008783	0.9987	0.9991	0.9995	0.695	0.8119
  V8156	-0.00903	0.9878	0.991	0.9942	0.58	0.8063
  V2064	-0.00146	0.9978	0.9985	0.9993	0.56	0.8063
  V231	0.0001992	1	1	1	0.61	0.845
  V2294	0.0005299	1	1.001	1.001	0.69	0.8333
  V1936	0.0004955	1	1	1.001	0.69	0.7986
  V4301	-0.004877	0.9928	0.9951	0.9974	0.56	0.8144
  V5412	0.0006626	1	1.001	1.001	0.58	0.86
  V436	-0.002985	0.9959	0.997	0.9982	0.67	0.78
  V4185	-0.001632	0.9976	0.9984	0.9992	0.55	0.7983
  V7402	-0.0007862	0.9988	0.9992	0.9997	0.56	0.815
  V256	-0.00185	0.9973	0.9982	0.999	0.505	0.8107
  V5680	-0.001789	0.9976	0.9982	0.9988	0.56	0.7656
  V5107	0.0002022	1	1	1	0.445	0.82
  V1818	-0.0002699	0.9996	0.9997	0.9999	0.56	0.8
  V2556	0.001084	1.001	1.001	1.002	0.71	0.8225
  V5995	0.0003753	1	1	1.001	0.6	0.795
  V1055	-0.0007926	0.9988	0.9992	0.9996	0.655	0.8025
  V3592	-0.0002495	0.9996	0.9998	0.9999	0.57	0.795
  V8749	-0.001049	0.9984	0.999	0.9995	0.56	0.8525
  V9027	-0.0007321	0.999	0.9993	0.9995	0.69	0.775
  V9031	0.0001451	1	1	1	0.605	0.855
  V2515	-0.00287	0.9959	0.9971	0.9984	0.555	0.8087
  V5801	0.0001998	1	1	1	0.705	0.7933
  V6285	-0.0005736	0.9991	0.9994	0.9997	0.56	0.7975
  V9965	0.005754	1.004	1.006	1.008	0.535	0.785
  V3545	0.0003086	1	1	1	0.59	0.895
  V2242	0.000183	1	1	1	0.58	0.795
  V7544	-4.926e-05	0.9999	1	1	0.56	0.735
  V9585	-0.002703	0.9962	0.9973	0.9984	0.685	0.7833
  V376	-0.0008348	0.9988	0.9992	0.9995	0.69	0.7907
  V1943	-0.0008827	0.9987	0.9991	0.9995	0.55	0.7717
  V4396	-7.793e-05	0.9999	0.9999	1	0.555	0.795
  V8972	-0.0007873	0.999	0.9992	0.9994	0.58	0.855
  V6975	0.0001912	1	1	1	0.625	0.85
  V7136	0.0002573	1	1	1	0.565	0.8025
  V7834	0.001728	1.001	1.002	1.002	0.55	0.875
  V8502	-0.00229	0.9969	0.9977	0.9985	0.665	0.7857
  V5818	-0.0001576	0.9998	0.9998	0.9999	0.59	0.76
  V9213	-0.003598	0.9946	0.9964	0.9982	0.56	0.799
  V5261	0.0001158	1	1	1	0.59	0.81
  V8038	-0.001343	0.998	0.9987	0.9993	0.54	0.805
  V7302	0.002523	1.001	1.003	1.004	0.56	0.805
  V8806	-0.001601	0.9977	0.9984	0.9991	0.67	0.7636
  V3077	-0.0001793	0.9997	0.9998	0.9999	0.615	0.82
  V9735	6.782e-05	1	1	1	0.685	0.7817
  V7891	0.0008476	1.001	1.001	1.001	0.685	0.788
  V7272	-0.004108	0.9941	0.9959	0.9977	0.66	0.83
  V8055	-0.0009857	0.9987	0.999	0.9993	0.55	0.7583
  V7628	-0.0002309	0.9997	0.9998	0.9999	0.69	0.85
  V1046	-0.002239	0.9968	0.9978	0.9988	0.56	0.803
  V9704	-0.0002038	0.9997	0.9998	0.9999	0.56	0.815
  V4189	0.000597	1	1.001	1.001	0.59	0.8
  V3014	-0.00336	0.9956	0.9966	0.9977	0.52	0.7733
  V8428	-6.677e-05	0.9999	0.9999	1	0.56	0.845
  V5417	-0.0009829	0.9987	0.999	0.9994	0.57	0.7542
  V6510	0.0001291	1	1	1	0.64	0.82
  V7984	-7.864e-05	0.9999	0.9999	1	0.56	0.815
  V2644	9.02e-05	1	1	1	0.645	0.68
  V762	0.001813	1.001	1.002	1.003	0.59	0.794
  V5	-0.0005269	0.9992	0.9995	0.9997	0.665	0.7667
  V4070	7.531e-05	1	1	1	0.69	0.78
  V2636	0.000279	1	1	1	0.56	0.785
  V7994	-0.001273	0.9983	0.9987	0.9992	0.695	0.78
  V7513	-0.0004128	0.9994	0.9996	0.9997	0.63	0.775
  V6958	-0.000157	0.9998	0.9998	0.9999	0.55	0.76
  V8517	0.0004858	1	1	1.001	0.53	0.7858
  V4158	-0.0008208	0.9988	0.9992	0.9995	0.65	0.815
  V9070	-0.0007899	0.9988	0.9992	0.9996	0.575	0.7925
  V7848	-0.0004472	0.9993	0.9996	0.9998	0.56	0.8
  V5473	-0.0008648	0.9988	0.9991	0.9995	0.665	0.765
  V4406	-0.0004664	0.9993	0.9995	0.9997	0.59	0.8183
  V4077	-7.37e-05	0.9999	0.9999	1	0.59	0.78
  V3771	5.946e-05	1	1	1	0.56	0.8
  V130	-0.00298	0.9955	0.997	0.9985	0.56	0.7788
  V2866	-0.000366	0.9995	0.9996	0.9998	0.685	0.78
  V22	0.001314	1.001	1.001	1.002	0.605	0.795
  V4522	0.0008308	1.001	1.001	1.001	0.57	0.835
  V3518	0.0003851	1	1	1.001	0.61	0.83
  V6408	0.0002553	1	1	1	0.565	0.84
  V898	-0.001234	0.9983	0.9988	0.9992	0.535	0.774
  V4069	-0.001291	0.9983	0.9987	0.9992	0.585	0.7758
  V8585	-0.0001983	0.9997	0.9998	0.9999	0.68	0.815
  V5038	-0.001578	0.9977	0.9984	0.9991	0.56	0.8
  V543	0.0002997	1	1	1	0.56	0.805
  V5811	-0.0003784	0.9995	0.9996	0.9998	0.695	0.772
  V9805	6.206e-05	1	1	1	0.585	0.78
  V3397	0.0001495	1	1	1	0.485	0.81
  V3692	-9.063e-05	0.9999	0.9999	0.9999	0.705	0.745
  V872	-0.00128	0.9982	0.9987	0.9993	0.675	0.8083
  V2438	-0.0002628	0.9996	0.9997	0.9999	0.56	0.78
  V3826	7.081e-05	1	1	1	0.595	0.785
  V723	-0.0006798	0.999	0.9993	0.9996	0.65	0.755
  V1097	-0.001195	0.9984	0.9988	0.9992	0.505	0.8017
  V5683	-0.0004635	0.9993	0.9995	0.9998	0.555	0.776
  V5721	-0.001379	0.998	0.9986	0.9992	0.56	0.7917
  V5378	-0.0001927	0.9997	0.9998	0.9999	0.505	0.785
  V6780	-0.0002067	0.9997	0.9998	0.9999	0.55	0.785
  V1144	0.0004821	1	1	1.001	0.56	0.81
  V2256	-0.001041	0.9986	0.999	0.9993	0.585	0.7675
  V9659	-0.0006409	0.9991	0.9994	0.9996	0.685	0.755
  V3339	-0.000356	0.9995	0.9996	0.9998	0.635	0.79
  V7196	6.39e-05	1	1	1	0.625	0.77
  V2747	0.0001241	1	1	1	0.54	0.7617
  V7220	-0.001881	0.9973	0.9981	0.999	0.68	0.78
  V7197	-0.0002963	0.9996	0.9997	0.9998	0.645	0.735
  V7062	-7.209e-05	0.9999	0.9999	1	0.57	0.78
  V4580	-0.001618	0.9977	0.9984	0.9991	0.56	0.7933
  V6928	-0.0003262	0.9995	0.9997	0.9999	0.64	0.83
  V4973	-0.001118	0.9985	0.9989	0.9993	0.685	0.7875
  V2358	0.0009184	1	1.001	1.001	0.585	0.8133
  V782	-0.0003762	0.9995	0.9996	0.9997	0.695	0.645
  V8411	-0.002771	0.9963	0.9972	0.9982	0.685	0.785
  V4289	0.0002936	1	1	1	0.56	0.78
  V4621	0.0004207	1	1	1.001	0.56	0.83
  V2168	-0.0001137	0.9998	0.9999	0.9999	0.545	0.75
  V7195	0.0001205	1	1	1	0.575	0.825
  V853	0.001699	1.001	1.002	1.002	0.56	0.805
  V2818	-0.0004952	0.9993	0.9995	0.9997	0.7	0.7617
  V2856	-0.0004648	0.9993	0.9995	0.9998	0.56	0.815
  V34	-0.001001	0.9986	0.999	0.9994	0.57	0.7787
  V9833	-0.0002156	0.9997	0.9998	0.9999	0.555	0.8
  V7674	0.0002963	1	1	1	0.56	0.785
  V3591	0.0001894	1	1	1	0.625	0.7675
  V6164	-0.0001628	0.9998	0.9998	0.9999	0.48	0.785
  V7584	-0.001591	0.9977	0.9984	0.9991	0.545	0.8325
  V1762	-0.0005671	0.9991	0.9994	0.9997	0.58	0.8125
  V1198	-0.0003833	0.9995	0.9996	0.9997	0.53	0.77
  V7319	-0.001278	0.9983	0.9987	0.9992	0.48	0.8
  V6597	-0.0002776	0.9996	0.9997	0.9999	0.56	0.77
  V2145	0.000995	1.001	1.001	1.001	0.455	0.875
  V6594	0.0007944	1.001	1.001	1.001	0.665	0.81
  V8621	-0.0006479	0.9991	0.9994	0.9996	0.565	0.855

  Table continues below

  	full.Accuracy	u.AUC	r.AUC	full.AUC	IDI	NRI
  V4352	0.8585	0.6266	0.8355	0.8569	0.05576	0.8459
  V584	0.86	0.6291	0.8243	0.8567	0.0743	0.4708
  V8726	0.87	0.6299	0.8762	0.8693	0.06857	1.115
  V729	0.8617	0.5625	0.8006	0.8601	0.1554	0.9944
  V4038	0.86	0.6765	0.8356	0.858	0.07251	0.7208
  V8247	0.88	0.5617	0.8523	0.877	0.06415	0.7419
  V8310	0.88	0.5089	0.8206	0.877	0.1009	1.041
  V5005	0.856	0.696	0.8402	0.8556	0.05488	0.7505
  V9215	0.8475	0.586	0.8053	0.8474	0.1066	0.8409
  V38	0.88	0.5	0.8787	0.8819	0.0559	1.182
  V241	0.815	0.5649	0.7748	0.8105	0.1006	0.8231
  V7748	0.853	0.6684	0.8065	0.8521	0.09573	0.9943
  V911	0.88	0.5982	0.8109	0.8831	0.1057	0.6315
  V3365	0.8489	0.6859	0.7995	0.8482	0.121	0.8774
  V9818	0.84	0.5747	0.8087	0.8397	0.09874	0.8763
  V9275	0.8475	0.5678	0.8026	0.8469	0.1196	0.8633
  V4584	0.8567	0.4968	0.8203	0.8559	0.1023	0.9176
  V3778	0.8575	0.5	0.836	0.8563	0.04957	0.5333
  V8313	0.88	0.5	0.8616	0.8831	0.05061	1.354
  V8245	0.8917	0.5507	0.8573	0.8927	0.07457	0.8988
  V8650	0.8525	0.5686	0.8212	0.8525	0.04908	0.4529
  V312	0.843	0.6814	0.7912	0.8424	0.1565	1.018
  V4194	0.82	0.6798	0.7912	0.8222	0.05858	0.7021
  V381	0.9	0.5718	0.832	0.8961	0.1263	0.8117
  V3161	0.8794	0.5601	0.8356	0.8775	0.07904	0.8417
  V3189	0.87	0.6388	0.8433	0.8705	0.06958	0.7468
  V1367	0.9	0.6027	0.8973	0.8961	0.04224	0.6218
  V698	0.8456	0.6558	0.7904	0.8456	0.08899	0.9888
  V8368	0.8344	0.6985	0.7911	0.8331	0.1228	0.8516
  V9458	0.85	0.5	0.8287	0.849	0.04764	0.7955
  V6584	0.8511	0.5325	0.8114	0.8512	0.1212	0.8357
  V9395	0.841	0.5722	0.7957	0.8406	0.09549	0.663
  V1831	0.8425	0.5974	0.8032	0.8425	0.123	0.8741
  V7284	0.855	0.5718	0.8336	0.8547	0.06577	0.6039
  V2309	0.8467	0.6985	0.8219	0.8463	0.08826	0.8637
  V3082	0.835	0.6445	0.804	0.8369	0.05574	0.9318
  V1289	0.815	0.5223	0.7963	0.8141	0.06124	0.6396
  V4564	0.8678	0.5	0.8438	0.866	0.05768	0.8297
  V2657	0.8775	0.4785	0.8519	0.8772	0.05431	0.7362
  V1476	0.8578	0.664	0.8153	0.8575	0.07902	0.92
  V3987	0.855	0.61	0.791	0.8584	0.09898	0.8401
  V8604	0.76	0.5491	0.7496	0.7565	0.07053	0.4821
  V2397	0.855	0.5	0.8381	0.8584	0.05183	1.269
  V1975	0.82	0.6997	0.7844	0.8207	0.0707	0.7668
  V1202	0.855	0.5223	0.8438	0.8584	0.04922	0.556
  V6959	0.8494	0.569	0.8046	0.8487	0.1065	0.8164
  V3170	0.8239	0.5	0.7958	0.8248	0.04622	0.6872
  V4960	0.8575	0.7041	0.8375	0.8574	0.09346	0.9606
  V6114	0.81	0.5479	0.7804	0.8072	0.06682	0.7516
  V2266	0.895	0.5	0.7938	0.8965	0.1435	1.089
  V6111	0.8775	0.554	0.8502	0.8778	0.06894	0.7492
  V5808	0.895	0.67	0.8446	0.8965	0.07519	0.5812
  V4595	0.8575	0.6043	0.8332	0.8557	0.1204	0.8093
  V6688	0.8339	0.5722	0.8058	0.8337	0.07125	0.5332
  V5321	0.8575	0.5	0.8394	0.8556	0.05986	0.8575
  V5672	0.8556	0.5629	0.8321	0.8539	0.05606	0.4444
  V1184	0.8471	0.5199	0.8231	0.8479	0.04879	0.5631
  V9180	0.82	0.5804	0.8072	0.821	0.09566	0.7792
  V5761	0.8375	0.5499	0.7774	0.8365	0.1395	0.6782
  V1248	0.8461	0.5946	0.8163	0.8453	0.07382	0.6423
  V4290	0.845	0.6692	0.7989	0.8432	0.1026	1.037
  V1173	0.835	0.5	0.8174	0.832	0.04696	0.9838
  V7857	0.83	0.683	0.7718	0.8298	0.06974	0.8308
  V3206	0.88	0.6741	0.8555	0.877	0.05935	1.071
  V4900	0.8475	0.6534	0.8202	0.8466	0.07035	0.8393
  V5400	0.8683	0.6607	0.7976	0.8703	0.1073	0.987
  V4198	0.85	0.6437	0.83	0.8484	0.09232	0.9651
  V4805	0.875	0.5	0.8332	0.875	0.1058	1.097
  V9123	0.85	0.5	0.8162	0.8478	0.07351	1.174
  V4183	0.8157	0.6928	0.7892	0.817	0.04679	0.6445
  V86	0.8406	0.683	0.7826	0.8404	0.1022	0.9478
  V5389	0.79	0.5	0.7748	0.7894	0.04088	0.3847
  V3708	0.8663	0.554	0.8265	0.8651	0.08941	0.8953
  V414	0.8267	0.6997	0.7958	0.8255	0.08981	0.8957
  V4326	0.8444	0.5532	0.7788	0.8429	0.1457	0.7311
  V6163	0.8475	0.4752	0.8005	0.8468	0.07944	0.7626
  V9617	0.8356	0.6972	0.8106	0.8348	0.09118	0.8397
  V8156	0.8413	0.586	0.8055	0.8406	0.1286	0.9243
  V2064	0.845	0.5718	0.8063	0.8458	0.06374	0.7167
  V231	0.89	0.5958	0.8458	0.8908	0.1121	1.297
  V2294	0.865	0.6843	0.8329	0.864	0.0848	0.974
  V1936	0.8379	0.6818	0.7981	0.8375	0.07062	0.6092
  V4301	0.8387	0.5	0.8151	0.8382	0.05573	0.6301
  V5412	0.89	0.558	0.8604	0.8908	0.08448	1.11
  V436	0.8356	0.6786	0.7782	0.8331	0.1046	0.9243
  V4185	0.8317	0.5666	0.7992	0.8314	0.07001	0.5054
  V7402	0.8383	0.5	0.8186	0.8415	0.04023	0.5657
  V256	0.8386	0.4935	0.8106	0.8385	0.08884	0.7526
  V5680	0.8163	0.545	0.7675	0.8162	0.1397	0.6905
  V5107	0.835	0.4351	0.8186	0.832	0.04422	0.5714
  V1818	0.825	0.5	0.7995	0.8212	0.05343	0.6429
  V2556	0.855	0.7045	0.822	0.8553	0.08107	0.9272
  V5995	0.84	0.5637	0.7963	0.8413	0.06597	0.3799
  V1055	0.8375	0.6798	0.8048	0.8403	0.07013	0.8835
  V3592	0.84	0.5832	0.7938	0.8413	0.04943	0.6136
  V8749	0.8825	0.5	0.8519	0.8817	0.05226	0.5065
  V9027	0.8258	0.6964	0.7746	0.8248	0.1453	0.9694
  V9031	0.885	0.5864	0.8535	0.8851	0.07045	0.5081
  V2515	0.84	0.5686	0.8093	0.8396	0.0805	0.5317
  V5801	0.8158	0.7062	0.7942	0.8157	0.07234	0.6212
  V6285	0.8275	0.5	0.7991	0.8265	0.06127	0.5345
  V9965	0.8343	0.5373	0.7863	0.8346	0.1161	0.8173
  V3545	0.925	0.5694	0.8965	0.9257	0.05216	0.9399
  V2242	0.815	0.5593	0.7975	0.8159	0.08507	0.655
  V7544	0.785	0.5	0.7403	0.7873	0.05095	0.3782
  V9585	0.8317	0.7041	0.7838	0.8355	0.09263	0.7386
  V376	0.8243	0.6928	0.7907	0.8238	0.1133	0.7985
  V1943	0.8183	0.5666	0.771	0.8171	0.09099	0.6964
  V4396	0.815	0.5442	0.7987	0.8153	0.05603	0.5455
  V8972	0.925	0.558	0.8547	0.9257	0.1458	1.278
  V6975	0.85	0.597	0.849	0.8515	0.06496	0.4529
  V7136	0.8575	0.5276	0.8011	0.8575	0.08292	0.6761
  V7834	0.925	0.5239	0.8738	0.9257	0.08558	0.8328
  V8502	0.8357	0.6692	0.7829	0.8357	0.1211	1.049
  V5818	0.865	0.5718	0.7565	0.8636	0.2023	0.8799
  V9213	0.824	0.5	0.7991	0.8246	0.05149	0.7614
  V5261	0.84	0.5475	0.8072	0.8389	0.03601	0.5227
  V8038	0.8437	0.5491	0.8061	0.8447	0.07224	0.7963
  V7302	0.8633	0.5	0.808	0.8666	0.09869	0.9995
  V8806	0.8036	0.6786	0.7626	0.8032	0.08914	0.8421
  V3077	0.845	0.6234	0.8186	0.8433	0.08205	0.8961
  V9735	0.7967	0.6847	0.7823	0.7965	0.05333	0.5758
  V7891	0.85	0.6871	0.7873	0.85	0.1243	0.9536
  V7272	0.865	0.683	0.8279	0.8651	0.08236	1.021
  V8055	0.8208	0.5361	0.7576	0.8216	0.149	0.7045
  V7628	0.855	0.6976	0.8502	0.8571	0.1034	1.042
  V1046	0.82	0.5	0.8039	0.821	0.06383	0.7951
  V9704	0.845	0.5	0.8129	0.8433	0.04583	0.5714
  V4189	0.865	0.5718	0.8001	0.8679	0.1608	1.088
  V3014	0.8358	0.5008	0.7731	0.8356	0.1534	0.7798
  V8428	0.83	0.5	0.847	0.83	0.04263	0.3912
  V5417	0.8142	0.5613	0.7531	0.815	0.1252	0.5717
  V6510	0.855	0.6213	0.8222	0.8571	0.1198	0.7305
  V7984	0.845	0.5	0.8153	0.8433	0.09113	1.07
  V2644	0.83	0.6331	0.6802	0.836	0.2196	0.9156
  V762	0.828	0.5548	0.7954	0.8267	0.08636	0.4487
  V5	0.81	0.6838	0.7657	0.8117	0.0624	0.6466
  V4070	0.825	0.683	0.778	0.8255	0.08406	0.6234
  V2636	0.83	0.5	0.7873	0.836	0.06481	0.4156
  V7994	0.8367	0.7021	0.7804	0.8371	0.1376	0.9188
  V7513	0.83	0.655	0.7748	0.836	0.09149	0.6039
  V6958	0.83	0.558	0.7577	0.83	0.06332	0.6315
  V8517	0.8217	0.4927	0.7887	0.8215	0.09749	0.6861
  V4158	0.84	0.6729	0.8172	0.8425	0.08606	0.892
  V9070	0.8425	0.5864	0.794	0.8423	0.04853	0.4708
  V7848	0.825	0.5	0.8013	0.8261	0.05285	0.47
  V5473	0.8175	0.668	0.765	0.818	0.08506	0.7854
  V4406	0.8583	0.5816	0.8187	0.8561	0.08216	0.8333
  V4077	0.84	0.5755	0.7768	0.8364	0.07831	0.6526
  V3771	0.825	0.5	0.7995	0.8255	0.03955	0.461
  V130	0.8088	0.5	0.7799	0.8104	0.04943	0.4233
  V2866	0.825	0.6883	0.7789	0.8243	0.09665	0.7179
  V22	0.8217	0.5706	0.7967	0.8237	0.09384	0.4935
  V4522	0.865	0.5759	0.8356	0.8673	0.1068	0.8036
  V3518	0.84	0.5714	0.8251	0.8364	0.08583	0.3604
  V6408	0.865	0.5227	0.8413	0.8673	0.07036	0.763
  V898	0.822	0.5179	0.7746	0.8218	0.1376	0.7412
  V4069	0.8283	0.5905	0.779	0.8281	0.1212	0.6653
  V8585	0.865	0.6912	0.8178	0.8673	0.1053	0.8847
  V5038	0.835	0.5	0.7971	0.8308	0.07097	1.229
  V543	0.8325	0.5	0.804	0.831	0.08096	0.5893
  V5811	0.804	0.6985	0.7721	0.8058	0.0921	0.7695
  V9805	0.81	0.5637	0.7792	0.8097	0.06154	0.3003
  V3397	0.8225	0.4379	0.8121	0.822	0.07478	0.6567
  V3692	0.805	0.7086	0.7382	0.8052	0.1313	0.8718
  V872	0.84	0.6952	0.8086	0.8413	0.07676	0.7684
  V2438	0.81	0.5718	0.7792	0.8097	0.04167	0.4042
  V3826	0.82	0.5775	0.7922	0.8235	0.06562	0.5373
  V723	0.795	0.6729	0.7528	0.7971	0.07975	0.6786
  V1097	0.85	0.4862	0.8034	0.8515	0.1313	0.8523
  V5683	0.816	0.5394	0.7761	0.816	0.0768	0.4153
  V5721	0.8233	0.5	0.7904	0.8212	0.07435	0.7327
  V5378	0.815	0.4911	0.7825	0.8178	0.1029	0.8101
  V6780	0.835	0.5653	0.7861	0.8369	0.05902	0.5714
  V1144	0.82	0.5	0.8157	0.8235	0.07051	0.6558
  V2256	0.8175	0.5905	0.7735	0.8188	0.1122	0.6466
  V9659	0.8175	0.7041	0.7557	0.8182	0.09467	0.6826
  V3339	0.815	0.6644	0.7906	0.8178	0.0664	0.7971
  V7196	0.835	0.608	0.7727	0.8369	0.1223	0.6623
  V2747	0.7983	0.5114	0.7633	0.8009	0.07011	0.4681
  V7220	0.822	0.6887	0.7782	0.8213	0.07693	0.8286
  V7197	0.8075	0.6441	0.736	0.8093	0.1219	0.9383
  V7062	0.78	0.5625	0.7877	0.7841	0.08419	0.4286
  V4580	0.82	0.5	0.798	0.8231	0.06899	0.6001
  V6928	0.855	0.636	0.8336	0.8584	0.04667	0.7273
  V4973	0.835	0.7041	0.7908	0.8375	0.1278	0.9261
  V2358	0.8333	0.5588	0.8175	0.8333	0.06853	0.7159
  V782	0.8	0.7143	0.6526	0.8032	0.2429	0.9627
  V8411	0.8333	0.7041	0.7861	0.8354	0.1158	0.7446
  V4289	0.81	0.567	0.7841	0.8121	0.08593	0.526
  V4621	0.84	0.567	0.8324	0.845	0.03835	0.414
  V2168	0.835	0.5268	0.7476	0.8344	0.1255	0.5325
  V7195	0.84	0.5633	0.8304	0.845	0.05021	0.8701
  V853	0.855	0.5	0.8076	0.8584	0.0967	0.7419
  V2818	0.8117	0.7029	0.7653	0.8144	0.1129	0.7873
  V2856	0.855	0.5	0.8178	0.8584	0.05488	0.8198
  V34	0.82	0.5674	0.7802	0.8216	0.09591	0.5593
  V9833	0.855	0.543	0.8044	0.8584	0.08836	0.7711
  V7674	0.8025	0.5	0.7892	0.803	0.07519	0.3304
  V3591	0.8375	0.6067	0.7675	0.8409	0.1046	0.6705
  V6164	0.81	0.4785	0.7886	0.8084	0.06476	0.1899
  V7584	0.86	0.5597	0.8334	0.858	0.09141	0.8466
  V1762	0.845	0.586	0.8192	0.8488	0.05123	0.4091
  V1198	0.88	0.5207	0.7739	0.8856	0.1764	0.8653
  V7319	0.8375	0.4554	0.8019	0.8373	0.1272	0.7654
  V6597	0.805	0.5	0.7715	0.8088	0.05755	0.3377
  V2145	0.885	0.4367	0.8726	0.8815	0.0814	0.9107
  V6594	0.885	0.6656	0.8097	0.8815	0.2285	1.166
  V8621	0.885	0.5227	0.8547	0.8815	0.1132	0.9042

  	z.IDI	z.NRI
  V4352	3.521	6.895
  V584	3.971	3.415
  V8726	4.013	9.727
  V729	6.046	8.203
  V4038	3.901	5.425
  V8247	4.073	5.764
  V8310	5.112	8.556
  V5005	3.442	5.8
  V9215	4.935	6.81
  V38	3.635	10.46
  V241	4.586	6.332
  V7748	4.682	8.282
  V911	4.765	4.761
  V3365	5.443	7.081
  V9818	4.603	7.255
  V9275	5.195	6.991
  V4584	4.842	7.393
  V3778	3.503	4.238
  V8313	3.764	13.46
  V8245	4.762	7.071
  V8650	3.486	3.276
  V312	6.244	8.617
  V4194	3.731	5.766
  V381	5.515	6.261
  V3161	3.926	6.59
  V3189	4.144	5.842
  V1367	2.972	4.594
  V698	4.414	8.286
  V8368	5.348	6.782
  V9458	3.43	6.22
  V6584	5.333	6.674
  V9395	4.741	5.327
  V1831	5.361	7.189
  V7284	4.105	4.45
  V2309	4.339	6.917
  V3082	3.519	7.751
  V1289	3.639	4.739
  V4564	3.7	6.687
  V2657	3.871	5.657
  V1476	4.271	7.475
  V3987	4.808	6.59
  V8604	3.928	3.489
  V2397	4.123	12.32
  V1975	4.441	6.142
  V1202	3.631	4.431
  V6959	4.775	6.313
  V3170	3.779	6.55
  V4960	4.518	7.865
  V6114	3.84	5.925
  V2266	5.976	9.093
  V6111	4.251	5.902
  V5808	4.265	4.265
  V4595	4.927	6.43
  V6688	4.079	4.152
  V5321	3.68	7.258
  V5672	3.451	3.287
  V1184	3.462	4.529
  V9180	4.214	6.086
  V5761	5.757	5.175
  V1248	4.1	5.081
  V4290	4.88	8.789
  V1173	3.508	8.953
  V7857	4.118	6.636
  V3206	3.421	9.405
  V4900	3.965	6.915
  V5400	5.39	8.662
  V4198	4.755	8.113
  V4805	4.694	9.192
  V9123	4.338	10.5
  V4183	3.528	5.01
  V86	4.791	7.81
  V5389	3.447	2.774
  V3708	4.582	7.467
  V414	4.471	7.147
  V4326	6.033	5.69
  V6163	4.322	5.995
  V9617	4.452	6.606
  V8156	5.51	7.673
  V2064	3.944	5.744
  V231	5.454	11.94
  V2294	4.614	7.907
  V1936	3.767	4.509
  V4301	4.124	5.762
  V5412	4.208	9.364
  V436	5.11	7.555
  V4185	4.007	3.952
  V7402	3.317	4.95
  V256	4.349	5.788
  V5680	5.771	5.326
  V5107	3.481	4.183
  V1818	3.499	5.114
  V2556	4.333	7.425
  V5995	3.798	2.965
  V1055	4.281	7.506
  V3592	3.666	4.849
  V8749	3.955	3.841
  V9027	5.865	7.96
  V9031	4.198	3.687
  V2515	4.421	4.217
  V5801	4.136	4.667
  V6285	3.765	4.207
  V9965	5.28	6.614
  V3545	3.858	7.464
  V2242	4.477	4.872
  V7544	3.377	2.808
  V9585	4.923	5.989
  V376	4.973	6.37
  V1943	4.515	5.472
  V4396	3.441	3.984
  V8972	6.657	11.6
  V6975	3.397	3.276
  V7136	4.559	5.056
  V7834	4.909	6.422
  V8502	5.418	8.87
  V5818	6.903	6.922
  V9213	3.891	7.312
  V5261	2.972	3.89
  V8038	4.161	6.392
  V7302	4.749	8.188
  V8806	4.583	6.794
  V3077	4.153	7.416
  V9735	3.501	4.277
  V7891	5.475	7.979
  V7272	4.483	9.031
  V8055	5.815	5.298
  V7628	4.955	8.693
  V1046	4.321	6.944
  V9704	3.542	4.331
  V4189	6.19	9.703
  V3014	6.211	6.368
  V8428	2.969	2.885
  V5417	5.498	4.273
  V6510	4.788	5.588
  V7984	4.774	9.09
  V2644	7.994	7.282
  V762	4.255	3.498
  V5	4.186	5.06
  V4070	4.556	4.649
  V2636	3.698	3.022
  V7994	5.703	7.492
  V7513	5.094	5.451
  V6958	3.879	4.991
  V8517	4.892	5.177
  V4158	4.678	7.522
  V9070	3.765	3.742
  V7848	3.431	3.668
  V5473	4.443	6.168
  V4406	4.27	6.664
  V4077	4.332	5.034
  V3771	3.355	3.357
  V130	3.906	4.913
  V2866	4.643	5.622
  V22	4.493	3.729
  V4522	5.177	6.191
  V3518	4.208	2.94
  V6408	4.452	5.827
  V898	5.583	5.624
  V4069	5.582	5.161
  V8585	5.415	7.751
  V5038	4.392	11.62
  V543	4.594	4.329
  V5811	4.406	5.919
  V9805	3.625	2.137
  V3397	4.182	4.975
  V3692	5.589	6.935
  V872	4.555	6.412
  V2438	3.399	3.278
  V3826	3.956	3.937
  V723	4.53	5.787
  V1097	5.632	6.627
  V5683	3.988	3.003
  V5721	4.369	6.077
  V5378	4.881	6.211
  V6780	3.773	4.331
  V1144	3.653	4.88
  V2256	5.321	5.025
  V9659	4.962	5.435
  V3339	3.994	6.411
  V7196	4.812	5.038
  V2747	4.065	3.39
  V7220	4.407	6.627
  V7197	5.549	7.568
  V7062	4.144	3.085
  V4580	4.335	5.057
  V6928	3.502	5.554
  V4973	5.751	8.182
  V2358	3.747	5.475
  V782	8.41	7.985
  V8411	5.563	5.702
  V4289	4.197	3.993
  V4621	3.245	2.969
  V2168	5.373	3.89
  V7195	3.695	6.81
  V853	4.829	5.605
  V2818	5.178	6.397
  V2856	3.786	6.666
  V34	4.727	4.286
  V9833	4.61	5.889
  V7674	3.808	2.394
  V3591	4.773	5.057
  V6164	3.801	1.406
  V7584	4.699	7.833
  V1762	3.873	3.146
  V1198	7.062	6.749
  V7319	5.494	5.965
  V6597	4.204	2.811
  V2145	5.076	7.212
  V6594	8.129	10.03
  V8621	5.751	7.104

• Accuracy: 0.93
• tAUC: 0.9302

• bootstrap: