Benchmarks for Leukemia Cancer Data Set

1 FRESA.CAD Benchmark

1.1 Leukemia Cancer Data Set

LeukemiaA <- read.delim("./Leukemia/Leukemia.txt")
#LeukemiaA <- read.delim("./Leukemia.txt")
Leukemia <- LeukemiaA[,-1]
rownames(Leukemia) <- LeukemiaA[,1]

LeukFRESA <- FRESA.Model(formula = Status ~ 1,data = Leukemia,repeats = 20)

1.2 Benchmark

reps <- 30;
topIncluded <- 100;

cp <- CVBenchmark(theData = Leukemia, theOutcome = "Status", reps = reps, fraction = 0.80, topIncluded = topIncluded)

elapcol <- names(cp$times[[1]]) == "elapsed"
cputimes <- list(Fresa = mean(cp$times$fresatime[ elapcol ]),LASSO = mean(cp$times$LASSOtime[ elapcol ]),RF = mean(cp$times$RFtime[ elapcol ]),SVM = mean(cp$times$SVMtime[ elapcol ]),FeaLect=mean(cp$times$FeaLecttime[ elapcol ]))

featsize <- list(Fresa = mean(cp$featSize$FRESASize),LASSO = mean(cp$featSize$LASSOSize),Univ = mean(cp$featSize$UNIVSize))

1.2.1 Results

#The Times
pander::pander(cputimes)

• Fresa: 7.232
• LASSO: 0.6897
• RF: 30.03
• SVM: 4.751
• FeaLect: 128.2

pander::pander(featsize)

• Fresa: 13.57
• LASSO: 21.57
• Univ: 1332

plotBEREvolution(cp,40,main="Balanced Error Rate", location="topright")

bp <- barPlotCiError(as.matrix(cp$errorciTable),metricname="Balanced Error",thesets=thesets,themethod=theMethod,main="Balanced Error",offsets = c(0.5,0.05),args.legend = list(x = "topright"))

pander::pander(bp$barMatrix,caption = "Balanced Error Rate",round = 3)

Balanced Error Rate

	Default Classifier	Filtered SVM Classifier
B:SWiMS	0.041	0.081
LASSO	0.051	0.031
RF	0.04	0.041
SVM	0.22	0.031

pander::pander(bp$ciTable,caption = "Balanced Error Rate with 95%CI",round = 3)

Balanced Error Rate with 95%CI

	Balanced Error	lower	upper
Default Classifier	0.041	0.003	0.174
Default Classifier	0.051	0.005	0.187
Default Classifier	0.04	0.005	0.168
Default Classifier	0.22	0.122	0.363
Filtered SVM Classifier	0.081	0.015	0.229
Filtered SVM Classifier	0.031	0.001	0.158
Filtered SVM Classifier	0.041	0.003	0.174
Filtered SVM Classifier	0.031	0.001	0.158

bp <- barPlotCiError(as.matrix(cp$accciTable),metricname="Accuracy",thesets=thesets,themethod=theMethod,main="Accuracy",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "Accuracy",round = 3)

Accuracy

	Default Classifier	Filtered SVM Classifier
B:SWiMS	0.958	0.931
LASSO	0.958	0.972
RF	0.972	0.958
SVM	0.847	0.972

pander::pander(bp$ciTable,caption = "Accuracy with 95%CI",round = 3)

Accuracy with 95%CI

	Accuracy	lower	upper
Default Classifier	0.958	0.883	0.991
Default Classifier	0.958	0.883	0.991
Default Classifier	0.972	0.903	0.997
Default Classifier	0.847	0.743	0.921
Filtered SVM Classifier	0.931	0.845	0.977
Filtered SVM Classifier	0.972	0.903	0.997
Filtered SVM Classifier	0.958	0.883	0.991
Filtered SVM Classifier	0.972	0.903	0.997

bp <- barPlotCiError(as.matrix(cp$aucTable),metricname="ROC AUC",thesets=thesets,themethod=theMethod,main="ROC AUC",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "ROC AUC",round = 3)

ROC AUC

	Default Classifier	Filtered SVM Classifier
B:SWiMS	1	1
LASSO	1	1
RF	1	1
SVM	1	1

pander::pander(bp$ciTable,caption = "ROC AUC with 95%CI",round = 3)

ROC AUC with 95%CI

	ROC AUC	lower	upper
Default Classifier	1	0.821	1
Default Classifier	1	0.909	1
Default Classifier	1	0.75	1
Default Classifier	1	0.841	1
Filtered SVM Classifier	1	0.932	1
Filtered SVM Classifier	1	0.907	1
Filtered SVM Classifier	1	0.889	1
Filtered SVM Classifier	1	0.864	1

1.3 Feature Summaries

gain <- length(LeukFRESA$BSWiMS.models$formula.list)/20
gplots::heatmap.2(gain*LeukFRESA$BSWiMS.models$bagging$formulaNetwork[1:30,1:30],trace="none",mar=c(10,10),main="B:SWiMS Formula Network")

pander::pander(summary(LeukFRESA$BSWiMS.model,caption="Leukemia",round = 3))

• coefficients:

  Table continues below

  	Estimate	lower	OR	upper	u.Accuracy
  M23197_at	0.001803	1.001	1.002	1.002	0.9399
  M27891_at	0.0001896	1	1	1	0.9394
  M31523_at	-0.0008728	0.9987	0.9991	0.9995	0.9356
  X95735_at	0.0001997	1	1	1	0.9365
  M92287_at	-0.0001946	0.9998	0.9998	0.9998	0.9051
  L09209_s_at	0.003881	0.994	1.004	1.014	0.9039
  M11722_at	-0.0002362	0.9997	0.9998	0.9999	0.9184
  U46499_at	0.0002396	1	1	1	0.9119
  M84526_at	0.0002106	1	1	1	0.9184
  D88422_at	0.0004971	1	1	1.001	0.9005
  X68560_at	-0.01333	0.9716	0.9868	1.002	0.7496
  J05243_at	-0.002204	0.9831	0.9978	1.013	0.8983
  X62654_rna1_at	0.0009264	0.9964	1.001	1.005	0.8638
  X59417_at	-0.0001517	0.9997	0.9998	1	0.8936
  M63379_at	0.0001173	1	1	1	0.8522
  U05259_rna1_at	-0.0001417	0.9997	0.9999	1	0.8749
  M96326_rna1_at	1.815e-05	1	1	1	0.8771
  M83652_s_at	0.0001332	1	1	1	0.8759
  M31211_s_at	-8.212e-05	0.9999	0.9999	0.9999	0.8493
  U16954_at	-0.0008586	0.9985	0.9991	0.9998	0.7943
  X51521_at	-0.0001253	0.9998	0.9999	0.9999	0.7851
  X17042_at	1.266e-05	1	1	1	0.8369
  HG1612.HT1612_at	-0.0002277	0.9996	0.9998	0.9999	0.8392
  X85116_rna1_s_at	4.495e-05	1	1	1	0.8014
  M13690_s_at	0.0006337	1.001	1.001	1.001	0.7695
  U77604_at	0.0001655	1	1	1	0.6702
  S76617_at	-0.0001731	0.9998	0.9998	0.9999	0.7926
  U89922_s_at	-4.13e-06	1	1	1	0.7376

  Table continues below

  	r.Accuracy	full.Accuracy	u.AUC	r.AUC	full.AUC
  M23197_at	0.5	0.9399	0.9399	0.5	0.9399
  M27891_at	0.5	0.9394	0.9394	0.5	0.9394
  M31523_at	0.5	0.9356	0.9356	0.5	0.9356
  X95735_at	0.5	0.9365	0.9365	0.5	0.9365
  M92287_at	0.5	0.9051	0.9051	0.5	0.9051
  L09209_s_at	0.5371	0.9188	0.9039	0.5371	0.9188
  M11722_at	0.5	0.9184	0.9184	0.5	0.9184
  U46499_at	0.5	0.9119	0.9119	0.5	0.9119
  M84526_at	0.5	0.9184	0.9184	0.5	0.9184
  D88422_at	0.5	0.9005	0.9005	0.5	0.9005
  X68560_at	0.861	0.9567	0.7496	0.861	0.9567
  J05243_at	0.6582	0.9266	0.8983	0.6582	0.9266
  X62654_rna1_at	0.8007	0.95	0.8638	0.8007	0.95
  X59417_at	0.7009	0.9441	0.8936	0.7009	0.9441
  M63379_at	0.5	0.8522	0.8522	0.5	0.8522
  U05259_rna1_at	0.6246	0.9235	0.8749	0.6246	0.9235
  M96326_rna1_at	0.5	0.8771	0.8771	0.5	0.8771
  M83652_s_at	0.6851	0.9121	0.8759	0.6851	0.9121
  M31211_s_at	0.5	0.8493	0.8493	0.5	0.8493
  U16954_at	0.7819	0.9433	0.7943	0.7819	0.9433
  X51521_at	0.6759	0.8922	0.7851	0.6759	0.8922
  X17042_at	0.5629	0.8635	0.8369	0.5629	0.8635
  HG1612.HT1612_at	0.7237	0.9261	0.8392	0.7237	0.9261
  X85116_rna1_s_at	0.8466	0.9167	0.8014	0.8466	0.9167
  M13690_s_at	0.839	0.9532	0.7695	0.839	0.9532
  U77604_at	0.7955	0.9043	0.6702	0.7955	0.9043
  S76617_at	0.8191	0.9202	0.7926	0.8191	0.9202
  U89922_s_at	0.5	0.7376	0.7376	0.5	0.7376

  	IDI	NRI	z.IDI	z.NRI
  M23197_at	0.8177	1.76	20.28	18.66
  M27891_at	0.8107	1.757	20.79	19.16
  M31523_at	0.798	1.743	19.55	18.3
  X95735_at	0.787	1.746	18.98	18.22
  M92287_at	0.7154	1.621	15.74	13.9
  L09209_s_at	0.6842	1.675	15.68	Inf
  M11722_at	0.7223	1.674	15.89	15.72
  U46499_at	0.6412	1.648	14.69	14.59
  M84526_at	0.728	1.674	16.45	16.49
  D88422_at	0.7214	1.602	15.92	13.99
  X68560_at	0.2802	1.643	6.524	Inf
  J05243_at	0.4839	1.68	9.96	1.873e+306
  X62654_rna1_at	0.3581	1.655	7.59	16.53
  X59417_at	0.4375	1.658	9.158	15.34
  M63379_at	0.5708	1.409	11.09	9.752
  U05259_rna1_at	0.5551	1.66	10.81	19.13
  M96326_rna1_at	0.6108	1.508	12.12	11.49
  M83652_s_at	0.4761	1.416	9.628	10.5
  M31211_s_at	0.5735	1.397	11.42	9.615
  U16954_at	0.3631	1.716	7.266	22.56
  X51521_at	0.4502	1.495	8.781	11.98
  X17042_at	0.5289	1.429	10.44	10.36
  HG1612.HT1612_at	0.4498	1.643	8.949	15.22
  X85116_rna1_s_at	0.2108	1.223	5.001	9
  M13690_s_at	0.2762	1.504	5.94	11.41
  U77604_at	0.276	1.494	5.976	10.97
  S76617_at	0.273	1.496	5.728	12.29
  U89922_s_at	0.3169	0.9504	6.532	5.378

• Accuracy: 1
• tAUC: 1

• bootstrap: