1. General data

metaheuristics <- c("GDE3", "NSGA-II", "NSGA-III", "myCellDE", "myMOCell", "mySPEA2")
heuristics <- c("SIAA", "ScaF", "SchF")
algorithms <- c("GDE3", "NSGA-II", "NSGA-III", "SIAA", "ScaF", "SchF", "myCellDE", "myMOCell", "mySPEA2")
instances <- c("CyberShake_100.xml", "Inspiral_100.xml", "Montage_100.xml", "Sipht_100.xml")
seeds <- seq(1, 30)

2. Timing results (meta-heuristics)

cs <- read.csv("2.analysis/metrics_CyberShake_100.xml.csv") %>% 
  filter(algorithm %in% metaheuristics) %>%
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "my", ""))
## Warning: package 'bindrcpp' was built under R version 3.4.4
li <- read.csv("2.analysis/metrics_Inspiral_100.xml.csv") %>% 
  filter(algorithm %in% metaheuristics) %>%
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "my", ""))
mo <- read.csv("2.analysis/metrics_Montage_100.xml.csv") %>% 
  filter(algorithm %in% metaheuristics) %>%
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "my", ""))
si <- read.csv("2.analysis/metrics_Sipht_100.xml.csv") %>% 
  filter(algorithm %in% metaheuristics) %>%
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "my", ""))

csBest <- cs %>% 
  filter(indicator=="Hypervolume" | indicator=="ThreadTime" | indicator=="WallTime") %>% 
  select(algorithm, median, indicator) %>% 
  spread(indicator, "median") %>%
  arrange(desc(Hypervolume)) %>%
  head(1) %>%
  bind_cols(instance="CyberShake_100.xml", .)
  
liBest <- li  %>% 
  filter(indicator=="Hypervolume" | indicator=="ThreadTime" | indicator=="WallTime") %>% 
  select(algorithm, median, indicator) %>% 
  spread(indicator, "median") %>%
  arrange(desc(Hypervolume)) %>%
  head(1) %>%
  bind_cols(instance="Inspiral_100.xml", .)
  
moBest <- mo  %>% 
  filter(indicator=="Hypervolume" | indicator=="ThreadTime" | indicator=="WallTime") %>% 
  select(algorithm, median, indicator) %>% 
  spread(indicator, "median") %>%
  arrange(desc(Hypervolume)) %>%
  head(1) %>%
  bind_cols(instance="Montage_100.xml", .)
  
siBest <- si  %>% 
  filter(indicator=="Hypervolume" | indicator=="ThreadTime" | indicator=="WallTime") %>% 
  select(algorithm, median, indicator) %>% 
  spread(indicator, "median") %>%
  arrange(desc(Hypervolume)) %>%
  head(1) %>%
  bind_cols(instance="Sipht_100.xml", .)
  
results <- bind_rows(csBest, liBest, moBest, siBest)

results
## # A tibble: 4 x 5
##   instance           algorithm Hypervolume ThreadTime WallTime
##   <chr>              <chr>           <dbl>      <dbl>    <dbl>
## 1 CyberShake_100.xml CellDE          0.842    1.09e12 1173169.
## 2 Inspiral_100.xml   CellDE          0.904    8.34e11  905060.
## 3 Montage_100.xml    GDE3            0.984    8.30e11  926637.
## 4 Sipht_100.xml      CellDE          0.891    9.42e11 1017405.

3. Quality Indicators

Functions

3.1 Summary

Results are grouped by instance and sorted by hypervolume (best first).

indicators <- metrics %>% 
  filter(indicator %in% c("Hypervolume", "AdditiveEpsilonIndicator", "InvertedGenerationalDistance")) %>%
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "my", "")) %>% 
  select(-min, -max, -indifferentAlgorithms) %>%
  mutate(instance=str_replace(instance, "_100.xml", "")) %>%
  group_by(instance, indicator) %>%
  spread(indicator, median) %>%
  arrange(instance, algorithm)

# write latex table
indtable <- indicators %>% 
  select(Instance=instance, Algorithm=algorithm, HV=Hypervolume, AEI=AdditiveEpsilonIndicator, IGD=InvertedGenerationalDistance) %>%
  mutate(HV=round(HV, digits = 3)) %>%
  mutate(AEI=round(AEI, digits = 3)) %>%
  mutate(IGD=round(IGD, digits = 3))

write.table(indtable, file = "scripts/R/tables/indicator-rows.tex", sep = " & ", row.names = FALSE, quote = FALSE, eol = " \\tabularnewline\n")

amount <- 9
indicators %>% 
  filter(row_number() <= amount) %>%
  arrange(instance, desc(Hypervolume))
## # A tibble: 36 x 5
## # Groups:   instance [4]
##    instance   algorithm AdditiveEpsilonIn~ Hypervolume InvertedGeneration~
##    <chr>      <chr>                  <dbl>       <dbl>               <dbl>
##  1 CyberShake CellDE                0.145        0.842               0.276
##  2 CyberShake ScaF                  0.130        0.826               0.401
##  3 CyberShake SIAA                  0.127        0.825               0.400
##  4 CyberShake GDE3                  0.173        0.814               0.180
##  5 CyberShake SchF                  0.149        0.810               0.402
##  6 CyberShake NSGA-III              0.208        0.707               0.371
##  7 CyberShake MOCell                0.202        0.694               0.370
##  8 CyberShake NSGA-II               0.259        0.578               0.436
##  9 CyberShake SPEA2                 0.299        0.559               0.466
## 10 Inspiral   SIAA                  0.0534       0.929               0.185
## # ... with 26 more rows

Hypervolume results

amount <- 9

# hypervolume
hv <- metrics %>% 
  filter(indicator == "Hypervolume") %>%
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "my", "")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "\\[", "\\{")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "\\]", "\\}")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, ";", ",")) %>%
  mutate(instance=str_replace(instance, "_100.xml", "")) %>%
  group_by(instance) %>%
  arrange(instance, desc(median))

# write latex table
hvtable <- hv %>% 
  ungroup() %>%
  mutate(Median=round(median, digits = 3)) %>%
  mutate(Min=round(min, digits = 3)) %>%
  mutate(Max=round(max, digits = 3)) %>%
  select(Instance=instance, Algorithm=algorithm, Median, Min, Max, IndifferentAlgorithms=indifferentAlgorithms) %>% 
  group_by(Instance)

write.table(hvtable, file = "scripts/R/tables/hypervolume-rows.tex", sep = " & ", row.names = FALSE, quote = FALSE, eol = " \\tabularnewline\n")

amount <- 9
hv %>% 
  filter(row_number() <= amount)
## # A tibble: 36 x 7
## # Groups:   instance [4]
##    instance   algorithm indicator   median   min   max indifferentAlgorit~
##    <chr>      <chr>     <fct>        <dbl> <dbl> <dbl> <chr>              
##  1 CyberShake CellDE    Hypervolume  0.842 0.743 0.915 {SIAA, ScaF, SchF,~
##  2 CyberShake ScaF      Hypervolume  0.826 0.767 0.904 {SIAA, CellDE, Sch~
##  3 CyberShake SIAA      Hypervolume  0.825 0.774 0.895 {ScaF, CellDE, Sch~
##  4 CyberShake GDE3      Hypervolume  0.814 0.739 0.955 {SIAA, ScaF, CellD~
##  5 CyberShake SchF      Hypervolume  0.810 0.753 0.916 {SIAA, ScaF, CellD~
##  6 CyberShake NSGA-III  Hypervolume  0.707 0.506 0.864 {MOCell}           
##  7 CyberShake MOCell    Hypervolume  0.694 0.512 0.825 {NSGA-III}         
##  8 CyberShake NSGA-II   Hypervolume  0.578 0.290 0.743 {SPEA2}            
##  9 CyberShake SPEA2     Hypervolume  0.559 0.342 0.665 {NSGA-II}          
## 10 Inspiral   SIAA      Hypervolume  0.929 0.908 0.953 {}                 
## # ... with 26 more rows

Best performing algorithm: GDE3

Note: CellDE has no significant differences with GDE3 in the Inspiral_100 instance

Hypervolume plots

ggplot(data = hv %>% arrange(desc(median)), aes(x=algorithm, y=median, color=algorithm, fill=algorithm, aplha=0.2)) +
  geom_bar(stat="identity") +
  facet_wrap(~instance, nrow = 2) +
  theme(legend.position = "bottom", axis.text.x = element_text(angle = 90, hjust = 1)) +
  guides(alpha = FALSE, color = FALSE)

ggsave("scripts/R/images/hypervolume.eps", last_plot(), device = "eps")
## Saving 7 x 5 in image

Quality indicators

quality <- metrics %>% 
  filter(indicator %in% c("Hypervolume", "AdditiveEpsilonIndicator", "InvertedGenerationalDistance")) %>%
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  mutate(indifferentAlgorithms=str_replace_all(indifferentAlgorithms, "my", "")) %>%
  mutate(instance=str_replace(instance, "_100.xml", "")) %>%
  group_by(instance) %>%
  arrange(instance, desc(median))

ggplot(data = quality, aes(x=algorithm, y=median, color=algorithm, fill=algorithm, aplha=0.2)) +
  geom_bar(stat="identity") +
  facet_grid(indicator~instance, scales = "free_y") +
  theme(legend.position = "bottom", axis.text.x = element_text(angle = 90, hjust = 1)) +
  guides(alpha = FALSE, color = FALSE)

ggsave("scripts/R/images/indicators.eps", last_plot(), device = "eps")
## Saving 7 x 5 in image

Significantly different algorithms

hv %>% filter(indifferentAlgorithms == "[]") %>%
  group_by(instance) %>%
  arrange(instance, desc(median))
## # A tibble: 0 x 7
## # Groups:   instance [0]
## # ... with 7 variables: instance <chr>, algorithm <chr>, indicator <fct>,
## #   median <dbl>, min <dbl>, max <dbl>, indifferentAlgorithms <chr>

4. Pareto front plots

4.1. Reference sets

4.1.1. Scatter plots (combined)

ggplot(data = referenceFronts, aes(x = makespan, y = cost, width = makespan_sd, height = cost_sd, alpha = 0, color = I("blue"))) +  geom_tile() +
  #geom_line() + 
  theme(legend.position = "bottom") +
  guides(alpha = FALSE, color = FALSE) +
  facet_wrap(~instance, ncol = 2, scales = "free")

4.1.2. Scatter plots (per instance)

The 60 solutions that minimize makespan/max(makespan) + cost/max(cost) are selected.

plotReferenceFronts("CyberShake_100.xml")

plotReferenceFronts("Inspiral_100.xml")

plotReferenceFronts("Montage_100.xml")

plotReferenceFronts("Sipht_100.xml")

4.1.3. Parallel coordinates per instance

ggparcoord(data = referenceFronts, columns = 2:5, groupColumn = "instance", scale = "uniminmax")

#  facet_wrap(~instance, ncol = 2)
  
ref <- referenceFronts %>% filter(instance == "CyberShake_100.xml")
ggparcoord(data = ref, columns = 2:5, scale = "uniminmax") + ggtitle(ref$instance)

ref <- referenceFronts %>% filter(instance == "Inspiral_100.xml")
ggparcoord(data = ref, columns = 2:5, scale = "uniminmax") + ggtitle(ref$instance)

ref <- referenceFronts %>% filter(instance == "Montage_100.xml")
ggparcoord(data = ref, columns = 2:5, scale = "uniminmax") + ggtitle(ref$instance)

ref <- referenceFronts %>% filter(instance == "Sipht_100.xml")
ggparcoord(data = ref, columns = 2:5, scale = "uniminmax") + ggtitle(ref$instance)

4.2. Algorithm fronts

For this section only the following algorithms are considered:

  • GDE3
  • ScalingFirst (ScaF)
  • SchedulingFirst (SchF)
  • Spot Instances Aware Autoscaling (SIAA)
plotAlgorithmFronts <- function(selectedInstance) {
  
  referenceFronts <- referenceFronts %>% 
    filter(instance == selectedInstance) %>%
   # arrange(makespan/max(makespan) + cost/max(cost)) %>%
    top_n(n = 30, wt = -(makespan/max(makespan)))
  
  algorithmFronts <- algorithmFronts %>% 
    # filter(algorithm=="GDE3" | algorithm=="myCellDE" | algorithm=="SIAA" | algorithm=="SchF" | algorithm=="ScaF") %>% 
    filter(instance == selectedInstance) %>%
    group_by(algorithm) %>%
    top_n(n = 10, wt = -(makespan/max(makespan)))
    #arrange(makespan/max(makespan) + cost/max(cost)) %>%
    #head(60)
  
  ggplot(data = referenceFronts, aes(x = makespan, y = cost, width = makespan_sd, height = cost_sd, alpha = 0.2)) +
    geom_tile(aes(color = "Reference set"), fill = NA) +
    geom_point(aes(color = "Reference set", shape = "Reference set")) +
    geom_tile(data=algorithmFronts, aes(color = algorithm), fill = NA) + # reference
    geom_point(data=algorithmFronts, aes(color = algorithm, shape = "Algorithm")) + 
    facet_wrap(~algorithm, ncol = 2) +#, scales = "free"
    ggtitle(selectedInstance) +
    theme(legend.position = "bottom") +
    guides(alpha = FALSE) #, color = FALSE
    #facet_wrap(~instance, ncol = 4, scales = "free")
}

4.2.1. Scatter plots

Reference sets are plotted in blue.

plotAlgorithmFronts("CyberShake_100.xml")

plotAlgorithmFronts("Inspiral_100.xml")

plotAlgorithmFronts("Montage_100.xml")

plotAlgorithmFronts("Sipht_100.xml")

4.2.2. Parallel coordinates

selectedAlgorithms <- c("GDE3", "myCellDE", "SIAA", "ScaF", "SchF")
#selectedAlgorithms <- c("myCellDE", "SIAA", "ScaF", "SchF")

alg <- algorithmFronts %>% 
  filter(algorithm %in% selectedAlgorithms) %>% 
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  filter(instance == "CyberShake_100.xml") %>%
  mutate(instance=str_replace(instance, "_100.xml", ""))
p1 <- ggparcoord(data = alg, columns = 3:6, groupColumn=1, scale = "uniminmax") + 
  ggtitle(alg$instance) + 
  theme(legend.position="bottom")

alg <- algorithmFronts %>% 
  filter(algorithm %in% selectedAlgorithms) %>% 
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  filter(instance == "Inspiral_100.xml") %>%
  mutate(instance=str_replace(instance, "_100.xml", ""))
p2 <- ggparcoord(data = alg, columns = 3:6, groupColumn=1, scale = "uniminmax") + 
  ggtitle(alg$instance) + 
   theme(legend.position="bottom")

alg <- algorithmFronts %>% 
  filter(algorithm %in% selectedAlgorithms) %>% 
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  filter(instance == "Montage_100.xml") %>%
  mutate(instance=str_replace(instance, "_100.xml", ""))
p3 <- ggparcoord(data = alg, columns = 3:6, groupColumn=1, scale = "uniminmax") + 
  ggtitle(alg$instance) + 
  theme(legend.position="bottom")
print(min(alg$cost_sd))
## [1] 0
print(max(alg$cost_sd))
## [1] 0.2226688
alg <- algorithmFronts %>% 
  filter(algorithm %in% selectedAlgorithms) %>% 
  mutate(algorithm=str_replace(algorithm, "my", "")) %>%
  filter(instance == "Sipht_100.xml") %>%
  mutate(instance=str_replace(instance, "_100.xml", ""))
p4 <- ggparcoord(data = alg, columns = 3:6, groupColumn=1, scale = "uniminmax") + 
  ggtitle(alg$instance) + 
  theme(legend.position = "bottom")

p1

p2

p3

p4

g <- arrangeGrob(p1, p2, p3, p4, ncol = 2)
ggsave("scripts/R/images/parallel-coordinates.eps", g, device = "eps", width = 10, height = 8, dpi = 150, units = "in")

5. Solutions analysis

# single fronts
# combinations <- expand.grid(algorithm=selectedAlgorithms, instance=instances, seed=seeds) %>% filter(!(algorithm == "SIAA" & instance == "Sipht_100.xml" & (seed == 4 | seed == 28)))

combinations <- expand.grid(algorithm=selectedAlgorithms, instance=instances, seed=seeds) %>% filter(!(algorithm == "GDE3" & instance == "Sipht_100.xml" & (seed == 10 | seed == 19 | seed == 30)))

singleFronts <- bind_rows(apply(combinations, 1, loadSingleFront)) 

singleFrontsVMsGathered <- singleFronts %>%
  gather("instanceType", "amount", 4:11)

singleFrontsBidsGathered <- singleFronts %>%
  mutate(c3.2xlarge_b = ifelse(c3.2xlarge_s > 0, c3.2xlarge_b, -1)) %>%  # set level to -1 if no spot vms
  mutate(m3.2xlarge_b = ifelse(m3.2xlarge_s > 0, m3.2xlarge_b, -1)) %>%
  mutate(m3.medium_b = ifelse(m3.medium_s > 0, m3.medium_b, -1)) %>%
  mutate(r3.xlarge_b = ifelse(r3.xlarge_s > 0, r3.xlarge_b, -1)) %>%
  gather("instanceType", "level", 12:15)

5.1. Histograms of the amount of requested virtual machines per algorithm

# fill = str_split(instanceType, "_")[[1]][2] == "od")
ggplot(singleFrontsVMsGathered %>% filter(amount > 0) %>% filter(), aes(x = amount, fill = instanceType)) + 
  geom_histogram(bins = 10) + 
  facet_wrap(~algorithm, scales = "free") +
  theme(legend.position="bottom")

ggsave("scripts/R/images/vms-algorithm.eps", last_plot(), device = "eps")
## Saving 7 x 5 in image
ggplot(singleFrontsVMsGathered %>% filter(amount > 0) %>% arrange(algorithm), aes(x = amount, color = instanceType)) + 
  geom_freqpoly(bins = 10) + 
  facet_wrap(~algorithm, scales = "free") +
  theme(legend.position="bottom")

ggsave("scripts/R/images/vms-algorithm-poly.eps", last_plot(), device = "eps")
## Saving 7 x 5 in image

5.2. Histograms of the amount of requested virtual machines per instance

ggplot(singleFrontsVMsGathered %>% filter(amount > 0), aes(x = amount, fill = algorithm)) + 
  geom_histogram(bins = 20) + 
  facet_wrap(~instance, scales = "free") +
  theme(legend.position="bottom")

ggsave("scripts/R/images/vms-instance.eps", last_plot(), device = "eps")
## Saving 7 x 5 in image
ggplot(singleFrontsVMsGathered %>% filter(amount > 0), aes(x = amount, color = algorithm)) + 
  geom_freqpoly(bins = 30) + 
  facet_wrap(~instance, scales = "free") +
  theme(legend.position="bottom")

ggsave("scripts/R/images/vms-instance-poly.eps", last_plot(), device = "eps")
## Saving 7 x 5 in image

5.3. Histograms of the amount of times a bid level was selected per algorithm

# fill = str_split(instanceType, "_")[[1]][2] == "od")
ggplot(singleFrontsBidsGathered %>% filter(level >= 0 & level <= 10), aes(x = level, fill = instanceType)) + 
  geom_histogram(bins = 11) + 
  facet_wrap(~algorithm, scales = "free_y") +
  theme(legend.position="bottom")

ggsave("scripts/R/images/bids-algorithm.eps", last_plot(), device = "eps", width = 10, height = 6, dpi = 150, units = "in")


ggplot(singleFrontsBidsGathered %>% filter(level >= 0 & level <= 10) %>% arrange(algorithm), aes(x = level, color = instanceType)) + 
  geom_freqpoly(bins = 11) + 
  facet_wrap(~algorithm, scales = "free") +
  theme(legend.position="bottom")

ggsave("scripts/R/images/bids-algorithm-poly.eps", last_plot(), device = "eps", width = 10, height = 6, dpi = 150, units = "in")

#min(singleFrontsBidsGathered$c3.2xlarge_b)
#max(singleFrontsBidsGathered$c3.2xlarge_b)

6. Correlation between variables

These plots consider the solutions of all of the studied algorithms.

6.1. Raw variables

bidCorrectedSingleFronts <- singleFronts %>%
  mutate(c3.2xlarge_b = ifelse(c3.2xlarge_s > 0, c3.2xlarge_b, NA)) %>%  # set level to NA if no spot vms
  mutate(m3.2xlarge_b = ifelse(m3.2xlarge_s > 0, m3.2xlarge_b, NA)) %>%
  mutate(m3.medium_b = ifelse(m3.medium_s > 0, m3.medium_b, NA)) %>%
  mutate(r3.xlarge_b = ifelse(r3.xlarge_s > 0, r3.xlarge_b, NA))

method <- "spearman"
continuous <- TRUE
if (continuous) {
  p1 <- plotCorrelationMatrix(bidCorrectedSingleFronts, "CyberShake_100.xml", method)
  p2 <- plotCorrelationMatrix(bidCorrectedSingleFronts, "Inspiral_100.xml", method)
  p3 <- plotCorrelationMatrix(bidCorrectedSingleFronts, "Montage_100.xml", method)
  p4 <- plotCorrelationMatrix(bidCorrectedSingleFronts, "Sipht_100.xml", method)
} else {
  p1 <- plotDiscretizedCorrelationMatrix(bidCorrectedSingleFronts, "CyberShake_100.xml", method)
  p2 <- plotDiscretizedCorrelationMatrix(bidCorrectedSingleFronts, "Inspiral_100.xml", method)
  p3 <- plotDiscretizedCorrelationMatrix(bidCorrectedSingleFronts, "Montage_100.xml", method)
  p4 <- plotDiscretizedCorrelationMatrix(bidCorrectedSingleFronts, "Sipht_100.xml", method)
}

p1

p2

p3

p4

g <- arrangeGrob(p1, p2, p3, p4, ncol = 2)
ggsave("scripts/R/images/raw-correlations.eps", g, device = "eps", width = 10, height = 8.5, dpi = 150, units = "in")

6.2. Modified variables

modifiedVariableSingleFronts <- singleFronts %>%
  mutate(c3.2xlarge_SR = c3.2xlarge_s / (c3.2xlarge_s + c3.2xlarge_od + 1)) %>%
  mutate(m3.2xlarge_SR = m3.2xlarge_s / (m3.2xlarge_s + m3.2xlarge_od + 1)) %>%
  mutate(m3.medium_SR = m3.medium_s / (m3.medium_s + m3.medium_od + 1)) %>%
  mutate(r3.xlarge_SR = r3.xlarge_s / (r3.xlarge_s + r3.xlarge_od + 1)) %>%
  # mutate(c3.2xlarge_BR = c3.2xlarge_s / (c3.2xlarge_b + 1)) %>%
  # mutate(m3.2xlarge_BR = m3.2xlarge_s / (m3.2xlarge_b + 1)) %>%
  # mutate(m3.medium_BR = m3.medium_s / (m3.medium_b + 1)) %>%
  # mutate(r3.xlarge_BR = r3.xlarge_s / (r3.xlarge_b + 1)) %>%
  select(-ends_with('_od'), -ends_with('_s'), ends_with('_b'), ends_with('_SR'), makespan, makespan_sd, cost, cost_sd)

method <- "spearman"
if (continuous) {
  p1 <- plotCorrelationMatrix(modifiedVariableSingleFronts, "CyberShake_100.xml", method)
  p2 <- plotCorrelationMatrix(modifiedVariableSingleFronts, "Inspiral_100.xml", method)
  p3 <- plotCorrelationMatrix(modifiedVariableSingleFronts, "Montage_100.xml", method)
  p4 <- plotCorrelationMatrix(modifiedVariableSingleFronts, "Sipht_100.xml", method)
} else {
  p1 <- plotDiscretizedCorrelationMatrix(modifiedVariableSingleFronts, "CyberShake_100.xml", method)
  p2 <- plotDiscretizedCorrelationMatrix(modifiedVariableSingleFronts, "Inspiral_100.xml", method)
  p3 <- plotDiscretizedCorrelationMatrix(modifiedVariableSingleFronts, "Montage_100.xml", method)
  p4 <- plotDiscretizedCorrelationMatrix(modifiedVariableSingleFronts, "Sipht_100.xml", method)
}

p1

p2

p3

p4

g <- arrangeGrob(p1, p2, p3, p4, ncol = 2)
ggsave("scripts/R/images/mod-correlations.eps", g, device = "eps", width = 10, height = 8.5, dpi = 150, units = "in")

6.3 Pair plots of objective functions

plotPairPlots <- function(data, selectedInstance, method) {
  p <- ggpairs(data %>% filter(instance == selectedInstance) %>% select(algorithm, makespan, makespan_sd, cost, cost_sd), 
               aes(color = algorithm, alpha = 0.2), 
               upper = list(continuous = wrap('cor', method = method)), 
               lower = list(continuous = 'cor')) +
    ggtitle(str_split(selectedInstance, "_")[[1]])
  
  return(p)
}


method <- "spearman"
plotPairPlots(modifiedVariableSingleFronts, "CyberShake_100.xml", method)
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

plotPairPlots(modifiedVariableSingleFronts, "Inspiral_100.xml", method)
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

plotPairPlots(modifiedVariableSingleFronts, "Montage_100.xml", method)
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

plotPairPlots(modifiedVariableSingleFronts, "Sipht_100.xml", method)
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

# ggpairs(modifiedVariableSingleFronts %>% filter(instance == "CyberShake_100.xml") %>% select(-instance, -seed), aes(color = algorithm))
# ggpairs(modifiedVariableSingleFronts %>% filter(instance == "Inspiral_100.xml") %>% select(-instance, -seed), aes(color = algorithm))
# ggpairs(modifiedVariableSingleFronts %>% filter(instance == "Montage_100.xml") %>% select(-instance, -seed), aes(color = algorithm))
# ggpairs(modifiedVariableSingleFronts %>% filter(instance == "Sipht_100.xml") %>% select(-instance, -seed), aes(color = algorithm))