Cnt-Set-Mata Analysis

#=========================================================

Warning messages:
1: In readChar(file, size, TRUE) : truncating string with embedded nuls
2: In readChar(file, size, TRUE) : truncating string with embedded nuls
3: In readChar(file, size, TRUE) : truncating string with embedded nuls
4: In readChar(file, size, TRUE) : truncating string with embedded nuls
5: In readChar(file, size, TRUE) : truncating string with embedded nuls
6: In readChar(file, size, TRUE) : truncating string with embedded nuls
7: In readChar(file, size, TRUE) : truncating string with embedded nuls

# PREAMBLE
#=========================================================

# load the plotting library
suppressMessages(library(ggplot2))
library(gridExtra)
library(tikzDevice)


theme_set(theme_bw())

options(scipen=999)  # turn-off scientific notation like 1e+48

# size of point for scatterplots
POINT_SIZE = 0.5
#POINT_SIZE = 1

# timeout
TIMEOUT = params$timeout
TIMEOUT_VAL = 1.05 * TIMEOUT

# saturate
#TIME_MIN = 0.01 # seconds
TIME_MIN = 0.1 # seconds

BIG_SIZE=3
SMALL_SIZE=2


# FUNCTIONS
read_file <- function(file) {
  filename = paste0(file)
  df <- read.csv2(filename,
                  header=TRUE,
                  sep=";",
                  dec=",",
                  comment.char="",
                  quote="\"",
                  strip.white=TRUE,
                  allowEscapes=FALSE,
                  stringsAsFactors=FALSE)
  
  
  return(df)
}

plot_scatter_log <- function(df, xlab, ylab, xstring=xlab, ystring=ylab) {
  pscat <- ggplot(df, aes_string(x=xlab, y=ylab)) +
    geom_point(size=POINT_SIZE) +
    geom_abline(size=0.1) +
    geom_vline(size=0.1, xintercept=TIMEOUT_VAL, linetype="dashed") +
    geom_hline(size=0.1, yintercept=TIMEOUT_VAL, linetype="dashed") +
    scale_x_log10() +
    scale_y_log10() +
    theme(axis.text.y = element_text(angle = 90, hjust = 0.5)) +
    #coord_fixed(xlim = c(TIME_MIN, TIMEOUT_VAL), ylim = c(0.1, TIMEOUT_VAL)) +
    #coord_fixed(xlim = c(TIME_MIN, TIMEOUT_VAL), ylim = c(TIME_MIN, TIMEOUT_VAL)) +
        coord_fixed(xlim = c(TIME_MIN, TIMEOUT_VAL), ylim = c(TIME_MIN, TIMEOUT_VAL)) +

    labs(
      #title="Title",
      #subtitle="Subtitle",
      x=xstring,
      y=ystring)
  return(pscat)
}

make_tikz <- function(file, picture, width=2.5, height=2.5) {
  font_size <- 1
  tikz(file=file, onefile=T, width=width, height=height)
  plot(picture)
  garbage <- dev.off()
}

make_pdf <- function(file, picture, width=5, height=5) {
  pdf(file=file, onefile=T, width, height)
  plot(picture)
  garbage <- dev.off()
}

df <- read_file(params$file_cmp)

colnames(df)[colnames(df) == "dot.net"] <- "dotnet"
colnames(df)[names(df) == "dot.net.matches"] <- "dotnet.matches"

######################### SANITIZE ###############################
# clean the data
df_new <- df[!grepl("File not found", df$srm.matches),]
print(paste0("Removing ", nrow(df) - nrow(df_new), " lines due to generation of input text"))

[1] "Removing 3214 lines due to generation of input text"

df <- df_new

tools.times <- c("re2g", "cad", "grep", "srm", "dotnet")
tools.times <- c("re2g", "cad", "srm", "dotnet")
tools.matches <- c("re2g.matches", "cad.matches", "grep.matches", "srm.matches", "dotnet.matches")
tools.matches <- c("re2g.matches", "cad.matches", "srm.matches", "dotnet.matches")


# change the type of columns other than the name
for (i in tools.times) {
  df[,i] <- sub(",", ".", df[,i])
  suppressWarnings(df[,i] <- as.numeric(df[,i]))
}

for (i in tools.matches) {
  suppressWarnings(df[,i] <- as.integer(df[,i]))
}

df$src <- as.factor(df$src)

# get rid of extremal values
df[tools.times][is.na(df[tools.times])] <- TIMEOUT_VAL
#df[is.na(df)] <- TIMEOUT_VAL
#df[df == 0.00] <- TIME_MIN
df[,tools.times][df[,tools.times] < TIME_MIN] <- TIME_MIN



# clean the data
#df_new <- df[df$Lines != "ERROR WHILE CONVERTING TO DCA.",]
#print(paste0("Removing ", nrow(df) - nrow(df_new), " lines due to converting to DCA error"))
#df <- df_new



############################## COUNTING TIMEOUTS #######################
timeouts.re2 <- nrow(df[df$re2g == TIMEOUT_VAL,])
timeouts.ca <- nrow(df[df$cad == TIMEOUT_VAL,])
#timeouts.grep <- nrow(df[df$grep == TIMEOUT_VAL,])
timeouts.srm <- nrow(df[df$srm == TIMEOUT_VAL,])
timeouts.dotnet <- nrow(df[df$dotnet == TIMEOUT_VAL,])
timeouts.re2.and.ca <- nrow(df[df$cad == TIMEOUT_VAL & df$re2g == TIMEOUT_VAL,])

# | **Timeouts grep**    | `r timeouts.grep`  |

These are results of the experiments for Counting Set Automata:

File	results-11-05-2020/table-ALL-ondra-processed.csv
Timeout	600 s
TIMEOUT_VAL	630 s
TIME_MIN	0.1
Benchmarks	1111
Timeouts CA	0
Timeouts RE2	3
Timeouts SRM	14
Timeouts .NET	7

df

Summary of benchmarks

df_benches <- data.frame(summary(df$src))
df_benches

Sanity checks

#df$inconsistent <- df$re2g.matches != df$grep.matches | df$re2g.matches != df$srm.matches | df$re2g.matches != df$dotnet.matches | df$re2g.matches != df$cad.matches

df$inconsistent <- df$re2g.matches != df$srm.matches | df$re2g.matches != df$dotnet.matches | df$re2g.matches != df$cad.matches


#df$grep.re2.mismatch <- !is.na(df$re2g.matches) & !is.na(df$grep.matches) & df$re2g.matches != df$grep.matches
#df_grep_re2_mismatch <- df[df$grep.re2.mismatch,]

df$re2.ca.mismatch <- !is.na(df$re2g.matches) & !is.na(df$cad.matches) & df$re2g.matches != df$cad.matches
df_re2_ca_mismatch <- df[df$re2.ca.mismatch,]

# | **grep and RE2 mismatched** | `r nrow(df_grep_re2_mismatch)` |

CA and RE2 mismatched

11

RE2 and CA mismatches

df_re2_ca_mismatch

Scatter Plots

plot.and.tikz <- function(df, xlab, ylab, xstring=xlab, ystring=ylab, width=4, height=width) {
  pic <- plot_scatter_log(df, xlab, ylab, xstring, ystring)
  #make_tikz(paste0("figs/", xlab, "-vs-", ylab, ".tikz"), pic, width, height)
  make_pdf(paste0("figs/", xlab, "-vs-", ylab, ".pdf"), pic, width, height)
  pic
}

plot1 <- plot.and.tikz(df, "re2g", "cad", xstring="RE2 [s]", ystring="CA [s]", width=BIG_SIZE)
#plot2 <- plot.and.tikz(df, "grep", "cad", xstring="grep [s]", ystring="CA [s]")
plot3 <- plot.and.tikz(df, "srm", "cad", xstring="SRM [s]", ystring="CA [s]", width=SMALL_SIZE)
plot4 <- plot.and.tikz(df, "dotnet", "cad", xstring=".NET [s]", ystring="CA [s]", width=SMALL_SIZE)
plot5 <- plot.and.tikz(df, "srm", "re2g")
#plot6 <- plot.and.tikz(df, "grep", "re2g")
plot7 <- plot.and.tikz(df, "dotnet", "re2g")
#plot8 <- plot.and.tikz(df, "srm", "grep")
#plot9 <- plot.and.tikz(df, "dotnet", "grep")
plot10 <- plot.and.tikz(df, "dotnet", "srm")



grid.arrange(plot1, plot3, ncol = 2)

grid.arrange(plot4, plot5, ncol = 2)

grid.arrange(plot7, plot10, ncol = 2)

#grid.arrange(plot1, plot2, ncol = 2)
#grid.arrange(plot3, plot4, ncol = 2)
#grid.arrange(plot5, plot6, ncol = 2)
#grid.arrange(plot7, plot8, ncol = 2)
#grid.arrange(plot9, plot10, ncol = 2)

Histograms

hist1 <- ggplot(df, aes(x=re2g)) +
  geom_histogram(color="blue", fill="lightblue") +
  scale_y_log10()

hist2 <- ggplot(df, aes(x=cad)) + 
  geom_histogram(color="blue", fill="lightblue") +
  scale_y_log10()

hist3 <- ggplot(df, aes(x=srm)) + 
  geom_histogram(color="blue", fill="lightblue") +
  scale_y_log10()

#hist4 <- ggplot(df, aes(x=grep)) + 
#  geom_histogram(color="blue", fill="lightblue") +
#  scale_y_log10()

hist5 <- ggplot(df, aes(x=dotnet)) + 
  geom_histogram(color="blue", fill="lightblue") +
  scale_y_log10()

grid.arrange(hist1, hist2, ncol = 2)

grid.arrange(hist3, hist5, ncol = 2)

#grid.arrange(hist5, ncol = 2)

Finding winners

#df$min <-pmin(df$grep, df$srm, df$re2g, df$dotnet, df$cad)
#df$enemy.min <- pmin(df$grep, df$srm, df$re2g, df$dotnet)
df$min <-pmin(df$srm, df$re2g, df$dotnet, df$cad)
df$enemy.min <- pmin(df$srm, df$re2g, df$dotnet)

#winners.grep <- nrow(df[df$min == df$grep,])
winners.re2 <- nrow(df[df$min == df$re2g,])
winners.ca <- nrow(df[df$min == df$cad,])
winners.srm <- nrow(df[df$min == df$srm,])
winners.dotnet <- nrow(df[df$min == df$dotnet,])

# | **grep**    | `r winners.grep`  |

Winner
CA	72
RE2	961
SRM	27
.NET	54

plot.and.tikz(df, "enemy.min", "cad", xstring="best enemy", ystring="CA [s]")

How much we are better than RE2

df$re2.vs.ca <- df$re2g / df$cad
df_sorted <- df[order(df$re2.vs.ca, decreasing=TRUE),]

#df_sorted[,c("src", "pattern", "file", "re2g", "cad", "re2.vs.ca", "re2.ca.mismatch")]
df_sorted

NA

Experiments with increasing counter value

big <- read_file(params$file_big)
big$re2g <- sub(",", ".", big$re2g)
big$re2g <- as.numeric(big$re2g)

NAs introduced by coercion

big

together <- big[, c("Counter", "re2g")]
names(together)[2] <- "time"
together$approach <- "RE2"

tmp <- big[, c("Counter", "cad")]
names(tmp)[2] <- "time"
tmp$approach <- "CA"
together <- rbind(together, tmp)

tmp <- big[, c("Counter", "srm")]
names(tmp)[2] <- "time"
tmp$approach <- "SRM"
together <- rbind(together, tmp)

tmp <- big[, c("Counter", "dot.net")]
names(tmp)[2] <- "time"
tmp$approach <- ".NET"
together <- rbind(together, tmp)

tmp <- big[, c("Counter", "grep")]
names(tmp)[2] <- "time"
tmp$approach <- "grep"
together <- rbind(together, tmp)

BIG_STEP=100

# remove too many points
together <- together[together$Counter %% BIG_STEP == 0,]

big_plot <- ggplot(data=together, aes(x=Counter, y=time, colour=approach)) +
  geom_line() +
  geom_point(aes(shape=approach)) +
  xlim(NA,1250) +
  ylim(NA,75) +
  geom_hline(size=0.1, yintercept=0, linetype="dashed") +

  theme(legend.position = c(.05, .95),
        legend.justification = c("left", "top"),
        #legend.box.background = element_rect(color="black", size=0.5),
        legend.box.just = "right",
        legend.margin = margin(1, 1, 1, 1),
        legend.title = element_blank()) +
  labs(
      #title="Title",
      #subtitle="Subtitle",
      x="k",
      y="time [s]")
        

  # geom_line(data = big, aes(x = Counter, y = re2g), color = "red") +
  # geom_line(data = big, aes(x = Counter, y = ca), color = "blue") +
  # xlab('counter value') +
  # ylab('time [s]')

#make_tikz(paste0("figs/big_plot.tikz"), big_plot, width=2.7, height=2.7)
make_pdf(paste0("figs/big_plot.pdf"), big_plot, width=BIG_SIZE, height=BIG_SIZE)


plot(big_plot)

# Information about DCAs

# df_dcas = read.csv2(params$file_dca,
#                   header=TRUE,
#                   sep="\t",
#                   dec=".",
#                   comment.char="",
#                   quote="",
#                   strip.white=TRUE,
#                   stringsAsFactors=FALSE)
# 
# # sanitize
# df_dcas$timeouts.classical[is.na(df_dcas$timeouts.classical)] <- 0

#df_dcas

#tms.classical <- df_dcas[df_dcas$timeouts.classical == 1,]

#compute_timeouts <- function(df, col) {
#  tmp <- df[df[, col] == TIMEOUT_VAL,]
#  tmp
#}