# ========================
library(microbenchmark)
library(compiler)
library(plyr)
# Test functions
square <- function (x) x^2
f_for_nofunc <- function(n) for (x in 1:n) x^2 # for loop with no external function call
f_for <- function(n) for (x in 1:n) square(x) # for loop that calls external function
f_lapply <- function(n) lapply(1:n, square)
f_vapply <- function(n) vapply(1:n, square, numeric(1))
# Compiled versions of functions
c_square <- cmpfun(square)
c_for_nofunc <- cmpfun(function(n) for (x in 1:n) x^2)
c_for <- cmpfun(function(n) for (x in 1:n) c_square(x))
c_lapply <- cmpfun(function(n) lapply(1:n, c_square))
c_vapply <- cmpfun(function(n) vapply(1:n, c_square, numeric(1)))
# Run benchmarks for n=10^pow10 cycles
run_benchmarks <- function(pow10) {
n <- 10^pow10
# Run a maximum of 100 iterations, and drop off until there's 1 iteration for 1e7
times <- round(min(1e7/n, 100))
res <- microbenchmark(
f_for_nofunc(n), f_for(n), f_lapply(n), f_vapply(n),
c_for_nofunc(n), c_for(n), c_lapply(n), c_vapply(n),
times = times)
res$size <- n
res
}
# Run benchmarks
res <- ldply(1:7, run_benchmarks)
# Summarize
res_summary <- ddply(res, c("expr", "size"), summarise,
N=length(time), m_time=mean(time), sd=sd(time), se=sd/sqrt(N))
# Calculate time per iteration
res_summary$iter_time <- res_summary$m_time / res_summary$size
# Put compiled and function type into separate columns
res_summary$compiled <- "non-compiled"
res_summary$compiled[grepl("^c_", res_summary$expr)] <- "compiled"
res_summary$type <- sub("^._", "", res_summary$expr)
# print it out
arrange(res_summary, size, compiled, type)
## expr size N m_time sd se iter_time compiled type
## 1 c_for_nofunc(n) 1e+01 100 3.204e+03 5.831e+02 5.831e+01 320.4 compiled for_nofunc(n)
## 2 c_for(n) 1e+01 100 8.976e+03 7.924e+03 7.924e+02 897.6 compiled for(n)
## 3 c_lapply(n) 1e+01 100 1.557e+04 2.045e+03 2.045e+02 1557.3 compiled lapply(n)
## 4 c_vapply(n) 1e+01 100 1.772e+04 2.997e+03 2.997e+02 1771.9 compiled vapply(n)
## 5 f_for_nofunc(n) 1e+01 100 5.818e+03 3.025e+03 3.025e+02 581.8 non-compiled for_nofunc(n)
## 6 f_for(n) 1e+01 100 1.158e+04 2.741e+03 2.741e+02 1158.4 non-compiled for(n)
## 7 f_lapply(n) 1e+01 100 1.686e+04 2.119e+03 2.119e+02 1685.9 non-compiled lapply(n)
## 8 f_vapply(n) 1e+01 100 1.862e+04 2.848e+03 2.848e+02 1861.6 non-compiled vapply(n)
## 9 c_for_nofunc(n) 1e+02 100 2.563e+04 9.893e+04 9.893e+03 256.3 compiled for_nofunc(n)
## 10 c_for(n) 1e+02 100 5.868e+04 1.560e+04 1.560e+03 586.8 compiled for(n)
## 11 c_lapply(n) 1e+02 100 9.575e+04 2.787e+04 2.787e+03 957.5 compiled lapply(n)
## 12 c_vapply(n) 1e+02 100 9.425e+04 2.243e+04 2.243e+03 942.5 compiled vapply(n)
## 13 f_for_nofunc(n) 1e+02 100 3.281e+04 7.794e+03 7.794e+02 328.1 non-compiled for_nofunc(n)
## 14 f_for(n) 1e+02 100 7.605e+04 2.001e+04 2.001e+03 760.5 non-compiled for(n)
## 15 f_lapply(n) 1e+02 100 1.055e+05 2.543e+04 2.543e+03 1054.8 non-compiled lapply(n)
## 16 f_vapply(n) 1e+02 100 1.207e+05 8.997e+04 8.997e+03 1207.4 non-compiled vapply(n)
## 17 c_for_nofunc(n) 1e+03 100 1.348e+05 3.809e+04 3.809e+03 134.8 compiled for_nofunc(n)
## 18 c_for(n) 1e+03 100 5.485e+05 2.510e+05 2.510e+04 548.5 compiled for(n)
## 19 c_lapply(n) 1e+03 100 8.432e+05 2.433e+05 2.433e+04 843.2 compiled lapply(n)
## 20 c_vapply(n) 1e+03 100 8.339e+05 2.113e+05 2.113e+04 833.9 compiled vapply(n)
## 21 f_for_nofunc(n) 1e+03 100 3.092e+05 1.262e+05 1.262e+04 309.2 non-compiled for_nofunc(n)
## 22 f_for(n) 1e+03 100 9.390e+05 2.756e+06 2.756e+05 939.0 non-compiled for(n)
## 23 f_lapply(n) 1e+03 100 9.700e+05 2.873e+05 2.873e+04 970.0 non-compiled lapply(n)
## 24 f_vapply(n) 1e+03 100 9.717e+05 2.717e+05 2.717e+04 971.7 non-compiled vapply(n)
## 25 c_for_nofunc(n) 1e+04 100 1.914e+06 2.442e+06 2.442e+05 191.4 compiled for_nofunc(n)
## 26 c_for(n) 1e+04 100 6.205e+06 3.163e+06 3.163e+05 620.5 compiled for(n)
## 27 c_lapply(n) 1e+04 100 9.235e+06 1.813e+06 1.813e+05 923.5 compiled lapply(n)
## 28 c_vapply(n) 1e+04 100 9.950e+06 5.656e+06 5.656e+05 995.0 compiled vapply(n)
## 29 f_for_nofunc(n) 1e+04 100 3.379e+06 9.725e+05 9.725e+04 337.9 non-compiled for_nofunc(n)
## 30 f_for(n) 1e+04 100 7.552e+06 2.076e+06 2.076e+05 755.2 non-compiled for(n)
## 31 f_lapply(n) 1e+04 100 1.055e+07 1.656e+06 1.656e+05 1055.3 non-compiled lapply(n)
## 32 f_vapply(n) 1e+04 100 1.074e+07 1.935e+06 1.935e+05 1074.1 non-compiled vapply(n)
## 33 c_for_nofunc(n) 1e+05 100 1.666e+07 7.012e+06 7.012e+05 166.6 compiled for_nofunc(n)
## 34 c_for(n) 1e+05 100 6.223e+07 1.160e+07 1.160e+06 622.3 compiled for(n)
## 35 c_lapply(n) 1e+05 100 1.194e+08 2.419e+07 2.419e+06 1194.0 compiled lapply(n)
## 36 c_vapply(n) 1e+05 100 1.014e+08 1.835e+07 1.835e+06 1013.7 compiled vapply(n)
## 37 f_for_nofunc(n) 1e+05 100 3.541e+07 9.246e+06 9.246e+05 354.1 non-compiled for_nofunc(n)
## 38 f_for(n) 1e+05 100 8.256e+07 1.496e+07 1.496e+06 825.6 non-compiled for(n)
## 39 f_lapply(n) 1e+05 100 1.299e+08 2.480e+07 2.480e+06 1298.8 non-compiled lapply(n)
## 40 f_vapply(n) 1e+05 100 1.165e+08 2.008e+07 2.008e+06 1165.2 non-compiled vapply(n)
## 41 c_for_nofunc(n) 1e+06 10 1.681e+08 5.260e+07 1.663e+07 168.1 compiled for_nofunc(n)
## 42 c_for(n) 1e+06 10 5.650e+08 6.245e+07 1.975e+07 565.0 compiled for(n)
## 43 c_lapply(n) 1e+06 10 2.018e+09 2.219e+08 7.017e+07 2017.8 compiled lapply(n)
## 44 c_vapply(n) 1e+06 10 8.968e+08 6.586e+07 2.083e+07 896.8 compiled vapply(n)
## 45 f_for_nofunc(n) 1e+06 10 3.710e+08 8.384e+07 2.651e+07 371.0 non-compiled for_nofunc(n)
## 46 f_for(n) 1e+06 10 8.047e+08 1.401e+08 4.432e+07 804.7 non-compiled for(n)
## 47 f_lapply(n) 1e+06 10 1.815e+09 1.284e+08 4.060e+07 1815.2 non-compiled lapply(n)
## 48 f_vapply(n) 1e+06 10 1.080e+09 1.007e+08 3.183e+07 1080.1 non-compiled vapply(n)
## 49 c_for_nofunc(n) 1e+07 1 1.428e+09 NA NA 142.8 compiled for_nofunc(n)
## 50 c_for(n) 1e+07 1 5.070e+09 NA NA 507.0 compiled for(n)
## 51 c_lapply(n) 1e+07 1 4.239e+10 NA NA 4239.2 compiled lapply(n)
## 52 c_vapply(n) 1e+07 1 9.599e+09 NA NA 959.9 compiled vapply(n)
## 53 f_for_nofunc(n) 1e+07 1 4.312e+09 NA NA 431.2 non-compiled for_nofunc(n)
## 54 f_for(n) 1e+07 1 6.840e+09 NA NA 684.0 non-compiled for(n)
## 55 f_lapply(n) 1e+07 1 2.504e+10 NA NA 2503.8 non-compiled lapply(n)
## 56 f_vapply(n) 1e+07 1 1.018e+10 NA NA 1018.1 non-compiled vapply(n)
# Plot
library(ggplot2)
library(scales)
xlog <- scale_x_log10(breaks = trans_breaks("log10", function(x) 10^x),
labels = trans_format("log10", math_format(10^.x)))
ylog <- scale_y_log10(breaks = trans_breaks("log10", function(x) 10^x),
labels = trans_format("log10", math_format(10^.x)))
p <- ggplot(res_summary, aes(x=size, y=m_time, linetype=compiled, colour=type)) +
geom_line() + geom_point()
# Show with log-log scale
p + xlog + ylog +
expand_limits(y=0) +
xlab("size of data (log)") + ylab("total time (ns) (log)")
# Time per iteration on y, log(size) on x
ggplot(res_summary, aes(x=size, y=iter_time, linetype=compiled, colour=type)) +
geom_line() + geom_point() +
expand_limits(y=0) +
xlog + xlab("size of data (log)") + ylab("Time per iteration (ns)")
