library(ggplot2)
library(plyr)
library(dplyr)
## 
## Attaching package: 'dplyr'
## 
## The following objects are masked from 'package:plyr':
## 
##     arrange, desc, failwith, id, mutate, summarise, summarize
## 
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## 
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(reshape2)
library(xtable)

Fastutil Microbenchmarks

I wrote a microbenchmark using JMH that measures fast and slow iteration over a Fastutil Long2DoubleMap. Each operation is a single pass over a 1000-element map.

Results:

JVM Fast (ns/op) Slow (ns/op)
Java 8 3141 5649
Java 6 3225 6632

Each loop is about 1500ns slower on Java 8, and 3400ns (100%) slower on Java 6.

Do we care?

Rating Iteration Microbenchmarks

When iterating over a bunch of ratings, either from a packed DAO or a CSV file, it is almost a tossup and depends on which JVM is in use whether fast iteration is actually faster. It seems to be slightly slower on average.

bench.data = read.csv('results.csv')

ggplot(bench.data) +
  aes(x=Test, y=Speed) +
  geom_point() +
  geom_errorbar(aes(ymin=Speed-Err, ymax=Speed+Err, width=0.25)) +
  facet_grid(JDK ~ .) +
  ylab("Time (ms)")

FunkSVD Iteration

I ran some evaluations of a basic FunkSVD recommender (40 features, 125 iterations per feature) with fast iteration turned off (made it equivalent to normal iteration in PackedPreferenceSnapshot).

data.fast = read.csv("eval-fast.csv")
data.unfast = read.csv("eval-unfast.csv")

data.fast.j6 = read.csv("eval-fast-j6.csv")
data.unfast.j6 = read.csv("eval-unfast-j6.csv")


data.merged = rbind(mutate(data.fast, mode='fast', jvm='1.8'),
                    mutate(data.unfast, mode='unfast', jvm='1.8'),
                    mutate(data.fast.j6, mode='fast', jvm='1.6'),
                    mutate(data.unfast.j6, mode='unfast', jvm='1.6'))

ggplot(data.merged) +
  aes(x=mode, y=BuildTime / 1000.0) +
  geom_boxplot() +
  facet_grid(DataSet ~ jvm, scales="free_y")

For ML-1M, it take takes about 2 more seconds (<4%) to build each model with fast iteration turned off on Java 8; Java 6 has a much more substantial slowdown. Fortunately, we don’t really care about Java 6.

General Evaluation Performance

Load data from running both fast and unfast, item-item and FunkSVD, on 3 JVMs.

jdks = c('Java6', 'Java7', 'Java8')
modes = c('fast', 'unfast')
conditions = expand.grid(JDK=jdks, Mode=modes)
eval.data = ddply(conditions, .(JDK, Mode), function(cond) {
  fn = sprintf("eval-%s-%s.csv", cond$JDK, cond$Mode)
  read.csv(fn)
})
eval.data = mutate(eval.data, BuildTime=BuildTime/1000.0, TestTime=TestTime/1000.0)

Let’s look at FunkSVD:

ggplot(select(eval.data, Algorithm=='FunkSVD')) +
  aes(x=Mode, y=BuildTime) +
  geom_boxplot() +
  facet_grid(DataSet ~ JDK, scales="free_y") +
  ylab("Build time (s)") +
  ggtitle("FunkSVD training time")

ggplot(select(eval.data, Algorithm=='FunkSVD')) +
  aes(x=Mode, y=TestTime) +
  geom_boxplot() +
  facet_grid(DataSet ~ JDK, scales="free_y") +
  ylab("Test time (s)") +
  ggtitle("FunkSVD testing time")

And Item-Item:

ggplot(select(eval.data, Algorithm=='ItemItem')) +
  aes(x=Mode, y=BuildTime) +
  geom_boxplot() +
  facet_grid(DataSet ~ JDK, scales="free_y") +
  ylab("Build time (s)") +
  ggtitle("Item-Item training time")

ggplot(select(eval.data, Algorithm=='ItemItem')) +
  aes(x=Mode, y=TestTime) +
  geom_boxplot() +
  facet_grid(DataSet ~ JDK, scales="free_y") +
  ylab("Test time (s)") +
  ggtitle("Item-Item testing time")

Let’s analyze speedups. Using gemoetric mean:

gmean = function(xs) {
  exp(mean(log(xs)))
}
eval.summary = summarize(group_by(eval.data, JDK, Mode, Algorithm, DataSet),
                         BuildTime=gmean(BuildTime),
                         TestTime=gmean(TestTime))
eval.wide = reshape(eval.summary, direction='wide',
                    v.names=c('BuildTime', 'TestTime'), 
                    idvar=c('JDK', 'Algorithm', 'DataSet'),
                    timevar='Mode')
eval.wide = mutate(eval.wide,
                   BuildSpeedup = 1 - BuildTime.fast / BuildTime.unfast,
                   TestSpeedup = 1 - TestTime.fast / TestTime.unfast)
build.speedups = dcast(select(eval.wide, JDK, Algorithm, DataSet, BuildSpeedup),
                       Algorithm + DataSet ~ JDK)
## Using BuildSpeedup as value column: use value.var to override.
test.speedups = dcast(select(eval.wide, JDK, Algorithm, DataSet, TestSpeedup),
                       Algorithm + DataSet ~ JDK)
## Using TestSpeedup as value column: use value.var to override.

Let’s just look at that build speed table:

print(xtable(select(eval.wide, JDK, Algorithm, DataSet, BuildTime.unfast, BuildTime.fast)),
      type='html')
JDK Algorithm DataSet BuildTime.unfast BuildTime.fast
1 Java6 FunkSVD ml-100k 2.89 2.41
2 Java6 FunkSVD ml-1m 28.63 25.69
3 Java6 ItemItem ml-100k 1.19 1.19
4 Java6 ItemItem ml-1m 19.90 18.85
5 Java7 FunkSVD ml-100k 2.69 2.41
6 Java7 FunkSVD ml-1m 28.45 24.83
7 Java7 ItemItem ml-100k 1.26 1.12
8 Java7 ItemItem ml-1m 18.75 18.55
9 Java8 FunkSVD ml-100k 2.66 2.41
10 Java8 FunkSVD ml-1m 27.29 25.57
11 Java8 ItemItem ml-100k 1.34 1.15
12 Java8 ItemItem ml-1m 20.05 20.15

Build speedups (geometric mean of each data set and algorithm combination, the fraction of unfast time saved by fast):

print(xtable(build.speedups), type='html')
Algorithm DataSet Java6 Java7 Java8
1 FunkSVD ml-100k 0.17 0.10 0.09
2 FunkSVD ml-1m 0.10 0.13 0.06
3 ItemItem ml-100k 0.01 0.11 0.14
4 ItemItem ml-1m 0.05 0.01 -0.01

On bigger data sets, with newer Java, the speedup goes down (particularly, for item-item, it is 0). Let’s look at test speedups:

print(xtable(test.speedups), type='html')
Algorithm DataSet Java6 Java7 Java8
1 FunkSVD ml-100k 0.38 0.27 0.19
2 FunkSVD ml-1m -0.02 0.24 0.06
3 ItemItem ml-100k 0.06 0.41 0.37
4 ItemItem ml-1m 0.22 0.09 -0.01