Rparallelism

Rparallelism

See this original blog

foreach library

knitr::opts_chunk$set(echo = TRUE)
library(foreach)

foreach

foreach(i = 1:3) %do% {
  sqrt(i)
}

## [[1]]
## [1] 1
## 
## [[2]]
## [1] 1.414214
## 
## [[3]]
## [1] 1.732051

lapply(1:3, function(i) {
  sqrt(i)
})

## [[1]]
## [1] 1
## 
## [[2]]
## [1] 1.414214
## 
## [[3]]
## [1] 1.732051

foreach(i = 1:3, .combine = 'c') %do% {
  sqrt(i)
}

## [1] 1.000000 1.414214 1.732051

res <- lapply(1:3, function(i) {
  sqrt(i)
})
do.call('c', res)

## [1] 1.000000 1.414214 1.732051

Using clusters

# Example registering clusters
cl <- parallel::makeCluster(2)
doParallel::registerDoParallel(cl)
foreach(i = 1:3, .combine = 'c') %dopar% {
  sqrt(i)
}

## [1] 1.000000 1.414214 1.732051

parallel::stopCluster(cl)

bigstatsr lib

library(bigstatsr)
mat3 <- FBM(5, 8)
cl <- parallel::makeCluster(2)
doParallel::registerDoParallel(cl)
tmp3 <- foreach(j = 1:8, .combine = 'c') %:%
  foreach(i = 1:5, .combine = 'c') %dopar% {
    mat3[i, j] <- i + j
    NULL
  }
parallel::stopCluster(cl)
mat3[]

##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8]
## [1,]    2    3    4    5    6    7    8    9
## [2,]    3    4    5    6    7    8    9   10
## [3,]    4    5    6    7    8    9   10   11
## [4,]    5    6    7    8    9   10   11   12
## [5,]    6    7    8    9   10   11   12   13

bigstatsr::FBM

mat <- matrix(0, 1e4, 1e4); mat[] <- rnorm(length(mat))
cl <- parallel::makeCluster(2)
doParallel::registerDoParallel(cl)
system.time(
  tmp <- foreach(k = 1:2, .combine = 'c') %dopar% {
    Sys.sleep(1)
    mat[1, 1]
  }
)

##    user  system elapsed 
##   4.666   0.420   8.746

parallel::stopCluster(cl)

mat2 <- FBM(1e4, 1e4); mat2[] <- rnorm(length(mat2))
cl <- parallel::makeCluster(2)
doParallel::registerDoParallel(cl)
system.time(
  tmp <- foreach(k = 1:2, .combine = 'c') %dopar% {
    Sys.sleep(1)
    mat2[1, 1]
  }
)

##    user  system elapsed 
##   0.012   0.002   1.516

parallel::stopCluster(cl)

File lock

So each process uses some lock to perform its incrementation so that the data can???t be changed by some other process in the meantime.

library(flock)
lock <- tempfile()
mat2 <- FBM(1, 1, init = 0)
mat2[]

## [1] 0

cl <- parallel::makeCluster(2)
doParallel::registerDoParallel(cl)
foreach(k = 1:10, .combine = 'c') %dopar% {
  locked <- flock::lock(lock)
  mat2[1, 1] <- mat2[1, 1] + k
  flock::unlock(locked)
  NULL
}

## NULL

cores to use

parallel::detectCores()

## [1] 4

bigstatsr::nb_cores()

## [1] 2