library(raster)
## Loading required package: sp
r <- raster(volcano)
## is land NA, or sea? (threshold here on a value as an example)
r[r < 120] <- NA
plot(r)
## points in the "sea"
pts <- cbind(c(0.2, 1), c(0.1, 0.3))
points(pts)

## find minimum distance to each
xy <- coordinates(r)[!is.na(r[]), ]
plot(r)
## loop over points
len <- numeric(nrow(pts))
idx <- integer(nrow(pts))
for (i in seq_len(nrow(pts))) {
## use longlat = TRUE for ellipsoidal distance, defaults to Cartesian
lens <- sp::spDistsN1(xy, pts[i,,drop = FALSE])
len[i] <- min(lens)
idx[i] <- which.min(lens)
lines(rbind(pts[i,, drop = FALSE], xy[idx[i], , drop = FALSE]))
}
points(xy[idx, , drop = FALSE])
