library(raster)
## Loading required package: sp
r <- raster(volcano)

## is land NA, or sea? (threshold here on a value as an example)

r[r < 120] <- NA
plot(r)

## points in the "sea"
pts <- cbind(c(0.2, 1), c(0.1, 0.3))
points(pts)

## find minimum distance to each
xy <- coordinates(r)[!is.na(r[]), ]

plot(r)

## loop over points
len <- numeric(nrow(pts))
idx <- integer(nrow(pts))
for (i in seq_len(nrow(pts))) {
  ## use longlat = TRUE for ellipsoidal distance, defaults to Cartesian
  lens <- sp::spDistsN1(xy, pts[i,,drop = FALSE])
  len[i] <- min(lens)
  idx[i] <- which.min(lens)
  lines(rbind(pts[i,, drop = FALSE], xy[idx[i], , drop = FALSE]))
}

points(xy[idx, , drop = FALSE])