PRML NN

library(nnet)
library(ggplot2)
library(reshape2)

# learnx <- seq(-5, 5, by = 0.01)
learnx <- seq(-1, 1, by = 0.01)
learny1 <- learnx^2
learny2 <- sin(learnx)
learny3 <- abs(learnx)
learny4 <- ifelse(learnx > 1, 1, -1)

NN <- function(x, y, ...) {
    model <- nnet(x = x, y = y, size = 4, ...)
    res <- predict(model, data.frame(x))
}

pred1 <- NN(learnx, learny1, decay = 5e-04)

## # weights:  13
## initial  value 19.045074 
## iter  10 value 18.224799
## iter  20 value 1.665926
## iter  30 value 0.448904
## iter  40 value 0.361256
## iter  50 value 0.349943
## iter  60 value 0.344585
## iter  70 value 0.315504
## iter  80 value 0.313659
## iter  90 value 0.311510
## iter 100 value 0.310968
## final  value 0.310968 
## stopped after 100 iterations

pred2 <- NN(learnx, learny2, decay = 5e-04)

## # weights:  13
## initial  value 125.752990 
## iter  10 value 55.386151
## iter  20 value 55.237336
## iter  30 value 49.088247
## iter  40 value 27.972629
## iter  50 value 27.842837
## iter  60 value 27.795784
## iter  70 value 27.749524
## iter  80 value 27.742124
## iter  90 value 27.739482
## iter 100 value 27.738236
## final  value 27.738236 
## stopped after 100 iterations

pred3 <- NN(learnx, learny3, decay = 5e-04)

## # weights:  13
## initial  value 32.963340 
## iter  10 value 16.885441
## iter  20 value 1.412931
## iter  30 value 0.514083
## iter  40 value 0.426109
## iter  50 value 0.391155
## iter  60 value 0.365444
## iter  70 value 0.364284
## iter  80 value 0.362767
## iter  90 value 0.362219
## iter 100 value 0.362058
## final  value 0.362058 
## stopped after 100 iterations

pred4 <- NN(learnx, learny4, decay = 5e-04)

## # weights:  13
## initial  value 571.893515 
## iter  10 value 201.810329
## iter  20 value 201.042977
## iter  30 value 201.028012
## iter  40 value 201.026530
## iter  40 value 201.026530
## iter  40 value 201.026529
## final  value 201.026529 
## converged

df <- rbind(data.frame(x = learnx, y = learny1, pred = pred1, place = "p1"), 
    data.frame(x = learnx, y = learny2, pred = pred2, place = "p2"), data.frame(x = learnx, 
        y = learny3, pred = pred3, place = "p3"), data.frame(x = learnx, y = learny4, 
        pred = pred4, place = "p4"))
df.m <- melt(df, id.var = c("x", "place"))
ggplot(df.m, aes(x, value)) + geom_point(aes(color = variable)) + 
    facet_wrap(~place, scales = "free")