Statistical Prediction and Optimization

Statistical Prediction and Optimization

Statistical Prediction

Outline

  1. Training and Testing
  2. Cross-validation

Goal: Learn the concepts, terminology, and techniques in statistical prediction!

Part 1: Training and Testing

Recall: Regression

We want to know the relationship between our covariates (X’s) and our response (Y), we can do this with a method called regression. Regression provides us with a statistical method to conduct

Statistical Prediction

Often, we use a linear model even when we know that the “truth” is not a purely linear relationship with normally-distributed errors.

Statistical Prediction

Often, we use a linear model even when we know that the “truth” is not a purely linear relationship with normally-distributed errors.

Does this mean our model is wrong? Yes, in some sense.

Statistical Prediction

Often, we use a linear model even when we know that the “truth” is not a purely linear relationship with normally-distributed errors.

Does this mean our model is wrong? Yes, in some sense.

Does this mean our model is bad? No, not necessarily!

If we consider a linear model only to be a rough approximation, we can still make use of it for statistical prediction!

Regression Model Validation

Model Performance Metrics

Load Packages

 # library(tidyverse), for data manipulation and visualization

 # library(caret), for computing cross-validation methods

Example 1

# Load Data
Focus_New <- Focus %>%
select(GPA, SEX, AGE, MATH, COMP, COLLEGE)

head(Focus_New)
##    GPA SEX AGE MATH COMP COLLEGE
## 1 2.39   F  22   28   26     A&S
## 2 3.37   M  21   23   24     A&S
## 3 2.63   F  22   25   21     Bus
## 4 2.77   M  21   19   19    Educ
## 5 3.79   M  22   28   32     A&S
## 6 3.34   M  21   26   23     A&S

Example 1

set.seed(123)
training <- Focus_New$GPA %>%
    createDataPartition(p = 0.8, list = FALSE)
train.data  <- Focus_New[training, ]
test.data <- Focus_New[-training, ]
training
##        Resample1
##   [1,]         2
##   [2,]         4
##   [3,]         5
##   [4,]         7
##   [5,]         8
##   [6,]        10
##   [7,]        11
##   [8,]        12
##   [9,]        13
##  [10,]        14
##  [11,]        16
##  [12,]        17
##  [13,]        18
##  [14,]        19
##  [15,]        20
##  [16,]        21
##  [17,]        22
##  [18,]        23
##  [19,]        25
##  [20,]        26
##  [21,]        27
##  [22,]        28
##  [23,]        29
##  [24,]        30
##  [25,]        31
##  [26,]        32
##  [27,]        33
##  [28,]        36
##  [29,]        37
##  [30,]        39
##  [31,]        40
##  [32,]        42
##  [33,]        43
##  [34,]        44
##  [35,]        45
##  [36,]        46
##  [37,]        47
##  [38,]        48
##  [39,]        49
##  [40,]        50
##  [41,]        51
##  [42,]        53
##  [43,]        54
##  [44,]        55
##  [45,]        56
##  [46,]        57
##  [47,]        58
##  [48,]        59
##  [49,]        61
##  [50,]        62
##  [51,]        66
##  [52,]        67
##  [53,]        68
##  [54,]        69
##  [55,]        71
##  [56,]        72
##  [57,]        74
##  [58,]        75
##  [59,]        76
##  [60,]        77
##  [61,]        78
##  [62,]        79
##  [63,]        80
##  [64,]        81
##  [65,]        82
##  [66,]        83
##  [67,]        84
##  [68,]        85
##  [69,]        87
##  [70,]        88
##  [71,]        90
##  [72,]        91
##  [73,]        92
##  [74,]        93
##  [75,]        94
##  [76,]        95
##  [77,]        97
##  [78,]        98
##  [79,]        99
##  [80,]       100
##  [81,]       101
##  [82,]       102
##  [83,]       103
##  [84,]       104
##  [85,]       105
##  [86,]       106
##  [87,]       107
##  [88,]       108
##  [89,]       109
##  [90,]       112
##  [91,]       113
##  [92,]       114
##  [93,]       115
##  [94,]       116
##  [95,]       117
##  [96,]       118
##  [97,]       119
##  [98,]       120
##  [99,]       121
## [100,]       122
## [101,]       123
## [102,]       124
## [103,]       125
## [104,]       126
## [105,]       127
## [106,]       128
## [107,]       129
## [108,]       130
## [109,]       132
## [110,]       133
## [111,]       134
## [112,]       136
## [113,]       137
## [114,]       139
## [115,]       140
## [116,]       141
## [117,]       143
## [118,]       145
## [119,]       146
## [120,]       147
## [121,]       150
## [122,]       152
## [123,]       153
## [124,]       154
## [125,]       156
## [126,]       158
## [127,]       159
## [128,]       160
## [129,]       161
## [130,]       162
## [131,]       163
## [132,]       164
## [133,]       165
## [134,]       167
## [135,]       168
## [136,]       170
## [137,]       171
## [138,]       172
## [139,]       174
## [140,]       175
## [141,]       176
## [142,]       179
## [143,]       180
## [144,]       181
## [145,]       183
## [146,]       184
## [147,]       185
## [148,]       186
## [149,]       188
## [150,]       189
## [151,]       191
## [152,]       192
## [153,]       193
## [154,]       194
## [155,]       195
## [156,]       196
## [157,]       197
## [158,]       198
## [159,]       199
## [160,]       200

Example 1

# Build the model
model <- lm(GPA ~., data = train.data)
summary(model)
## 
## Call:
## lm(formula = GPA ~ ., data = train.data)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -1.40287 -0.30440  0.05946  0.30290  1.15076 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  1.57437    0.44516   3.537 0.000539 ***
## SEXM        -0.12596    0.08006  -1.573 0.117751    
## AGE         -0.01349    0.01592  -0.848 0.397983    
## MATH         0.01084    0.01446   0.750 0.454638    
## COMP         0.06213    0.01805   3.442 0.000747 ***
## COLLEGEBus  -0.06397    0.09047  -0.707 0.480580    
## COLLEGEEduc  0.04282    0.11077   0.387 0.699628    
## COLLEGEHss  -0.03434    0.13868  -0.248 0.804780    
## COLLEGENurs  0.23392    0.17607   1.329 0.185994    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.4581 on 151 degrees of freedom
## Multiple R-squared:  0.2297, Adjusted R-squared:  0.1889 
## F-statistic: 5.629 on 8 and 151 DF,  p-value: 2.977e-06

Example 1

# Make predictions and compute the R2, RMSE and MAE
predictions <- model %>% predict(test.data)
data.frame( R2 = R2(predictions, test.data$GPA),
            RMSE = RMSE(predictions, test.data$GPA),
            MAE = MAE(predictions, test.data$GPA))
##          R2      RMSE       MAE
## 1 0.2584404 0.4775062 0.3916094

Example 1 (New)

set.seed(123)
training1 <- Focus_New$GPA %>%
    createDataPartition(p = 0.6, list = F)
train.data1  <- Focus_New[training1, ]
test.data1 <- Focus_New[-training1, ]
training1
##        Resample1
##   [1,]         3
##   [2,]         6
##   [3,]        13
##   [4,]        14
##   [5,]        15
##   [6,]        16
##   [7,]        17
##   [8,]        18
##   [9,]        19
##  [10,]        20
##  [11,]        21
##  [12,]        22
##  [13,]        23
##  [14,]        25
##  [15,]        26
##  [16,]        27
##  [17,]        28
##  [18,]        29
##  [19,]        30
##  [20,]        31
##  [21,]        33
##  [22,]        34
##  [23,]        37
##  [24,]        38
##  [25,]        39
##  [26,]        40
##  [27,]        42
##  [28,]        43
##  [29,]        45
##  [30,]        47
##  [31,]        48
##  [32,]        49
##  [33,]        50
##  [34,]        51
##  [35,]        52
##  [36,]        54
##  [37,]        55
##  [38,]        58
##  [39,]        59
##  [40,]        62
##  [41,]        65
##  [42,]        66
##  [43,]        68
##  [44,]        69
##  [45,]        70
##  [46,]        76
##  [47,]        78
##  [48,]        79
##  [49,]        80
##  [50,]        83
##  [51,]        84
##  [52,]        85
##  [53,]        87
##  [54,]        88
##  [55,]        90
##  [56,]        92
##  [57,]        94
##  [58,]        97
##  [59,]        98
##  [60,]       100
##  [61,]       102
##  [62,]       103
##  [63,]       104
##  [64,]       105
##  [65,]       107
##  [66,]       109
##  [67,]       110
##  [68,]       112
##  [69,]       113
##  [70,]       114
##  [71,]       117
##  [72,]       118
##  [73,]       119
##  [74,]       120
##  [75,]       122
##  [76,]       123
##  [77,]       124
##  [78,]       126
##  [79,]       127
##  [80,]       131
##  [81,]       133
##  [82,]       140
##  [83,]       141
##  [84,]       142
##  [85,]       144
##  [86,]       145
##  [87,]       148
##  [88,]       149
##  [89,]       150
##  [90,]       152
##  [91,]       153
##  [92,]       155
##  [93,]       156
##  [94,]       158
##  [95,]       159
##  [96,]       160
##  [97,]       164
##  [98,]       165
##  [99,]       167
## [100,]       168
## [101,]       170
## [102,]       171
## [103,]       172
## [104,]       173
## [105,]       174
## [106,]       175
## [107,]       179
## [108,]       180
## [109,]       181
## [110,]       183
## [111,]       184
## [112,]       185
## [113,]       187
## [114,]       191
## [115,]       192
## [116,]       193
## [117,]       195
## [118,]       196
## [119,]       197
## [120,]       199

Example 1 (New)

# Build the model
model_1 <- lm(GPA ~., data = train.data1)
summary(model_1)
## 
## Call:
## lm(formula = GPA ~ ., data = train.data1)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -1.44692 -0.21943  0.00847  0.28915  0.89673 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)   
## (Intercept)  1.33366    0.47933   2.782  0.00634 **
## SEXM        -0.11621    0.08816  -1.318  0.19012   
## AGE         -0.02459    0.01537  -1.600  0.11238   
## MATH         0.03481    0.01608   2.165  0.03249 * 
## COMP         0.05616    0.01978   2.840  0.00537 **
## COLLEGEBus  -0.01603    0.09837  -0.163  0.87086   
## COLLEGEEduc  0.21165    0.11527   1.836  0.06900 . 
## COLLEGEHss   0.04466    0.14026   0.318  0.75076   
## COLLEGENurs  0.09631    0.20374   0.473  0.63734   
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.4235 on 111 degrees of freedom
## Multiple R-squared:  0.3284, Adjusted R-squared:   0.28 
## F-statistic: 6.785 on 8 and 111 DF,  p-value: 3.177e-07

Example 1 (New)

# Make predictions and compute the R2, RMSE and MAE
predictions1 <- model_1 %>% predict(test.data1)
data.frame( R2 = R2(predictions1, test.data1$GPA),
            RMSE = RMSE(predictions1, test.data1$GPA),
            MAE = MAE(predictions1, test.data1$GPA))
##          R2      RMSE       MAE
## 1 0.1101083 0.5291119 0.4130891

Example 1 (New)

# Make predictions and compute the R2, RMSE and MAE
predictions <- model %>% predict(test.data)
data.frame( R2 = R2(predictions, test.data$GPA),
            RMSE = RMSE(predictions, test.data$GPA),
            MAE = MAE(predictions, test.data$GPA))
##          R2      RMSE       MAE
## 1 0.2584404 0.4775062 0.3916094
predictions1 <- model_1 %>% predict(test.data1)
data.frame( R2 = R2(predictions1, test.data1$GPA),
            RMSE = RMSE(predictions1, test.data1$GPA),
            MAE = MAE(predictions1, test.data1$GPA))
##          R2      RMSE       MAE
## 1 0.1101083 0.5291119 0.4130891

Example 1 (New)

Cross-validation

K-Fold Cross Validation

K-Fold Cross Validation in R (Step-by-Step)

One commonly used method for doing this is known as k-fold cross-validation, which uses the following approach:

Example

Example

# library(caret)
# specify the cross-validation method
ctrl <- trainControl(method = "cv", number = 5)

Example

#fit a regression model and use k-fold CV to evaluate performance
model <- train(GPA ~ ., data = Focus_New, method = "lm", trControl = ctrl)
print(model)
## Linear Regression 
## 
## 200 samples
##   5 predictor
## 
## No pre-processing
## Resampling: Cross-Validated (5 fold) 
## Summary of sample sizes: 160, 160, 160, 160, 160 
## Resampling results:
## 
##   RMSE       Rsquared   MAE      
##   0.4595335  0.2504267  0.3662939
## 
## Tuning parameter 'intercept' was held constant at a value of TRUE

Interpretations:

Interpretations (cont’d):

Interpretations (cont’d):

Example (New Method)

# library(caret)
# specify the cross-validation method
ctrl <- trainControl(method = "repeatedcv", number = 5,  repeats = 3)

Example (New Method)

#fit a regression model and use k-fold CV to evaluate performance
model <- train(GPA ~ ., data = Focus_New, method = "lm", trControl = ctrl)
print(model)
## Linear Regression 
## 
## 200 samples
##   5 predictor
## 
## No pre-processing
## Resampling: Cross-Validated (5 fold, repeated 3 times) 
## Summary of sample sizes: 160, 160, 160, 160, 160, 160, ... 
## Resampling results:
## 
##   RMSE       Rsquared   MAE      
##   0.4669719  0.2050479  0.3732131
## 
## Tuning parameter 'intercept' was held constant at a value of TRUE

Applying k-Fold Cross-Validation to Logistic Regression

Recall

head(Binary)  
##   admit gre  gpa rank id
## 1     0 380 3.61    3  1
## 2     1 660 3.67    3  2
## 3     1 800 4.00    1  3
## 4     1 640 3.19    4  4
## 5     0 520 2.93    4  5
## 6     1 760 3.00    2  6

Recall

model <- glm(Binary$admit ~ Binary$gre + Binary$gpa + factor(Binary$rank),  family = "binomial")
summary(model)
## 
## Call:
## glm(formula = Binary$admit ~ Binary$gre + Binary$gpa + factor(Binary$rank), 
##     family = "binomial")
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -1.6268  -0.8662  -0.6388   1.1490   2.0790  
## 
## Coefficients:
##                       Estimate Std. Error z value Pr(>|z|)    
## (Intercept)          -3.989979   1.139951  -3.500 0.000465 ***
## Binary$gre            0.002264   0.001094   2.070 0.038465 *  
## Binary$gpa            0.804038   0.331819   2.423 0.015388 *  
## factor(Binary$rank)2 -0.675443   0.316490  -2.134 0.032829 *  
## factor(Binary$rank)3 -1.340204   0.345306  -3.881 0.000104 ***
## factor(Binary$rank)4 -1.551464   0.417832  -3.713 0.000205 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 499.98  on 399  degrees of freedom
## Residual deviance: 458.52  on 394  degrees of freedom
## AIC: 470.52
## 
## Number of Fisher Scoring iterations: 4

Logistic Regression

set.seed(123)
# Partition data and create index matrix of selected values
index <- createDataPartition(Binary$admit, p=.8, list=FALSE, times=1)

Create test and train data frames

Binary <- as.data.frame(Binary)

train <- Binary[index,]
test <- Binary[-index,]


# Verify number of rows (cases) in each data frame
nrow(train)
## [1] 320
nrow(test)
## [1] 80

Re-label values of outcome variable for train and test

train$admit[train$admit==1] <- "yes"
train$admit[train$admit==0] <- "no"

test$admit[test$admit==1] <- "yes"
test$admit[test$admit==0] <- "no"


# Convert outcome variable to factor for each data frame
train$admit <- as.factor(train$admit)
test$admit <- as.factor(test$admit)

Model

ctrlspecs <- trainControl(method="cv", 
                          number=10, 
                          savePredictions="all",
                          classProbs=TRUE)

set.seed(1234)
model_2 <- train(admit ~ gre + gpa + factor(rank),  data=train, method="glm", family = "binomial",  trControl=ctrlspecs)
summary(model_2)
## 
## Call:
## NULL
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -1.6790  -0.8721  -0.6442   1.1154   2.1120  
## 
## Coefficients:
##                  Estimate Std. Error z value Pr(>|z|)    
## (Intercept)     -4.363439   1.303098  -3.349 0.000812 ***
## gre              0.001809   0.001237   1.462 0.143666    
## gpa              1.019065   0.378182   2.695 0.007046 ** 
## `factor(rank)2` -0.791972   0.355269  -2.229 0.025799 *  
## `factor(rank)3` -1.424809   0.390766  -3.646 0.000266 ***
## `factor(rank)4` -1.541866   0.467099  -3.301 0.000964 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 403.57  on 319  degrees of freedom
## Residual deviance: 369.32  on 314  degrees of freedom
## AIC: 381.32
## 
## Number of Fisher Scoring iterations: 4

Model

print(model_2)
## Generalized Linear Model 
## 
## 320 samples
##   3 predictor
##   2 classes: 'no', 'yes' 
## 
## No pre-processing
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 288, 289, 288, 287, 287, 288, ... 
## Resampling results:
## 
##   Accuracy   Kappa    
##   0.7058498  0.2123983

Estimate the importance of different predictors

varImp(model_2)
## glm variable importance
## 
##                 Overall
## `factor(rank)3`  100.00
## `factor(rank)4`   84.19
## gpa               56.43
## `factor(rank)2`   35.12
## gre                0.00

Predict outcome using model from training data based on testing data

predictions <- predict(model_2, newdata=test)

# Create confusion matrix to assess model fit/performance on test data
confusionMatrix(data=predictions, test$admit)
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction no yes
##        no  51  19
##        yes  6   4
##                                           
##                Accuracy : 0.6875          
##                  95% CI : (0.5741, 0.7865)
##     No Information Rate : 0.7125          
##     P-Value [Acc > NIR] : 0.7350          
##                                           
##                   Kappa : 0.0826          
##                                           
##  Mcnemar's Test P-Value : 0.0164          
##                                           
##             Sensitivity : 0.8947          
##             Specificity : 0.1739          
##          Pos Pred Value : 0.7286          
##          Neg Pred Value : 0.4000          
##              Prevalence : 0.7125          
##          Detection Rate : 0.6375          
##    Detection Prevalence : 0.8750          
##       Balanced Accuracy : 0.5343          
##                                           
##        'Positive' Class : no              
## 

Optimization

Optimization in R

Linear Optimization

Linear Optimization Steps

Here are the basic steps in Linear Optimization:

Objective Functions and Decision Variables

Example

Let’s get introduced to Farmer Jean, who runs a small farm in a little valley:

Decision Variables

Objective Function

Constraints

Constraints

Coming back to Farmer Jean, she has two types of constraints:

Constraints

Optimization Model

Using R to solve Linear Optimization

Using R to solve Linear Optimization

objective_function <- c(0.15, 0.40)

constraint_matrix <- matrix(c(0.20, 0.70, 1, 1) , ncol=2 , byrow=TRUE) 
constraint_matrix
##      [,1] [,2]
## [1,]  0.2  0.7
## [2,]  1.0  1.0
constraint_directions <- c("<=", "<=")

constraint_limits <- c(100, 200)

Using R to solve Linear Optimization

lp.solution <- lp("max", objective_function, constraint_matrix, 
    constraint_directions, constraint_limits, compute.sens=TRUE)

Using R to solve Linear Optimization

# defining parameters
objective_function <- c(0.15, 0.40)
constraint_matrix <- matrix(c(0.20, 0.70, 1, 1) , ncol=2 , byrow=TRUE) 
constraint_directions <- c("<=", "<=")
constraint_limits <- c(100, 200)

# solving model
lp.solution <- lp("max", objective_function, constraint_matrix, 
    constraint_directions, constraint_limits, compute.sens=TRUE)
lp.solution
## Success: the objective function is 60
lp.solution$solution
## [1]  80 120

Sensitivity Analysis

Sensitivity Analysis

# sensitivity analysis on coefficients
lp.solution$sens.coef.from 
## [1] 0.1142857 0.1500000
lp.solution$sens.coef.to
## [1] 0.400 0.525

Another Example

Another Example

objective_function_1 <- c(25, 20)

constraint_matrix_1 <- matrix(c(20, 12, 1/15, 1/15) , ncol=2 , byrow=TRUE) 
constraint_matrix_1
##             [,1]        [,2]
## [1,] 20.00000000 12.00000000
## [2,]  0.06666667  0.06666667
constraint_directions_1 <- c("<=", "<=")

constraint_limits_1 <- c(1800, 8)

Another Example (Solving in R)

lp.solution_1 <- lp("max", objective_function_1, constraint_matrix_1, constraint_directions_1, constraint_limits_1, compute.sens=TRUE)

lp.solution_1$solution
## [1] 45 75
lp.solution_1
## Success: the objective function is 2625

Another Example ( Sensitivity Analysis)

# sensitivity analysis on coefficients
lp.solution_1$sens.coef.from 
## [1] 20 15
lp.solution_1$sens.coef.to
## [1] 33.33333 25.00000

Quadratic Optimization

Quadratic Optimization in R

Example

Example

library(quadprog)
# Matrix appearing in the quadratic function
Dmat <- matrix(c(0.125, 0.08, -0.01, 0.08, 0.405, -0.02, -0.01, -0.02, 0.005), 3, 3)

# Vector appearing in the quadratic function
dvec <- c(0, 0, 0)

# Matrix defining the constraints
Amat <- t(matrix(c(1, 21, 1, 0, 0, 1, 30, 0, 1, 0, 1, 8, 0, 0, 1), 5, 3))

# Vector holding the value of b_0
bvec <- c(1, 18, 0, 0, 0)

qp <- solve.QP(Dmat, dvec, Amat, bvec, meq = 1)
qp
## $solution
## [1] 0.4019549 0.2170267 0.3810185
## 
## $value
## [1] 0.02379223
## 
## $unconstrained.solution
## [1] 0 0 0
## 
## $iterations
## [1] 3 0
## 
## $Lagrangian
## [1] -0.049686557  0.005403946  0.000000000  0.000000000  0.000000000
## 
## $iact
## [1] 1 2

Trivial Example

n <- 3
x <- c(10, 15, 40)
y <- c(12, 45, 52)
X <- cbind(rep(1, n), x)

# Regression
z <- lm(y ~ x)
coef(z)
## (Intercept)           x 
##  14.8064516   0.9935484

Another Example (Application in Statistics)

# Optimization
library(quadprog)
z1 <- solve.QP(t(X) %*% X, t(y) %*% X, matrix(nr=2, nc=0), numeric(), 0 )

z1$solution 
## [1] 14.8064516  0.9935484

Another Example (Application in Statistics)

# Sample data
n <- 100
x1 <- rnorm(n)
x2 <- rnorm(n)
y <- 1 + x1 + x2 + rnorm(n)
X <- cbind(rep(1,n), x1, x2)

# Regression
r <- lm(y ~ x1 + x2)
coef(r)
## (Intercept)          x1          x2 
##   0.9581813   0.9820620   1.0602544

Another Example (Application in Statistics)

# Optimization
library(quadprog)
s <- solve.QP(t(X) %*% X, t(y) %*% X, matrix(nr=3,nc=0), numeric(), 0 )

s$solution # Identical
## [1] 0.9581813 0.9820620 1.0602544

Functions

Examples

Example 1

f <- function(x)(print(x) - 1/3)^2
xmin <- optimize(f, interval = c(0, 1))
## [1] 0.381966
## [1] 0.618034
## [1] 0.236068
## [1] 0.3333333
## [1] 0.3332926
## [1] 0.333374
## [1] 0.3333333
xmin
## $minimum
## [1] 0.3333333
## 
## $objective
## [1] 0
plot(f, 0, 1)
##   [1] 0.00 0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 0.09 0.10 0.11 0.12 0.13 0.14
##  [16] 0.15 0.16 0.17 0.18 0.19 0.20 0.21 0.22 0.23 0.24 0.25 0.26 0.27 0.28 0.29
##  [31] 0.30 0.31 0.32 0.33 0.34 0.35 0.36 0.37 0.38 0.39 0.40 0.41 0.42 0.43 0.44
##  [46] 0.45 0.46 0.47 0.48 0.49 0.50 0.51 0.52 0.53 0.54 0.55 0.56 0.57 0.58 0.59
##  [61] 0.60 0.61 0.62 0.63 0.64 0.65 0.66 0.67 0.68 0.69 0.70 0.71 0.72 0.73 0.74
##  [76] 0.75 0.76 0.77 0.78 0.79 0.80 0.81 0.82 0.83 0.84 0.85 0.86 0.87 0.88 0.89
##  [91] 0.90 0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98 0.99 1.00

Example 2

f <- function(x) return(abs(x-2) + 2*abs(x-1))
xmin <- optimize(f, interval = c(0, 3))
xmin
## $minimum
## [1] 1.000009
## 
## $objective
## [1] 1.000009
plot(f, 0, 3)

Maximum Likelihood Estimator (MLE)

Maximum Likelihood Estimator (MLE)

name <- function(pars, object){
  declarations 
  logL <- loglikelihood_function 
  return(-logL)
}

Example: Poisson

library(knitr)
poisson_lik <- function(mu, y){
  n <- nrow(y) 
  logL <- sum(y)*log(mu) - n*mu
  return(-logL)
}

Example: Optimizing the Log-Likelihood