Least Square function that will be used for both problem 1 and 2

myLSsolution <- function(mtxA, vctb){

  # Compute transpose(A) x A
  tranAA <- t(mtxA) %*% mtxA

  # Compute transpose(A) X b
  tranAb <- t(mtxA) %*% vctb
 
  # find xhat
  xhat <- solve(tranAA, tranAb)
  
  # find error
  e <- vctb - mtxA %*% xhat
  sqerr <- sum(t(e) %*% e)
  err <- sqrt(sqerr)
  return(list("xhat"=xhat,"sqerr"=sqerr,"err"=err))
}

PROBLEM SET 1

Properties of least squares solution

# Set A matrix
options(digits=2)

# Matrix A and vector b
A <- matrix(c(1,1,1,1,0,1,3,4), ncol=2)
b <- c(0,8,8,20)
A
##      [,1] [,2]
## [1,]    1    0
## [2,]    1    1
## [3,]    1    3
## [4,]    1    4
b
## [1]  0  8  8 20

Solve for \(\hat{x}\)

results1<-myLSsolution(A,b)
results1$xhat
##      [,1]
## [1,]    1
## [2,]    4

What is the squared error of this solution

results1$sqerr
## [1] 44

Instead of b = [0;8;8;20], sart with p=[1;5;13;17] and find the exact solution

p <- c(1,5,13,17)

results_p <- myLSsolution(A,p)

# xhat
results_p$xhat
##      [,1]
## [1,]    1
## [2,]    4
# square error
results_p$sqerr
## [1] 0

show that the error e=b-p=[-1;3;-5;3].

e = as.matrix(b) - as.matrix(p)
e
##      [,1]
## [1,]   -1
## [2,]    3
## [3,]   -5
## [4,]    3

show that the error e is orthogonal to p and to each of the columns of A.

# Dot product should be zero
p %*% e
##      [,1]
## [1,]    0
# take transpose(A) to get dot product of columns of A with e
t(A) %*% e
##      [,1]
## [1,]    0
## [2,]    0

PROBLEM SET 2

Compute best fitting solution for auto-mpg data

# read data into table
autompg <- read.table("auto-mpg.data", col.names = c("displacement","horsepower","weight","acceleration","mpg"))

#Extract A and b
A <- as.matrix(autompg[1:4])
b <- as.matrix(autompg[5])
head(A)
##      displacement horsepower weight acceleration
## [1,]          307        130   3504           12
## [2,]          350        165   3693           12
## [3,]          318        150   3436           11
## [4,]          304        150   3433           12
## [5,]          302        140   3449           10
## [6,]          429        198   4341           10
head(b)
##      mpg
## [1,]  18
## [2,]  15
## [3,]  18
## [4,]  16
## [5,]  17
## [6,]  15
myLSsolution(A,b)
## $xhat
##                  mpg
## displacement -0.0300
## horsepower    0.1571
## weight       -0.0062
## acceleration  1.9973
## 
## $sqerr
## [1] 13101
## 
## $err
## [1] 114

Equation is \(\hat{mpg}\) = -0.0300*displacement + 0.1571*horsepower - 0.0062*weight + 1.9973*acceleartion + (constant)