A <- matrix(c(1,1,1,1,0,1,3,4), nrow = 4)
b <- matrix(c(0,8,8,20), nrow=4)
A
##      [,1] [,2]
## [1,]    1    0
## [2,]    1    1
## [3,]    1    3
## [4,]    1    4
b
##      [,1]
## [1,]    0
## [2,]    8
## [3,]    8
## [4,]   20

Write R markdown script to compute ATA and ATb

A_trans_A <- t(A)%*% A
A_trans_A
##      [,1] [,2]
## [1,]    4    8
## [2,]    8   26
A_trans_b <- t(A)%*%b
A_trans_b
##      [,1]
## [1,]   36
## [2,]  112
#calculate x_tilda
x_tilda <-solve(A_trans_A) %*% A_trans_b
x_tilda
##      [,1]
## [1,]    1
## [2,]    4
#if we make x = x_tilda then e^2 = (Ax_tilda - b)^2

e<- b - (A %*% x_tilda)  
e
##      [,1]
## [1,]   -1
## [2,]    3
## [3,]   -5
## [4,]    3
e_sq <- (e[1])^2 + (e[2]^2) + (e[3]^2) + (e[4]^2)
e_sq
## [1] 44

Ax = b = p + e

p = [1; 5; 13; 17]

Let’s do the same but with b=p

p <- matrix(c(1,5,13,17), nrow=4)
p
##      [,1]
## [1,]    1
## [2,]    5
## [3,]   13
## [4,]   17
A_trans_p <- t(A)%*%p
A_trans_p
##      [,1]
## [1,]   36
## [2,]  112
#calculate x_tilda
x_tilda <-solve(A_trans_A) %*% A_trans_p
x_tilda
##      [,1]
## [1,]    1
## [2,]    4
#if we make x = x_tilda then e^2 = (Ax_tilda - p)^2

e_p<- (A %*% x_tilda) - p
e_p
##              [,1]
## [1,] 0.000000e+00
## [2,] 8.881784e-16
## [3,] 3.552714e-15
## [4,] 3.552714e-15
#We notice that e is  zero
e_p_sq <- (e_p[1])^2 + (e_p[2]^2) + (e_p[3]^2) + (e_p[4]^2)



e_p_sq
## [1] 2.603241e-29

Show that the error e = b-p= [-1;3;-5;3].

Since the projection vector p is orthogonal to the difference e = b-Ax_tilda???

e_2 <- b - (A%*%x_tilda)
e_2
##      [,1]
## [1,]   -1
## [2,]    3
## [3,]   -5
## [4,]    3

Show that e is orthogonal to p.

For e to be orthogonal to p the dot product must be equal to 0

dot_e_p <- sum(e_2 *p)
#notice it is equal to zero
dot_e_p
## [1] -1.030287e-13

Show that e is orthogonal to the column space of A

Again, we do the dot product of e and column 1 , 2 of A

dot_e_A1 <- sum(e_2*A[,1])
dot_e_A2 <- sum(e_2 * A[,2])
dot_e_A1
## [1] -7.993606e-15
dot_e_A2
## [1] -2.575717e-14
#Notice that both equal to zero

PROBLEM 2

Import data file into R matrix

filepath <- c("https://raw.githubusercontent.com/nobieyi00/CUNY_MSDA_R/master/auto-mpg.data")

Auto_mpg <-read.table(filepath,header = FALSE, sep = "")

A <- as.matrix(Auto_mpg[,1:4])

b <- as.matrix(Auto_mpg[,5])

Currently Ax =b has no solution

When the length of e is as small as possible, x_tilda is the least squares solution. Lets compute x_tilda

A_trans_A <- t(A)%*% A
#A_trans_A

A_trans_b <- t(A)%*%b
#A_trans_b

#calculate x_tilda
x_tilda <-solve(A_trans_A) %*% A_trans_b
#x_tilda

The best fitting solution for the mpg is that

t(A) x A x x_tilda = t(A) x b

Lets now calculate the fitting error between the predicted mpg and actual mpg We know that projection p= A x x_tilda

e<- b-(A %*% x_tilda)

The least square solution makes Magnitude of (Ax -b)^2 as small as possible.

Let calculate the magnitude of e

n<-nrow(e)
sum_e <-0
for (i in 1:n)
{
  sum_e <- sum_e + e[i]^2
}
#Magnitude of error is 
sum_e
## [1] 13101.43

Lets confirm that we found the best fit solution Let’s do the same but with b=p

p <- A %*% x_tilda


A_trans_p <- t(A)%*%p

#calculate x_tilda
x_tilda <-solve(A_trans_A) %*% A_trans_p


#if we make x = x_tilda then e^2 = (Ax_tilda - p)^2

e_p<- (A %*% x_tilda) - p


#We notice that e is  zero
e_p_sq <- (e_p[1])^2 + (e_p[2]^2) + (e_p[3]^2) + (e_p[4]^2)



e_p_sq
## [1] 5.750354e-25

We notice that magnitude of error is zero

Test that e is orthogonal to p, so dot product equal to zero

dot_e_p <- sum(e *p)
#notice it is equal to zero
dot_e_p
## [1] -7.34909e-10

Test that e is orthogonal to the column space of A

Again, we do the dot product of e and column 1 , 2,3,4 of A

dot_e_A1 <- sum(e*A[,1])
dot_e_A2 <- sum(e * A[,2])
dot_e_A3 <- sum(e * A[,3])
dot_e_A4 <- sum(e * A[,4])
dot_e_A1
## [1] -1.125269e-08
dot_e_A2
## [1] -4.384479e-09
dot_e_A3
## [1] -1.471572e-07
dot_e_A4
## [1] -6.504872e-10
#Notice that all equal to zero