\[ Corr(\epsilon_i, \epsilon_j) = 0 \ ,\forall \ i, \ j \ (i \neq j) \]
\[ \epsilon \sim \mathrm{N}(0, \sigma^2) \]
Then,
\[ Y \sim \mathrm{N}(\alpha +\beta X, \sigma^2) \]
Ohters: random sampling, linear relationship etc.
\[ \sum{e_i^2} = \sum(y_i - \hat{y_i})^2 \rightarrow \ minimise \\\]
\[ \begin{align} \text{FOC 1}&\\ \frac{\partial}{\partial \hat{\alpha}}\sum(y_i-\hat{\alpha} - \hat{\beta}x_i) &= -2 \sum(y_i-\hat{\alpha} - \hat{\beta}x_i)\\ &=0\\ \end{align} \\\]
\[\begin{align} &\implies \ \sum(y_i-\hat{\alpha} - \hat{\beta}x_i) =0 \\ &\implies \ \hat{\alpha} = \overline{y} -\hat{\beta}\overline{x} \dashrightarrow (i) \end{align} \]
\[ \begin{align} \text{FOC 2}&\\ \frac{\partial}{\partial \hat{\beta}}\sum(y_i-\hat{\alpha} - \hat{\beta}x_i) &= -2 \sum(y_i-\hat{\alpha} - \hat{\beta}x_i)(x_i) \\ &=0 \\ \end{align} \] \[ \begin{align} &\implies \sum{x_iy_i}-\hat{\alpha}\sum{x_i} - \hat{\beta}\sum{x_i^2} =0 \\ &\implies n\sum{x_iy_i}-(\sum{y_i} - \hat{\beta}\sum{x_i})\sum{x_i} - n\hat{\beta}\sum{x_i^2} =0 \dashrightarrow (substitution, i) \\ \end{align} \]
Therefore, \[ \hat{\beta} = \frac{n\sum{x_i}{y_i}-\sum{x_i}\sum{y_i}}{n\sum{x_i^2}-\sum{(x_i)^2}} \]
Meaning, \[ \hat{\beta}=\frac{s_{xy}}{s_{(xx)}} = \frac{s_{xy}}{s_{x}^2} = r \frac{s_y}{s_x} = \frac{\sum{(x_i-\overline{x})(y_i-\overline{y})}}{\sum{(x_i-\overline{x})^2}} \]
\[ \begin{align} &\sum{e_i} = 0 \ (from \ OLS \ derivation) \\ &\sum{X_ie_i} = 0 \ (from \ OLS \ derivation) \\ &\sum{\hat{Y_i}e_i} = 0 \ (from \ the \ first\ two) \end{align} \]
\[ \begin{align} SE &= RMSE && \text{(RMSE: Square root of Mean Square Error)} \\ &= \sqrt{\frac{\sum{(Y_i-\hat{Y_i})^2}}{n-2}} && \text{(degrees of freedom = n-2)}\\ &=\sqrt{\frac{\sum{e_i^2}}{n-2}} \\ &=(sd \ of \ residuals) && (\sum{e_i} = 0)\\ &=s \\ \end{align} \]
Plus, \[ \begin{align} MSE = \frac{\sum{e_i^2}}{n-2} && \text{(MSE: Mean Square Error)}\\ SSE = \sum{e_i^2} && \text{(SSE: Sum Square Error)}\\ \end{align} \]
\[ \sum{(Y_i-\overline{Y})^2} = \sum{(Y_i-\hat{Y_i})} + \sum{(\hat{Y_i}-\overline{Y})^2} \] (easily proven when expanded)
\[ \begin{align} &R^2=1-\frac{SSE}{SST} = \frac{SSR}{SST} \\ &cf) \ R^2 = r^2 \rightarrow (does \ not \ hold \ up \ in \ multiple \ regressions) \end{align} \] (easily proven when expanded)
\[ \begin{align} &E(\hat{\beta}) = \beta \\ &Var(\hat{\beta}) = \frac{\sigma^2}{\sum{(x_i-\overline{x})^2}} \\ &\hat{\beta} \sim \mathrm N({E(\hat{\beta})},{Var(\hat{\beta})}) \end{align} \]
Derivation →
\[ \begin{align} &\hat{\beta} = \frac{\sum{(x_i-\overline{x})(y_i-\overline{y})}}{\sum{(x_i-\overline{x})^2}}\\ &\implies \hat{\beta} = \frac{\sum{(x_i-\overline{x})y_i}}{\sum{(x_i-\overline{x})^2}}\\ &\implies E(\hat{\beta}) = \frac{\sum{(x_i-\overline{x})}}{\sum{(x_i-\overline{x})^2}}E(y_i) \ (x: exgeneous) \ (y: random \ variable) \end{align} \]
Since, \[ E(y_i) = (\alpha + \beta x_i) \]
Conclusion: \[ E(\hat{\beta}) = \beta \]
As for variance: \[
\begin{align}
&Var(\hat{\beta}) \\
&= Var(\frac{\sum{(x_i-\overline{x})y_i}}{\sum{(x_i-\overline{x})^2}}) \\
&= \frac{\sum{(x_i-\overline{x})^2}}{[\sum{(x_i-\overline{x})^2}]^2}Var(y_i) \ \dashrightarrow (*) \\
&= \frac{\sigma^2}{\sum{(x_i-\overline{x})^2}}
\end{align}
\] (*): break it down to y’s linear form then the logic can be seen.
And as y follows normal distribution, Beta hat follows normal distribution too.
As for, expected value and variance of the interval:
\[
\begin{align}
&E(\hat{\alpha}) =\alpha \\
&Var(\hat{\alpha}) = \frac{\sigma^2 \sum{x_i^2}}{n \sum{(x_i - \overline{x})^2}} \\
&\hat{\alpha} \sim \mathrm N(E(\hat{\alpha}), Var(\hat{\alpha}))
\end{align}
\]
\[ \hat{\beta} \sim \mathrm N(\beta,\frac{\sigma^2}{\sum{(x_i-\overline{x})^2}}) \]
Then,
Test-statistic: follows t distribution with df of n - 2 \[
\frac{\hat{\beta}-\beta}{s \sqrt{\frac{1}{\sum{(X_i - \overline{X})^2}}}} \sim \mathrm t(n-2)
\]
Derivation: \[
\begin{align}
&\frac{\hat{\beta}-\beta}{\sqrt{\frac{\sigma^2}{\sum{(X_i - \overline{X})^2}}}} \sim \mathrm N(0, 1^2) \dashrightarrow i)\\
&\frac{SSE}{\sigma^2} = \frac{\sum e_i^2}{\sigma^2} \dashrightarrow ii)\\
&= \frac{(n-2)[(\sum e_i^2)/(n-2)]}{\sigma^2} \sim \mathrm \chi^2(n-2) \dashrightarrow (**)
\end{align}
\] (**) When the population follows the normal distribution, df x s^2 / sigma^2 follows chi-square distribtion with the df.
By the definition of t-distribution, i) over square root of ii) follows t-distribution with df of (n-2).
On top of this, as we know first, test-statistic, and second, the distribution it follows, confidence intervals with given alpha(significance level) can also be deducted.