library(readxl)    
library(lmtest)    
## Loading required package: zoo
## 
## Attaching package: 'zoo'
## The following objects are masked from 'package:base':
## 
##     as.Date, as.Date.numeric
library(sandwich)  
library(car)       
## Loading required package: carData
library(tidyverse)   
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.2.1     ✔ readr     2.2.0
## ✔ forcats   1.0.1     ✔ stringr   1.6.0
## ✔ ggplot2   4.0.3     ✔ tibble    3.3.1
## ✔ lubridate 1.9.5     ✔ tidyr     1.3.2
## ✔ purrr     1.2.2
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ✖ dplyr::recode() masks car::recode()
## ✖ purrr::some()   masks car::some()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
district<-read_excel("district.xls")
district <- district[district$COMMTYPE != "Charters", ]                    
district <- district[!is.na(district$DAGC4X21R) & district$DAGC4X21R >= 0, ]
district <- district[!is.na(district$DPFVTOTK) & district$DPFVTOTK > 0, ]
#removing charters, districts with missing/hidden graduation rates, and those with no property value
district <- district |>
  mutate(grad       = DAGC4X21R,         # dependent variable: 4-year graduation rate
         tax_value  = DPFVTOTK,          # independent variable: taxable value per pupil
         log_tax    = log(DPFVTOTK),     # logged version used in the models
         econ       = DPETECOP,          # % economically disadvantaged
         enroll     = DPETALLC,          # total students
         log_enroll = log(DPETALLC),
                                         # urban = 1 if the district is one of the four city or suburb types, 0 if not
         urban      = ifelse(COMMTYPE == "Major Urban", 1, 0) +
                      ifelse(COMMTYPE == "Major Suburban", 1, 0) +
                      ifelse(COMMTYPE == "Other Central City", 1, 0) +
                      ifelse(COMMTYPE == "Other Central City Suburban", 1, 0))
model1 <- lm(grad ~ log_tax, data = district)
model2 <- lm(grad ~ log_tax + econ + log_enroll + urban, data = district)

summary(model1)  
## 
## Call:
## lm(formula = grad ~ log_tax, data = district)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -36.105  -2.034   1.216   3.823   5.145 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  90.3139     2.7675  32.634   <2e-16 ***
## log_tax       0.4204     0.2105   1.997   0.0461 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 4.739 on 958 degrees of freedom
## Multiple R-squared:  0.004144,   Adjusted R-squared:  0.003105 
## F-statistic: 3.987 on 1 and 958 DF,  p-value: 0.04614
summary(model2)
## 
## Call:
## lm(formula = grad ~ log_tax + econ + log_enroll + urban, data = district)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -36.905  -1.430   1.050   2.655   7.310 
## 
## Coefficients:
##              Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 114.83064    3.00150  38.258  < 2e-16 ***
## log_tax      -0.52643    0.19867  -2.650  0.00819 ** 
## econ         -0.08662    0.00739 -11.722  < 2e-16 ***
## log_enroll   -0.94947    0.13121  -7.236 9.46e-13 ***
## urban        -0.24827    0.43352  -0.573  0.56699    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 4.238 on 955 degrees of freedom
## Multiple R-squared:  0.2062, Adjusted R-squared:  0.2029 
## F-statistic: 62.01 on 4 and 955 DF,  p-value: < 2.2e-16
#regression models (table 2)
model_raw <- lm(grad ~ tax_value, data = district)
plot(model_raw, which = 1)

raintest(model_raw)
## 
##  Rainbow test
## 
## data:  model_raw
## Rain = 0.7412, df1 = 480, df2 = 478, p-value = 0.9995
# After the log transformation
plot(model1, which = 1)

raintest(model1)
## 
##  Rainbow test
## 
## data:  model1
## Rain = 0.74261, df1 = 480, df2 = 478, p-value = 0.9994
raintest(model2)
## 
##  Rainbow test
## 
## data:  model2
## Rain = 0.78951, df1 = 480, df2 = 475, p-value = 0.995
# Linearity: residual & rainbow test then log
plot(model2, which = 3)

bptest(model1)
## 
##  studentized Breusch-Pagan test
## 
## data:  model1
## BP = 1.2239, df = 1, p-value = 0.2686
bptest(model2)
## 
##  studentized Breusch-Pagan test
## 
## data:  model2
## BP = 29.678, df = 4, p-value = 5.693e-06
#Homoscedasticity: plot and BP test 
durbinWatsonTest(model1)
##  lag Autocorrelation D-W Statistic p-value
##    1      0.06211333      1.873778    0.05
##  Alternative hypothesis: rho != 0
durbinWatsonTest(model2)
##  lag Autocorrelation D-W Statistic p-value
##    1        0.017684      1.963723   0.542
##  Alternative hypothesis: rho != 0
# Independence of errors: Durbin-Watson test
plot(model1, which = 2)

shapiro.test(model1$residuals)
## 
##  Shapiro-Wilk normality test
## 
## data:  model1$residuals
## W = 0.82895, p-value < 2.2e-16
plot(model2, which = 2)

shapiro.test(model2$residuals)
## 
##  Shapiro-Wilk normality test
## 
## data:  model2$residuals
## W = 0.83589, p-value < 2.2e-16
# Normality of errors: Q-Q plot and Shapiro-Wilk test
vif(model2)
##    log_tax       econ log_enroll      urban 
##   1.113669   1.097808   2.108636   2.088264
# Multicollinearity
model2_log <- lm(log(grad) ~ log_tax + econ + log_enroll + urban, data = district)
bptest(model2_log)   # still p < .001, so no bueno
## 
##  studentized Breusch-Pagan test
## 
## data:  model2_log
## BP = 24.634, df = 4, p-value = 5.961e-05
#fixing the unequal spread
0.4204 * log(2)     
## [1] 0.2913991
-0.5264 * log(2)  
## [1] -0.3648727
# doubling property wealth (coefficient x log of 2)
-0.0866 * 10        # about -0.9 points
## [1] -0.866
#  adding 10 more points of economically disadvantaged students