Bachelor Thesis

Author

Koki Bodaiji

1 Setting up

setwd("C:/Users/KOUKI/Documents/University/THESIS/data")
getwd()

library(tidyverse)
library(gt)
library(conflicted)
library(janitor)
library(plm)
library(ggplot2)
library(psych)
library(MASS)
library(corrplot)
library(car)
library(modelsummary)
library(skimr)
library(rstatix)
library(naniar)
library(stringr)
library(sf)
library(tmap)
library(spdep)
library(spgwr)
library(spatialreg)
library(expm)
library(AER)
library(sem)
library(ISLR2)
library(leaps)
library(lattice)
library(glmnet)
library(e1071)
conflict_prefer("filter", "dplyr")
conflict_prefer("select", "dplyr")
conflict_prefer("lag", "dplyr")

2 Data

2.1 Residential area

data.resi_500m = read_sf("C:/Users/KOUKI/Documents/University/THESIS/residential_2019_500m_polygon.shp") %>%
  subset(ID != "20702")

2.2 Commercial area

data.comm_500m = read_sf("C:/Users/KOUKI/Documents/University/THESIS/commercial_2019_500m_polygon.shp")

3 Creating New Variables

3.1 Residential Area

3.1.1 Walkability Index

data.resi_500m = data.resi_500m %>%
  mutate(ZSP_facilities = (sqrt(data.resi_500m$all_facili)-mean(sqrt(data.resi_500m$all_facili)))/sd(sqrt(data.resi_500m$all_facili))) %>%
  mutate(ZSP_density = (sqrt(data.resi_500m$density)-mean(sqrt(data.resi_500m$density)))/sd(sqrt(data.resi_500m$density))) %>%
  mutate(ZSP_intersection = (sqrt(data.resi_500m$intersecti)-mean(sqrt(data.resi_500m$intersecti)))/sd(sqrt(data.resi_500m$intersecti))) %>%
  mutate(WI = (ZSP_facilities + ZSP_density + ZSP_intersection)/3)
wiplot = data.resi_500m[, c("all_facili", "density", "intersecti")] %>%
  st_drop_geometry()
colnames(wiplot) = 
  c("Facility access",
    "Population density",
    "Street connectivity")
pairs.panels(wiplot)

3.1.2 Diversity Index

data.resi_500m = data.resi_500m %>%
  rename_with(~paste0("DI_", .x), mining:urban_park)

data.resi_500m = data.resi_500m %>%
  mutate(across(starts_with("DI"), ~./all_facili, .names = "probability_{.col}")) %>%
  mutate(across(starts_with("probability"), list(~log(.)), .names = "log_{.col}"))

data.resi_500m = data.resi_500m %>%
  mutate_at(vars(starts_with("log")), ~ifelse(is.infinite(-.), 0, .))

data.resi_500m = data.resi_500m %>%
  mutate(mul_minig = probability_DI_mining * log_probability_DI_mining) %>%
  mutate(mul_construction = probability_DI_construc_1 * log_probability_DI_construc_1) %>%
  mutate(mul_manufacture = probability_DI_manufactur * log_probability_DI_manufactur) %>%
  mutate(mul_infrastructure = probability_DI_infrastruc * log_probability_DI_infrastruc) %>%
  mutate(mul_information = probability_DI_informatio * log_probability_DI_informatio) %>%
  mutate(mul_transportation = probability_DI_transporta * log_probability_DI_transporta) %>%
  mutate(mul_trade = probability_DI_trade * log_probability_DI_trade) %>%
  mutate(mul_finance = probability_DI_finance * log_probability_DI_finance) %>%
  mutate(mul_realestate = probability_DI_real_estat * log_probability_DI_real_estat) %>%
  mutate(mul_academic = probability_DI_academic * log_probability_DI_academic) %>%
  mutate(mul_accomodation = probability_DI_accomodati * log_probability_DI_accomodati) %>%
  mutate(mul_entertainment = probability_DI_entertainm * log_probability_DI_entertainm) %>%
  mutate(mul_education = probability_DI_education * log_probability_DI_education) %>%
  mutate(mul_medical = probability_DI_medical * log_probability_DI_medical) %>%
  mutate(mul_service = probability_DI_service * log_probability_DI_service) %>%
  mutate(mul_others = probability_DI_others * log_probability_DI_others) %>%
  mutate(mul_prefecture = probability_DI_prefecture * log_probability_DI_prefecture) %>%
  mutate(mul_city = probability_DI_city_facil * log_probability_DI_city_facil) %>%
  mutate(mul_school = probability_DI_school * log_probability_DI_school) %>%
  mutate(mul_park = probability_DI_urban_park * log_probability_DI_urban_park) %>%
  mutate(DI = -rowSums(across(starts_with("mul"))))

data.analysis.resi_500m = data.resi_500m %>%
  select(1,2,3,
         11:38,
         63:66,
         70,131) 
colnames(data.analysis.resi_500m) = 
  c("ID",
    "City_code",
    "City_code_text",
    "Area",
    "Water",
    "Gas",
    "Sewage",
    "Frontage_ratio",
    "Depth_ratio",
    "Ground_level",
    "Underground_level",
    "North",
    "East",
    "South",
    "West",
    "Front_road_width",
    "Station_distance",
    "Construction_ratio",
    "Floor_area",
    "Price_14y",
    "Price_15y",
    "Price_16y",
    "Price_17y",
    "Price_18y",
    "Price_19y",
    "Price_14y_log",
    "Price_15y_log",
    "Price_16y_log",
    "Price_17y_log",
    "Price_18y_log",
    "Price_19y_log",
    "PD",
    "City_area",
    "City_perimeter",
    "geometry",
    "WI",
    "DI")
data.analysis.resi_500m$DI = replace(data.analysis.resi_500m$DI, is.nan(data.analysis.resi_500m$DI), 0)

3.1.3 Gravelius Index

data.analysis.resi_500m = data.analysis.resi_500m %>%
  mutate(GI = City_perimeter / (2*(pi * City_area)^(1/2)))

3.2 Commercial Area

3.2.1 Walkability Index

data.comm_500m = data.comm_500m %>%
  mutate(ZSP_facilities = (sqrt(data.comm_500m$all_facili)-mean(sqrt(data.comm_500m$all_facili)))/sd(sqrt(data.comm_500m$all_facili))) %>%
  mutate(ZSP_density = (sqrt(data.comm_500m$density)-mean(sqrt(data.comm_500m$density)))/sd(sqrt(data.comm_500m$density))) %>%
  mutate(ZSP_intersection = (sqrt(data.comm_500m$intersecti)-mean(sqrt(data.comm_500m$intersecti)))/sd(sqrt(data.comm_500m$intersecti))) %>%
  mutate(WI = (ZSP_facilities + ZSP_density + ZSP_intersection)/3)
wiplot_comm = data.comm_500m[, c("all_facili", "density", "intersecti")] %>%
  st_drop_geometry()
colnames(wiplot) = c("Facility access",
    "Population density",
    "Street connectivity")
pairs.panels(wiplot_comm)

3.2.2 Diversity Index

data.comm_500m = data.comm_500m %>%
  rename_with(~paste0("DI_", .x), mining:urban_park)

data.comm_500m = data.comm_500m %>%
  mutate(across(starts_with("DI"), ~./all_facili, .names = "probability_{.col}")) %>%
  mutate(across(starts_with("probability"), list(~log(.)), .names = "log_{.col}"))

data.comm_500m = data.comm_500m %>%
  mutate_at(vars(starts_with("log")), ~ifelse(is.infinite(-.), 0, .))

data.comm_500m = data.comm_500m %>%
  mutate(mul_minig = probability_DI_mining * log_probability_DI_mining) %>%
  mutate(mul_construction = probability_DI_construc_1 * log_probability_DI_construc_1) %>%
  mutate(mul_manufacture = probability_DI_manufactur * log_probability_DI_manufactur) %>%
  mutate(mul_infrastructure = probability_DI_infrastruc * log_probability_DI_infrastruc) %>%
  mutate(mul_information = probability_DI_informatio * log_probability_DI_informatio) %>%
  mutate(mul_transportation = probability_DI_transporta * log_probability_DI_transporta) %>%
  mutate(mul_trade = probability_DI_trade * log_probability_DI_trade) %>%
  mutate(mul_finance = probability_DI_finance * log_probability_DI_finance) %>%
  mutate(mul_realestate = probability_DI_real_estat * log_probability_DI_real_estat) %>%
  mutate(mul_academic = probability_DI_academic * log_probability_DI_academic) %>%
  mutate(mul_accomodation = probability_DI_accomodati * log_probability_DI_accomodati) %>%
  mutate(mul_entertainment = probability_DI_entertainm * log_probability_DI_entertainm) %>%
  mutate(mul_education = probability_DI_education * log_probability_DI_education) %>%
  mutate(mul_medical = probability_DI_medical * log_probability_DI_medical) %>%
  mutate(mul_service = probability_DI_service * log_probability_DI_service) %>%
  mutate(mul_others = probability_DI_others * log_probability_DI_others) %>%
  mutate(mul_prefecture = probability_DI_prefecture * log_probability_DI_prefecture) %>%
  mutate(mul_city = probability_DI_city_facil * log_probability_DI_city_facil) %>%
  mutate(mul_school = probability_DI_school * log_probability_DI_school) %>%
  mutate(mul_park = probability_DI_urban_park * log_probability_DI_urban_park) %>%
  mutate(DI = -rowSums(across(starts_with("mul"))))

data.analysis.comm_500m = data.comm_500m %>%
  select(1,2,3,
         11:38,
         63:66,
         70,131)

colnames(data.analysis.comm_500m) = 
  c("ID",
    "City_code",
    "City_code_text",
    "Area",
    "Water",
    "Gas",
    "Sewage",
    "Frontage_ratio",
    "Depth_ratio",
    "Ground_level",
    "Underground_level",
    "North",
    "East",
    "South",
    "West",
    "Front_road_width",
    "Station_distance",
    "Construction_ratio",
    "Floor_area",
    "Price_14y",
    "Price_15y",
    "Price_16y",
    "Price_17y",
    "Price_18y",
    "Price_19y",
    "Price_14y_log",
    "Price_15y_log",
    "Price_16y_log",
    "Price_17y_log",
    "Price_18y_log",
    "Price_19y_log",
    "PD",
    "City_area",
    "City_perimeter",
    "geometry",
    "WI",
    "DI",
    "GI")

3.2.3 Gravelius Index

data.analysis.comm_500m = data.analysis.comm_500m %>%
  mutate(GI = City_perimeter / (2*(pi * City_area)^(1/2)))

4 Variable Selection

4.1 Residential Area

4.1.1 Data Processing

data.vs.resi <- data.analysis.resi_500m %>%
  select(-1,-2,-3,-20:-30, -33,-34) %>%
  filter(DI != 0) %>%
  filter(Front_road_width != 100) %>%
  mutate(Station_distance = ifelse(Station_distance == 0, 1, Station_distance)) %>%
  mutate(Construction_ratio = 0.01*Construction_ratio) %>%
  mutate(Floor_area = 0.01*Floor_area) %>%
  mutate(Area = log(Area)) %>%
  mutate(Station_distance = log(Station_distance))

data.non.dammy_resi <- data.vs.resi %>%
  st_drop_geometry() %>%
  select(-c("Water","Gas","Sewage","North","East","South","West")) %>%
  scale() %>%
  data.frame()
data.sd.vs.resi <- cbind(data.non.dammy_resi, data.vs.resi[c("Water","Gas","Sewage","North","East","South","West")])
data.sd.vs.resi <- data.sd.vs.resi %>%
  select(Price_19y_log,Area,Water,Gas,Sewage,Frontage_ratio,
                            Depth_ratio,Ground_level,
                            Underground_level,
                            North,
                            East,
                            South,
                            West,
                            Front_road_width,
                            Station_distance,
                            Construction_ratio,
                            WI,
                            DI,
                            GI,geometry)
4.1.1.1 Data Splitting
set.seed(1)
train.resi <- sample(c(TRUE, FALSE), nrow(data.sd.vs.resi), replace = TRUE)
test.resi <- (!train.resi)
data.train.resi <- data.sd.vs.resi[train.resi,] 
data.train.resi <- data.train.resi %>%
  select(-geometry)
data.test.resi <- data.sd.vs.resi[test.resi,]
skim(data.train.resi)
skim(data.test.resi)

y_resi <- data.train.resi[,1] %>%
  as.matrix()
x_resi <- data.train.resi[, 2:19] %>%
  as.matrix()
a<-1
4.1.1.2 Descriptive Statistics
des.resi = data.sd.vs.resi %>% 
  st_drop_geometry() %>%
  get_summary_stats()
des.tex.resi = data.sd.vs.resi %>%
  st_drop_geometry() %>%
  summary() %>%
  data.frame()
resiplot = cor(x_resi) %>%
  data.frame() %>%
  tibble::rownames_to_column(var = "Y")
resiplot.long <- pivot_longer(resiplot, 
                              cols = Area:GI,
                              names_to = "X", 
                              values_to = "rho") 
resiplot.long$X <- factor(resiplot.long$X, levels = unique(resiplot.long$X))
resiplot.long$Y <- factor(resiplot.long$Y, levels = rev(unique(resiplot.long$Y)))

resiplot.long %>%
  ggplot(aes(x = X, y = Y, fill = rho)) + 
  geom_tile() +
  geom_text(aes(label = sprintf("%0.2f", rho)),
            color = "black", size = 2) +
  theme(axis.text.y = element_text(size = 7),
        axis.text.x = element_text(size = 7, angle = -45, hjust = 0, vjust = 1)) +
  scale_fill_gradientn(limits = c(-1, 1), colours = c("#6D9EC1", "white", "#E46726")) +
  coord_equal()

4.1.2 Variable Selection Procedure

4.1.2.1 OLS
full_formula_resi <- Price_19y_log~
  Area +
  Water+
  Gas+
  Sewage+
  Frontage_ratio+
  Depth_ratio+
  Ground_level+
  Underground_level+
  North+
  East+
  South+
  West+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI

full_lm_resi <- lm(full_formula_resi, data.train.resi)

summary_full_lm_resi = list(full_lm_resi)
4.1.2.2 Best Subset
bs.resi <- regsubsets(Price_19y_log~., data.train.resi, nvmax = 18)
summary(bs.resi)
Subset selection object
Call: regsubsets.formula(Price_19y_log ~ ., data.train.resi, nvmax = 18)
18 Variables  (and intercept)
                   Forced in Forced out
Area                   FALSE      FALSE
Water                  FALSE      FALSE
Gas                    FALSE      FALSE
Sewage                 FALSE      FALSE
Frontage_ratio         FALSE      FALSE
Depth_ratio            FALSE      FALSE
Ground_level           FALSE      FALSE
Underground_level      FALSE      FALSE
North                  FALSE      FALSE
East                   FALSE      FALSE
South                  FALSE      FALSE
West                   FALSE      FALSE
Front_road_width       FALSE      FALSE
Station_distance       FALSE      FALSE
Construction_ratio     FALSE      FALSE
WI                     FALSE      FALSE
DI                     FALSE      FALSE
GI                     FALSE      FALSE
1 subsets of each size up to 18
Selection Algorithm: exhaustive
          Area Water Gas Sewage Frontage_ratio Depth_ratio Ground_level
1  ( 1 )  " "  " "   " " " "    " "            " "         " "         
2  ( 1 )  " "  " "   "*" " "    " "            " "         " "         
3  ( 1 )  " "  " "   "*" " "    " "            " "         " "         
4  ( 1 )  " "  " "   "*" " "    " "            " "         " "         
5  ( 1 )  "*"  " "   "*" " "    " "            " "         "*"         
6  ( 1 )  "*"  " "   "*" " "    " "            " "         "*"         
7  ( 1 )  "*"  " "   "*" " "    " "            " "         "*"         
8  ( 1 )  "*"  " "   "*" " "    " "            " "         "*"         
9  ( 1 )  "*"  " "   "*" " "    " "            " "         "*"         
10  ( 1 ) "*"  " "   "*" " "    " "            " "         "*"         
11  ( 1 ) "*"  " "   "*" " "    " "            "*"         "*"         
12  ( 1 ) "*"  " "   "*" "*"    " "            "*"         "*"         
13  ( 1 ) "*"  " "   "*" "*"    " "            "*"         "*"         
14  ( 1 ) "*"  " "   "*" "*"    " "            "*"         "*"         
15  ( 1 ) "*"  " "   "*" " "    " "            "*"         "*"         
16  ( 1 ) "*"  " "   "*" "*"    " "            "*"         "*"         
17  ( 1 ) "*"  "*"   "*" "*"    " "            "*"         "*"         
18  ( 1 ) "*"  "*"   "*" "*"    "*"            "*"         "*"         
          Underground_level North East South West Front_road_width
1  ( 1 )  " "               " "   " "  " "   " "  " "             
2  ( 1 )  " "               " "   " "  " "   " "  " "             
3  ( 1 )  " "               " "   " "  " "   " "  " "             
4  ( 1 )  " "               " "   " "  " "   " "  " "             
5  ( 1 )  " "               " "   " "  " "   " "  " "             
6  ( 1 )  " "               " "   " "  " "   " "  " "             
7  ( 1 )  " "               " "   " "  " "   " "  " "             
8  ( 1 )  " "               " "   " "  " "   " "  " "             
9  ( 1 )  " "               " "   " "  " "   " "  "*"             
10  ( 1 ) "*"               " "   " "  " "   " "  "*"             
11  ( 1 ) "*"               " "   " "  " "   " "  "*"             
12  ( 1 ) "*"               " "   " "  " "   " "  "*"             
13  ( 1 ) "*"               " "   " "  "*"   " "  "*"             
14  ( 1 ) "*"               "*"   " "  "*"   " "  "*"             
15  ( 1 ) "*"               "*"   "*"  "*"   "*"  "*"             
16  ( 1 ) "*"               "*"   "*"  "*"   "*"  "*"             
17  ( 1 ) "*"               "*"   "*"  "*"   "*"  "*"             
18  ( 1 ) "*"               "*"   "*"  "*"   "*"  "*"             
          Station_distance Construction_ratio WI  DI  GI 
1  ( 1 )  " "              " "                "*" " " " "
2  ( 1 )  " "              " "                "*" " " " "
3  ( 1 )  " "              "*"                "*" " " " "
4  ( 1 )  "*"              "*"                "*" " " " "
5  ( 1 )  " "              "*"                "*" " " " "
6  ( 1 )  "*"              "*"                "*" " " " "
7  ( 1 )  "*"              "*"                "*" "*" " "
8  ( 1 )  "*"              "*"                "*" "*" "*"
9  ( 1 )  "*"              "*"                "*" "*" "*"
10  ( 1 ) "*"              "*"                "*" "*" "*"
11  ( 1 ) "*"              "*"                "*" "*" "*"
12  ( 1 ) "*"              "*"                "*" "*" "*"
13  ( 1 ) "*"              "*"                "*" "*" "*"
14  ( 1 ) "*"              "*"                "*" "*" "*"
15  ( 1 ) "*"              "*"                "*" "*" "*"
16  ( 1 ) "*"              "*"                "*" "*" "*"
17  ( 1 ) "*"              "*"                "*" "*" "*"
18  ( 1 ) "*"              "*"                "*" "*" "*"
bs.resi.summary <- summary(bs.resi)
coef(bs.resi, 16)
       (Intercept)               Area                Gas             Sewage 
       -0.27644534        -0.14562402         0.46864272         0.05412035 
       Depth_ratio       Ground_level  Underground_level              North 
       -0.02465322         0.15661482         0.04259667        -0.05035303 
              East              South               West   Front_road_width 
       -0.05213811        -0.06701737        -0.05411085        -0.04354703 
  Station_distance Construction_ratio                 WI                 DI 
       -0.10465417        -0.12760989         0.49910745         0.06603887 
                GI 
       -0.04938984 
plot(bs.resi.summary$rss, xlab = "Number of Variables",
     ylab = "RSS", type = "l")

plot(bs.resi.summary$adjr2, xlab = "Number of Variables",
     ylab = "Adjusted RSq", type = "l")

which.max(bs.resi.summary$adjr2)
[1] 17
plot(bs.resi.summary$cp, xlab = "Number of Variables",
     ylab = "Cp", type = "l")

which.min(bs.resi.summary$cp)
[1] 16
plot(bs.resi.summary$bic, xlab = "Number of Variables",
     ylab = "BIC", type = "l")

plot(bs.resi, scale = "r2")

plot(bs.resi, scale = "adjr2")

plot(bs.resi, scale = "Cp")

plot(bs.resi, scale = "bic")

4.1.2.3 Stepwise
step2 = step(full_lm_resi, direction = "both", trace = FALSE)
summary(step2)

Call:
lm(formula = Price_19y_log ~ Area + Gas + Sewage + Depth_ratio + 
    Ground_level + Underground_level + North + East + South + 
    West + Front_road_width + Station_distance + Construction_ratio + 
    WI + DI + GI, data = data.train.resi)

Residuals:
     Min       1Q   Median       3Q      Max 
-2.59437 -0.32829  0.01873  0.35051  1.96880 

Coefficients:
                    Estimate Std. Error t value Pr(>|t|)    
(Intercept)        -0.276445   0.023776 -11.627  < 2e-16 ***
Area               -0.145624   0.006645 -21.915  < 2e-16 ***
Gas                 0.468643   0.014288  32.799  < 2e-16 ***
Sewage              0.054120   0.018387   2.943 0.003255 ** 
Depth_ratio        -0.024653   0.005485  -4.495 7.04e-06 ***
Ground_level        0.156615   0.006411  24.430  < 2e-16 ***
Underground_level   0.042597   0.005536   7.695 1.56e-14 ***
North              -0.050353   0.015086  -3.338 0.000848 ***
East               -0.052138   0.014248  -3.659 0.000254 ***
South              -0.067017   0.014995  -4.469 7.94e-06 ***
West               -0.054111   0.014441  -3.747 0.000180 ***
Front_road_width   -0.043547   0.005911  -7.367 1.89e-13 ***
Station_distance   -0.104654   0.005886 -17.781  < 2e-16 ***
Construction_ratio -0.127610   0.005769 -22.120  < 2e-16 ***
WI                  0.499107   0.007340  68.001  < 2e-16 ***
DI                  0.066039   0.006108  10.813  < 2e-16 ***
GI                 -0.049390   0.005491  -8.995  < 2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.5199 on 9228 degrees of freedom
Multiple R-squared:  0.7233,    Adjusted R-squared:  0.7228 
F-statistic:  1508 on 16 and 9228 DF,  p-value: < 2.2e-16
4.1.2.4 Lasso
lasso_resi = glmnet(x_resi, y_resi, alpha=a)
plot(lasso_resi, xvar="lambda", label=TRUE)

set.seed(12032018)
lasso_resi.cv = cv.glmnet(x_resi, y_resi, alpha = a, nfolds = 10)  
plot(lasso_resi.cv)  

lam.min_resi = lasso_resi.cv$lambda.min  
lasso.min_resi = glmnet(x_resi, y_resi, alpha = a, lambda = lam.min_resi)
coef(lasso.min_resi)
19 x 1 sparse Matrix of class "dgCMatrix"
                             s0
(Intercept)        -0.358431441
Area               -0.144339545
Water               0.072201368
Gas                 0.468743411
Sewage              0.052004706
Frontage_ratio      0.002077278
Depth_ratio        -0.022871564
Ground_level        0.154560149
Underground_level   0.041589384
North              -0.041154058
East               -0.044447223
South              -0.058257524
West               -0.046068637
Front_road_width   -0.042202757
Station_distance   -0.103720414
Construction_ratio -0.126794779
WI                  0.499598535
DI                  0.065338336
GI                 -0.048791919
4.1.2.5 Adaptive Lasso
betahat_resi <- coef(lasso.min_resi)
weight_resi = 1 / abs(betahat_resi[-1])
al_resi.cv <- cv.glmnet(x_resi, y_resi, alpha=1, penalty.factor = weight_resi)
al_resi.min <- glmnet(x_resi, y_resi, alpha=1, penalty.factor = weight_resi, lambda=al_resi.cv$lambda.min)
coef(al_resi.min)
19 x 1 sparse Matrix of class "dgCMatrix"
                            s0
(Intercept)        -0.36253858
Area               -0.14497805
Water               0.07807073
Gas                 0.46937772
Sewage              0.05020884
Frontage_ratio      .         
Depth_ratio        -0.02238296
Ground_level        0.15578178
Underground_level   0.04135003
North              -0.04182024
East               -0.04437817
South              -0.05914316
West               -0.04613820
Front_road_width   -0.04237048
Station_distance   -0.10401141
Construction_ratio -0.12771813
WI                  0.49992832
DI                  0.06503702
GI                 -0.04865102
al_resi.min$dev.ratio
[1] 0.7233153
plot(al_resi.cv)

4.1.2.6 Elastic Net
alpha <- seq(0.01, 0.99, 0.01)
mse.df <- NULL
for (i in 1:length(alpha)) {
  m <- cv.glmnet(x = x_resi, y = y_resi, family = "gaussian", alpha = alpha[i])
  mse.df <- rbind(mse.df, data.frame(alpha = alpha[i],
                                     mse = min(m$cvm)))
}
best.alpha <- mse.df$alpha[mse.df$mse == min(mse.df$mse)]
m <- cv.glmnet(x = x_resi, y = y_resi, family = "gaussian", alpha = best.alpha)
best.lambda <- m$lambda.min
en.model <- glmnet(x = x_resi, y = y_resi, family = "gaussian",
                   lambda = best.lambda, alpha = best.alpha)
coefficients(en.model)
19 x 1 sparse Matrix of class "dgCMatrix"
                             s0
(Intercept)        -0.359990634
Area               -0.144260390
Water               0.072668749
Gas                 0.468900971
Sewage              0.052846643
Frontage_ratio      0.002059391
Depth_ratio        -0.022761198
Ground_level        0.154379221
Underground_level   0.041624290
North              -0.040938800
East               -0.044341822
South              -0.058047431
West               -0.045894692
Front_road_width   -0.042077750
Station_distance   -0.103883320
Construction_ratio -0.126451648
WI                  0.498564571
DI                  0.065439070
GI                 -0.048848568

4.1.3 Multicolliniarity check

full_formula_resi <- Price_19y_log~
  Area +
  Water+
  Gas+
  Sewage+
  Frontage_ratio+
  Depth_ratio+
  Ground_level+
  Underground_level+
  North+
  East+
  South+
  West+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI
full_lm_resi <- lm(full_formula_resi, data.train.resi)
vif_full = vif(full_lm_resi) %>%
  data.frame() %>%
  setNames("ols")

bs_formula_resi <- Price_19y_log~
  Area +
  Gas+
  Sewage+
  Depth_ratio +
  Ground_level+
  Underground_level+
  North+
  East+
  South+
  West+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI
bs_lm_resi <- lm(bs_formula_resi, data.train.resi)

vif_bs = vif(bs_lm_resi) %>%
  data.frame() %>%
  setNames( "bs")

ls_formula_resi = Price_19y_log~
  Area +
  Water+
  Gas+
  Sewage+
  Frontage_ratio +
  Depth_ratio+
  Ground_level+
  Underground_level+
  North+
  East+
  South+
  West+
  Front_road_width+
  Station_distance+
  Construction_ratio +
  WI+
  DI+
  GI
ls_lm_resi <- lm(ls_formula_resi, data.train.resi)
vif_ls = vif(ls_lm_resi) %>%
  data.frame() 

al_formula_resi = Price_19y_log~
  Area +
  Water+
  Gas+
  Sewage+
  Depth_ratio+
  Ground_level+
  Underground_level+
  North+
  East+
  South+
  West+
  Front_road_width+
  Station_distance +
  Construction_ratio+
  WI+
  DI+
  GI
al_lm_resi <- lm(al_formula_resi, data.train.resi)
vif_al = vif(al_lm_resi) %>%
  data.frame() %>%
  setNames( "al") %>%
  t()

4.2 Commercial Area

4.2.1 Data Processing

data.vs.comm <- data.analysis.comm_500m %>%
  select(-1,-2,-3,-19:-30,-32, -33,-34) %>%
  mutate(Station_distance = ifelse(Station_distance == 0, 1, Station_distance)) %>%
  mutate(Construction_ratio = 0.01*Construction_ratio) %>%
  mutate(Area = log(Area)) %>%
  mutate(Station_distance = log(Station_distance))

data.non.dammy_comm <- data.vs.comm %>%
  st_drop_geometry() %>%
  select(-c("Water","Gas","Sewage","North","East","South","West")) %>%
  scale() %>%
  data.frame()
data.sd.vs.comm <- cbind(data.non.dammy_comm, data.vs.comm[c("Water","Gas","Sewage","North","East","South","West")])
data.sd.vs.comm <- data.sd.vs.comm %>%
  select(Price_19y_log,Area,Water,Gas,Sewage,Frontage_ratio,
         Depth_ratio,Ground_level,
         Underground_level,
         North,
         East,
         South,
         West,
         Front_road_width,
         Station_distance,
         Construction_ratio,
         WI,
         DI,
         GI,geometry)
4.2.1.1 Data Splitting
set.seed(1)
train.comm <- sample(c(TRUE, FALSE), nrow(data.sd.vs.comm), replace = TRUE)
test.comm <- (!train.comm)
data.train.comm <- data.sd.vs.comm[train.comm,] 
data.train.comm <- data.train.comm %>%
  select(-geometry)
data.test.comm <- data.sd.vs.comm[test.comm,]
skim(data.train.comm)
skim(data.test.comm)

y_comm <- data.train.comm[,1] %>%
  as.matrix()
x_comm <- data.train.comm[, 2:19] %>%
  as.matrix()
a<-1
4.2.1.2 Descriptive Statistics
des.comm = data.sd.vs.comm %>% 
  st_drop_geometry() %>%
  get_summary_stats()
des.tex.comm = data.sd.vs.comm %>%
  st_drop_geometry() %>%
  summary() %>%
  data.frame()

commplot = cor(x_comm) %>%
  data.frame() %>%
  tibble::rownames_to_column(var = "Y")
commplot.long <- pivot_longer(commplot, 
                              cols = Area:GI,
                              names_to = "X", 
                              values_to = "rho") 
commplot.long$X <- factor(commplot.long$X, levels = unique(commplot.long$X))
commplot.long$Y <- factor(commplot.long$Y, levels = rev(unique(commplot.long$Y)))

commplot.long %>%
  ggplot(aes(x = X, y = Y, fill = rho)) + 
  geom_tile() +
  geom_text(aes(label = sprintf("%0.2f", rho)),
            color = "black", size = 2) +
  theme(axis.text.y = element_text(size = 7),
        axis.text.x = element_text(size = 7, angle = -45, hjust = 0, vjust = 1)) +
  scale_fill_gradientn(limits = c(-1, 1), colours = c("#6D9EC1", "white", "#E46726")) +
  coord_equal()

4.2.2 Data Selection Procedure

4.2.2.1 OLS
full_formula_comm <- Price_19y_log~
  Area +
  Water+
  Gas+
  Sewage+
  Frontage_ratio+
  Depth_ratio+
  Ground_level+
  Underground_level+
  North+
  East+
  South+
  West+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI

full_lm_comm <- lm(full_formula_comm, data.train.comm)

summary_full_lm_comm = list(full_lm_comm)
4.2.2.2 Best Subset
bs.comm <- regsubsets(Price_19y_log~., data.train.comm, nvmax = 18)
summary(bs.comm)
Subset selection object
Call: regsubsets.formula(Price_19y_log ~ ., data.train.comm, nvmax = 18)
18 Variables  (and intercept)
                   Forced in Forced out
Area                   FALSE      FALSE
Water                  FALSE      FALSE
Gas                    FALSE      FALSE
Sewage                 FALSE      FALSE
Frontage_ratio         FALSE      FALSE
Depth_ratio            FALSE      FALSE
Ground_level           FALSE      FALSE
Underground_level      FALSE      FALSE
North                  FALSE      FALSE
East                   FALSE      FALSE
South                  FALSE      FALSE
West                   FALSE      FALSE
Front_road_width       FALSE      FALSE
Station_distance       FALSE      FALSE
Construction_ratio     FALSE      FALSE
WI                     FALSE      FALSE
DI                     FALSE      FALSE
GI                     FALSE      FALSE
1 subsets of each size up to 18
Selection Algorithm: exhaustive
          Area Water Gas Sewage Frontage_ratio Depth_ratio Ground_level
1  ( 1 )  " "  " "   " " " "    " "            " "         " "         
2  ( 1 )  " "  " "   " " " "    " "            " "         "*"         
3  ( 1 )  " "  " "   "*" " "    " "            " "         "*"         
4  ( 1 )  " "  " "   "*" " "    " "            " "         "*"         
5  ( 1 )  " "  " "   "*" " "    " "            " "         "*"         
6  ( 1 )  " "  " "   "*" " "    " "            "*"         "*"         
7  ( 1 )  " "  " "   "*" " "    " "            "*"         "*"         
8  ( 1 )  " "  " "   "*" " "    " "            "*"         "*"         
9  ( 1 )  " "  " "   "*" " "    " "            "*"         "*"         
10  ( 1 ) "*"  " "   "*" " "    " "            "*"         "*"         
11  ( 1 ) "*"  " "   "*" " "    " "            "*"         "*"         
12  ( 1 ) "*"  " "   "*" " "    " "            "*"         "*"         
13  ( 1 ) "*"  " "   "*" "*"    " "            "*"         "*"         
14  ( 1 ) "*"  " "   "*" "*"    " "            "*"         "*"         
15  ( 1 ) "*"  "*"   "*" "*"    " "            "*"         "*"         
16  ( 1 ) "*"  "*"   "*" "*"    "*"            "*"         "*"         
17  ( 1 ) "*"  "*"   "*" "*"    "*"            "*"         "*"         
18  ( 1 ) "*"  "*"   "*" "*"    "*"            "*"         "*"         
          Underground_level North East South West Front_road_width
1  ( 1 )  " "               " "   " "  " "   " "  " "             
2  ( 1 )  " "               " "   " "  " "   " "  " "             
3  ( 1 )  " "               " "   " "  " "   " "  " "             
4  ( 1 )  " "               " "   " "  " "   " "  " "             
5  ( 1 )  "*"               " "   " "  " "   " "  " "             
6  ( 1 )  "*"               " "   " "  " "   " "  " "             
7  ( 1 )  "*"               " "   " "  " "   " "  " "             
8  ( 1 )  "*"               " "   " "  " "   " "  " "             
9  ( 1 )  "*"               " "   " "  " "   " "  " "             
10  ( 1 ) "*"               " "   " "  " "   " "  " "             
11  ( 1 ) "*"               " "   " "  " "   " "  "*"             
12  ( 1 ) "*"               " "   " "  " "   "*"  "*"             
13  ( 1 ) "*"               " "   " "  " "   "*"  "*"             
14  ( 1 ) "*"               " "   " "  "*"   "*"  "*"             
15  ( 1 ) "*"               " "   " "  "*"   "*"  "*"             
16  ( 1 ) "*"               " "   " "  "*"   "*"  "*"             
17  ( 1 ) "*"               " "   "*"  "*"   "*"  "*"             
18  ( 1 ) "*"               "*"   "*"  "*"   "*"  "*"             
          Station_distance Construction_ratio WI  DI  GI 
1  ( 1 )  " "              " "                "*" " " " "
2  ( 1 )  " "              " "                "*" " " " "
3  ( 1 )  " "              " "                "*" " " " "
4  ( 1 )  "*"              " "                "*" " " " "
5  ( 1 )  "*"              " "                "*" " " " "
6  ( 1 )  "*"              " "                "*" " " " "
7  ( 1 )  "*"              " "                "*" "*" " "
8  ( 1 )  "*"              " "                "*" "*" "*"
9  ( 1 )  "*"              "*"                "*" "*" "*"
10  ( 1 ) "*"              "*"                "*" "*" "*"
11  ( 1 ) "*"              "*"                "*" "*" "*"
12  ( 1 ) "*"              "*"                "*" "*" "*"
13  ( 1 ) "*"              "*"                "*" "*" "*"
14  ( 1 ) "*"              "*"                "*" "*" "*"
15  ( 1 ) "*"              "*"                "*" "*" "*"
16  ( 1 ) "*"              "*"                "*" "*" "*"
17  ( 1 ) "*"              "*"                "*" "*" "*"
18  ( 1 ) "*"              "*"                "*" "*" "*"
bs.comm.summary <- summary(bs.comm)
coef(bs.comm, 11)
       (Intercept)               Area                Gas        Depth_ratio 
       -0.26978118        -0.02275399         0.37872356        -0.06051983 
      Ground_level  Underground_level   Front_road_width   Station_distance 
        0.19494149         0.12786064         0.01317412        -0.11901282 
Construction_ratio                 WI                 DI                 GI 
       -0.04185803         0.50792937         0.04587783        -0.04532151 
plot(bs.comm.summary$rss, xlab = "Number of Variables",
     ylab = "RSS", type = "l")

plot(bs.comm.summary$adjr2, xlab = "Number of Variables",
     ylab = "Adjusted RSq", type = "l")

which.max(bs.comm.summary$adjr2)
[1] 15
plot(bs.comm.summary$cp, xlab = "Number of Variables",
     ylab = "Cp", type = "l")

which.min(bs.comm.summary$cp)
[1] 11
plot(bs.comm.summary$bic, xlab = "Number of Variables",
     ylab = "BIC", type = "l")

plot(bs.comm, scale = "r2")

plot(bs.comm, scale = "adjr2")

plot(bs.comm, scale = "Cp")

plot(bs.comm, scale = "bic")

4.2.2.3 Stepwise
step2.comm = step(full_lm_comm, direction = "both", trace = FALSE)
summary(step2.comm)

Call:
lm(formula = Price_19y_log ~ Area + Gas + Depth_ratio + Ground_level + 
    Underground_level + Front_road_width + Station_distance + 
    Construction_ratio + WI + DI + GI, data = data.train.comm)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.91789 -0.32983 -0.00913  0.30225  2.59286 

Coefficients:
                    Estimate Std. Error t value Pr(>|t|)    
(Intercept)        -0.269781   0.018084 -14.919  < 2e-16 ***
Area               -0.022754   0.010533  -2.160   0.0308 *  
Gas                 0.378724   0.022704  16.681  < 2e-16 ***
Depth_ratio        -0.060520   0.009023  -6.708 2.33e-11 ***
Ground_level        0.194941   0.012505  15.589  < 2e-16 ***
Underground_level   0.127861   0.010285  12.432  < 2e-16 ***
Front_road_width    0.013174   0.009206   1.431   0.1525    
Station_distance   -0.119013   0.009388 -12.677  < 2e-16 ***
Construction_ratio -0.041858   0.010348  -4.045 5.36e-05 ***
WI                  0.507929   0.012525  40.552  < 2e-16 ***
DI                  0.045878   0.008992   5.102 3.55e-07 ***
GI                 -0.045322   0.008582  -5.281 1.37e-07 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 0.4802 on 3218 degrees of freedom
Multiple R-squared:  0.7702,    Adjusted R-squared:  0.7694 
F-statistic: 980.7 on 11 and 3218 DF,  p-value: < 2.2e-16
4.2.2.4 Lasso
lasso_comm = glmnet(x_comm, y_comm, alpha=a)
plot(lasso_comm, xvar="lambda", label=TRUE)

set.seed(12032018)
lasso_comm.cv = cv.glmnet(x_comm, y_comm, alpha = a, nfolds = 10)  
plot(lasso_comm.cv)  

lam.min_comm = lasso_comm.cv$lambda.min  
lasso.min_comm = glmnet(x_comm, y_comm, alpha = a, lambda = lam.min_comm)
coef(lasso.min_comm)
19 x 1 sparse Matrix of class "dgCMatrix"
                             s0
(Intercept)        -0.539708682
Area               -0.017072578
Water               0.224582928
Gas                 0.370791153
Sewage              0.041188022
Frontage_ratio      0.003956282
Depth_ratio        -0.057239933
Ground_level        0.194710471
Underground_level   0.125582691
North               .          
East                .          
South               0.013811715
West                0.021023428
Front_road_width    0.010208744
Station_distance   -0.116648109
Construction_ratio -0.036156320
WI                  0.507141367
DI                  0.043248267
GI                 -0.042412363
4.2.2.5 Adaptive Lasso
betahat_comm <- coef(lasso.min_comm)
weight_comm = 1 / abs(betahat_comm[-1])
al_comm.cv <- cv.glmnet(x_comm, y_comm, alpha=1, penalty.factor = weight_comm)
al_comm.min <- glmnet(x_comm, y_comm, alpha=1, penalty.factor = weight_comm, lambda=al_comm.cv$lambda.min)
coef(al_comm.min)
19 x 1 sparse Matrix of class "dgCMatrix"
                             s0
(Intercept)        -0.604727515
Area               -0.016523030
Water               0.285193228
Gas                 0.374180834
Sewage              0.043805789
Frontage_ratio      .          
Depth_ratio        -0.059490317
Ground_level        0.195918371
Underground_level   0.126680082
North               .          
East                .          
South               0.012385757
West                0.021583317
Front_road_width    0.007131572
Station_distance   -0.117572043
Construction_ratio -0.039441854
WI                  0.508253994
DI                  0.044460697
GI                 -0.043316605
4.2.2.6 Elastic Net
alpha <- seq(0.01, 0.99, 0.01)
mse.df <- NULL
for (i in 1:length(alpha)) {
  m <- cv.glmnet(x = x_comm, y = y_comm, family = "gaussian", alpha = alpha[i])
  mse.df <- rbind(mse.df, data.frame(alpha = alpha[i],
                                     mse = min(m$cvm)))
}
best.alpha <- mse.df$alpha[mse.df$mse == min(mse.df$mse)]
m <- cv.glmnet(x = x_comm, y = y_comm, family = "gaussian", alpha = best.alpha)
best.lambda <- m$lambda.min
en.model.comm <- glmnet(x = x_comm, y = y_comm, family = "gaussian",
                   lambda = best.lambda, alpha = best.alpha)
coefficients(en.model.comm)

4.2.3 Multicolliniarity check

full_formula_comm <- Price_19y_log~
  Area +
  Water+
  Gas+
  Sewage+
  Frontage_ratio+
  Depth_ratio+
  Ground_level+
  Underground_level+
  North+
  East+
  South+
  West+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI
full_lm_comm = lm(full_formula_comm, data.train.comm)
vif_full_comm = vif(full_lm_comm) %>%
  data.frame() %>%
  setNames("ols")

bs_formula_comm <- Price_19y_log~
  Area +
  Gas+
  Depth_ratio +
  Ground_level+
  Underground_level+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI
bs_lm_comm <- lm(bs_formula_comm, data.train.comm)

vif_bs_comm = vif(bs_lm_comm) %>%
  data.frame() %>%
  setNames( "bs")

sw_formula_comm = Price_19y_log~
  Area +
  Gas+
  Depth_ratio +
  Ground_level+
  Underground_level+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI
sw_lm_comm = lm(sw_formula_comm, data.train.comm)
vif_sw_comm = vif(sw_lm_comm) %>%
  data.frame()

ls_formula_comm = Price_19y_log~
  Area +
  Water+
  Gas+
  Sewage+
  Frontage_ratio +
  Depth_ratio+
  Ground_level+
  Underground_level+
  South+
  West+
  Front_road_width+
  Station_distance+
  Construction_ratio +
  WI+
  DI+
  GI
ls_lm_comm <- lm(ls_formula_comm, data.train.comm)
vif_ls_comm = vif(ls_lm_comm) %>%
  data.frame() 

al_formula_comm = Price_19y_log~
  Area +
  Water+
  Gas+
  Sewage+
  Depth_ratio+
  Ground_level+
  Underground_level+
  South+
  West+
  Front_road_width+
  Station_distance +
  Construction_ratio+
  WI+
  DI+
  GI
al_lm_comm <- lm(al_formula_comm, data.train.comm)
vif_al_comm = vif(al_lm_comm) %>%
  data.frame() %>%
  setNames( "al") %>%
  t()

5 Test Data

5.1 Residential Area

data.test.resi <- data.test.resi %>%
  select(-c("Water", "Frontage_ratio"))
x.test.resi <- data.test.resi[,2:17] %>%
  as.matrix()
y.test.resi <- data.test.resi[,1] %>%
  as.matrix()

plot.test.resi = cor(x.test.resi) %>%
  data.frame() %>%
  tibble::rownames_to_column(var = "Y")
resiplot.test.long <- pivot_longer(plot.test.resi, 
                              cols = Area:GI,
                              names_to = "X", 
                              values_to = "rho") 
resiplot.test.long$X <- factor(resiplot.test.long$X, levels = unique(resiplot.test.long$X))
resiplot.test.long$Y <- factor(resiplot.test.long$Y, levels = rev(unique(resiplot.test.long$Y)))

resiplot.test.long %>%
  ggplot(aes(x = X, y = Y, fill = rho)) + 
  geom_tile() +
  geom_text(aes(label = sprintf("%0.2f", rho)),
            color = "black", size = 2) +
  theme(axis.text.y = element_text(size = 7),
        axis.text.x = element_text(size = 7, angle = -45, hjust = 0, vjust = 1)) +
  scale_fill_gradientn(limits = c(-1, 1), colours = c("#6D9EC1", "white", "#E46726")) +
  coord_equal()

5.2 Commercial Area

data.test.comm <- data.test.comm %>%
  select(-c("Water","Sewage", "Frontage_ratio","North","East","South","West"))
x.test.comm <- data.test.comm[,2:12] %>%
  as.matrix()
y.test.comm <- data.test.comm[,1] %>%
  as.matrix()

plot.test.comm = cor(x.test.comm) %>%
  data.frame() %>%
  tibble::rownames_to_column(var = "Y")
commplot.test.long <- pivot_longer(plot.test.comm, 
                                   cols = Area:GI,
                                   names_to = "X", 
                                   values_to = "rho") 
commplot.test.long$X <- factor(commplot.test.long$X, levels = unique(commplot.test.long$X))
commplot.test.long$Y <- factor(commplot.test.long$Y, levels = rev(unique(commplot.test.long$Y)))

commplot.test.long %>%
  ggplot(aes(x = X, y = Y, fill = rho)) + 
  geom_tile() +
  geom_text(aes(label = sprintf("%0.2f", rho)),
            color = "black", size = 2) +
  theme(axis.text.y = element_text(size = 7),
        axis.text.x = element_text(size = 7, angle = -45, hjust = 0, vjust = 1)) +
  scale_fill_gradientn(limits = c(-1, 1), colours = c("#6D9EC1", "white", "#E46726")) +
  coord_equal()

6 Model Comparison

6.1 Residential Area

6.1.1 OLS

formula.test.resi = Price_19y_log~
  Area +
  Gas+
  Sewage+
  Depth_ratio +
  Ground_level+
  Underground_level+
  North+
  East+
  South+
  West+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI
lm.test.resi <- lm(formula.test.resi, data.test.resi)

6.1.2 Moran’s I

data.test.resi <- data.test.resi %>%
  st_as_sf()
resi_cen = st_centroid(data.test.resi)
Warning: st_centroid assumes attributes are constant over geometries
resi_coord = st_coordinates(resi_cen)
resi_nb <- tri2nb(resi_coord)
w_resi = nb2listw(resi_nb, style = "W")
moran.test(data.test.resi$GI, nb2listw(resi_nb, style = "W") )

    Moran I test under randomisation

data:  data.test.resi$GI  
weights: nb2listw(resi_nb, style = "W")    

Moran I statistic standard deviate = 120.13, p-value < 2.2e-16
alternative hypothesis: greater
sample estimates:
Moran I statistic       Expectation          Variance 
     7.380812e-01     -1.103144e-04      3.776194e-05 
lm.morantest(lm.test.resi, w_resi)  # global moran I

    Global Moran I for regression residuals

data:  
model: lm(formula = formula.test.resi, data = data.test.resi)
weights: w_resi

Moran I statistic standard deviate = 88.983, p-value < 2.2e-16
alternative hypothesis: greater
sample estimates:
Observed Moran I      Expectation         Variance 
    5.459835e-01    -5.234517e-04     3.772074e-05 
lm.LMtests(lm.test.resi, w_resi, test = c('LMlag', 'LMerr')) %>% 
  summary()
    Lagrange multiplier diagnostics for spatial dependence
data:  
model: lm(formula = formula.test.resi, data = data.test.resi)
weights: w_resi
 
      statistic parameter   p.value    
LMlag    9233.9         1 < 2.2e-16 ***
LMerr    7879.1         1 < 2.2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
lm.LMtests(lm.test.resi, w_resi, test = c('RLMlag', 'RLMerr')) %>% 
  summary()
    Lagrange multiplier diagnostics for spatial dependence
data:  
model: lm(formula = formula.test.resi, data = data.test.resi)
weights: w_resi
 
       statistic parameter   p.value    
RLMlag    2720.7         1 < 2.2e-16 ***
RLMerr    1365.9         1 < 2.2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

6.1.3 Spatial Rag Model

resi_rag = lagsarlm(formula.test.resi, data = data.test.resi,listw = w_resi , method = "eigen")
summary.resi.rag = list(resi_rag)
msummary(summary.resi.rag, stars = TRUE)

6.1.4 Spatial Error Model

resi_err = errorsarlm(formula.test.resi, data = data.test.resi, listw = w_resi)
summary.resi.err = list(resi_err)
msummary(summary.resi.err, stars = TRUE)
summary(resi_err)

6.1.5 Geographically Weighted Model

y.gwr.resi <- data.test.resi$Price_19y_log
x.gwr.resi <- subset(data.test.resi, T, c(Area, Gas, Sewage,Depth_ratio, Ground_level, Underground_level,
                                          North, East, South, West,
                                          Front_road_width, Station_distance, Construction_ratio, 
                                          WI, DI, GI)) %>%
  st_drop_geometry() %>%
  as.matrix()
geo.resi <- st_coordinates(resi_cen$geometry) 
colnames(geo.resi) = c("lon", "lat") %>%
  as.matrix(geo.resi)

h.resi = gwr.sel(y.gwr.resi~x.gwr.resi, coords = geo.resi)
gwr.resi <- gwr(y.gwr.resi~x.gwr.resi, coords = geo.resi, bandwidth = h.resi)
summary(gwr.resi$SDF)

6.2 Commercial Area

6.2.1 OLS

formula.test.comm = Price_19y_log~
  Area +
  Gas+
  Depth_ratio +
  Ground_level+
  Underground_level+
  Front_road_width+
  Station_distance+
  Construction_ratio+
  WI+
  DI+
  GI
lm.test.comm <- lm(formula.test.comm, data.test.comm)

6.2.2 Moran’s I

data.test.comm <- data.test.comm %>%
  st_as_sf()
comm_cen = st_centroid(data.test.comm)
Warning: st_centroid assumes attributes are constant over geometries
comm_coord = st_coordinates(comm_cen)
comm_nb <- tri2nb(comm_coord)
w_comm = nb2listw(comm_nb, style = "W")
moran.test(data.test.comm$GI, nb2listw(comm_nb, style = "W") )

    Moran I test under randomisation

data:  data.test.comm$GI  
weights: nb2listw(comm_nb, style = "W")    

Moran I statistic standard deviate = 62.85, p-value < 2.2e-16
alternative hypothesis: greater
sample estimates:
Moran I statistic       Expectation          Variance 
     0.6441817144     -0.0003061849      0.0001051507 
lm.morantest(lm.test.comm, w_comm)  # global moran I

    Global Moran I for regression residuals

data:  
model: lm(formula = formula.test.comm, data = data.test.comm)
weights: w_comm

Moran I statistic standard deviate = 45.542, p-value < 2.2e-16
alternative hypothesis: greater
sample estimates:
Observed Moran I      Expectation         Variance 
    0.4656209697    -0.0011412657     0.0001050448 
lm.LMtests(lm.test.comm, w_comm, test = c('LMlag', 'LMerr')) %>% 
  summary()
    Lagrange multiplier diagnostics for spatial dependence
data:  
model: lm(formula = formula.test.comm, data = data.test.comm)
weights: w_comm
 
      statistic parameter   p.value    
LMlag    2592.3         1 < 2.2e-16 ***
LMerr    2051.0         1 < 2.2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
lm.LMtests(lm.test.comm, w_comm, test = c('RLMlag', 'RLMerr')) %>% 
  summary()
    Lagrange multiplier diagnostics for spatial dependence
data:  
model: lm(formula = formula.test.comm, data = data.test.comm)
weights: w_comm
 
       statistic parameter   p.value    
RLMlag    829.10         1 < 2.2e-16 ***
RLMerr    287.84         1 < 2.2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

6.2.3 Spatial Rag Model

comm_rag = lagsarlm(formula.test.comm, data = data.test.comm,listw = w_comm , method = "eigen")
summary.comm.rag= list(comm_rag)
msummary(summary.comm.rag, stars = TRUE)
summary(comm_rag)

6.2.4 Spatial Error Model

comm_err = errorsarlm(formula.test.comm, data = data.test.comm, listw = w_comm)
summary.comm.err = list(comm_err)
msummary(summary.comm.err, stars = TRUE)
summary(comm_err)

6.2.5 Geographically Weighted Model

y.gwr.comm <- data.test.comm$Price_19y_log
x.gwr.comm <- subset(data.test.comm, T, c(Area, Gas, Depth_ratio, Ground_level, Underground_level, 
                                          Front_road_width, Station_distance, Construction_ratio, 
                                          WI, DI, GI)) %>%
  st_drop_geometry() %>%
  as.matrix()
geo.comm <- st_coordinates(comm_cen$geometry) 
colnames(geo.comm) = c("lon", "lat") %>%
as.matrix(geo.comm)

h.comm = gwr.sel(y.gwr.comm~x.gwr.comm, coords = geo.comm)
gwr.comm <- gwr(y.gwr.comm~x.gwr.comm, coords = geo.comm, bandwidth = h.comm)
gwr.comm$SDF
summary(gwr.comm$SDF)

7 Others

data.merge.resi <- data.vs.resi %>%
  mutate(Land_use = "residential") %>%
  select(-c("Floor_area", "PD")) %>%
  st_drop_geometry()
data.merge.comm <- data.vs.comm %>%
  mutate(Land_use = "commercial") %>%
  st_drop_geometry()
data.merge <-  rbind(data.merge.resi, data.merge.comm)
ggplot(data.merge, aes(x = Price_19y_log, fill = Land_use)) +
  geom_histogram()
`stat_bin()` using `bins = 30`. Pick better value with `binwidth`.