DATA 605 Foundations of Computational Mathematics Final
Brian Liles
December 10, 2018
- Probability
- Descriptive and Inferential Statistics
- Linear Algebra and Correlation
- Calculus Based Probability & Statistics
- Modeling
Load Libraries
library(tidyverse)## Warning: package 'tidyverse' was built under R version 3.4.3## -- Attaching packages ---------------------------------- tidyverse 1.2.1 --## v ggplot2 2.2.1 v purrr 0.2.4
## v tibble 1.4.1 v dplyr 0.7.4
## v tidyr 0.7.2 v stringr 1.2.0
## v readr 1.1.1 v forcats 0.2.0## Warning: package 'tibble' was built under R version 3.4.3## Warning: package 'tidyr' was built under R version 3.4.3## Warning: package 'readr' was built under R version 3.4.3## Warning: package 'purrr' was built under R version 3.4.3## Warning: package 'dplyr' was built under R version 3.4.2## Warning: package 'forcats' was built under R version 3.4.3## -- Conflicts ------------------------------------- tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x dplyr::lag() masks stats::lag()library(matlib)## Warning: package 'matlib' was built under R version 3.4.4library(MASS)## Warning: package 'MASS' was built under R version 3.4.4##
## Attaching package: 'MASS'## The following object is masked from 'package:dplyr':
##
## selectlibrary(pracma)## Warning: package 'pracma' was built under R version 3.4.4##
## Attaching package: 'pracma'## The following objects are masked from 'package:matlib':
##
## angle, inv## The following object is masked from 'package:purrr':
##
## crosslibrary(moments)## Warning: package 'moments' was built under R version 3.4.4Import Kaggle Train Data Set
#load train data into variable entitled house_price
house_kaggle <- read.csv('C:\\Users\\lizza\\Documents\\CUNY - Data Analytics\\Final\\train.csv')
#load test data into variable entitled house_test
house_test <- read.csv('C:\\Users\\lizza\\Documents\\CUNY - Data Analytics\\Final\\test.csv')Using the glimpse function we will explore the house_kaggle data
glimpse(house_kaggle)## Observations: 1,460
## Variables: 81
## $ Id <int> 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 1...
## $ MSSubClass <int> 60, 20, 60, 70, 60, 50, 20, 60, 50, 190, 20, 60,...
## $ MSZoning <fctr> RL, RL, RL, RL, RL, RL, RL, RL, RM, RL, RL, RL,...
## $ LotFrontage <int> 65, 80, 68, 60, 84, 85, 75, NA, 51, 50, 70, 85, ...
## $ LotArea <int> 8450, 9600, 11250, 9550, 14260, 14115, 10084, 10...
## $ Street <fctr> Pave, Pave, Pave, Pave, Pave, Pave, Pave, Pave,...
## $ Alley <fctr> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,...
## $ LotShape <fctr> Reg, Reg, IR1, IR1, IR1, IR1, Reg, IR1, Reg, Re...
## $ LandContour <fctr> Lvl, Lvl, Lvl, Lvl, Lvl, Lvl, Lvl, Lvl, Lvl, Lv...
## $ Utilities <fctr> AllPub, AllPub, AllPub, AllPub, AllPub, AllPub,...
## $ LotConfig <fctr> Inside, FR2, Inside, Corner, FR2, Inside, Insid...
## $ LandSlope <fctr> Gtl, Gtl, Gtl, Gtl, Gtl, Gtl, Gtl, Gtl, Gtl, Gt...
## $ Neighborhood <fctr> CollgCr, Veenker, CollgCr, Crawfor, NoRidge, Mi...
## $ Condition1 <fctr> Norm, Feedr, Norm, Norm, Norm, Norm, Norm, PosN...
## $ Condition2 <fctr> Norm, Norm, Norm, Norm, Norm, Norm, Norm, Norm,...
## $ BldgType <fctr> 1Fam, 1Fam, 1Fam, 1Fam, 1Fam, 1Fam, 1Fam, 1Fam,...
## $ HouseStyle <fctr> 2Story, 1Story, 2Story, 2Story, 2Story, 1.5Fin,...
## $ OverallQual <int> 7, 6, 7, 7, 8, 5, 8, 7, 7, 5, 5, 9, 5, 7, 6, 7, ...
## $ OverallCond <int> 5, 8, 5, 5, 5, 5, 5, 6, 5, 6, 5, 5, 6, 5, 5, 8, ...
## $ YearBuilt <int> 2003, 1976, 2001, 1915, 2000, 1993, 2004, 1973, ...
## $ YearRemodAdd <int> 2003, 1976, 2002, 1970, 2000, 1995, 2005, 1973, ...
## $ RoofStyle <fctr> Gable, Gable, Gable, Gable, Gable, Gable, Gable...
## $ RoofMatl <fctr> CompShg, CompShg, CompShg, CompShg, CompShg, Co...
## $ Exterior1st <fctr> VinylSd, MetalSd, VinylSd, Wd Sdng, VinylSd, Vi...
## $ Exterior2nd <fctr> VinylSd, MetalSd, VinylSd, Wd Shng, VinylSd, Vi...
## $ MasVnrType <fctr> BrkFace, None, BrkFace, None, BrkFace, None, St...
## $ MasVnrArea <int> 196, 0, 162, 0, 350, 0, 186, 240, 0, 0, 0, 286, ...
## $ ExterQual <fctr> Gd, TA, Gd, TA, Gd, TA, Gd, TA, TA, TA, TA, Ex,...
## $ ExterCond <fctr> TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA,...
## $ Foundation <fctr> PConc, CBlock, PConc, BrkTil, PConc, Wood, PCon...
## $ BsmtQual <fctr> Gd, Gd, Gd, TA, Gd, Gd, Ex, Gd, TA, TA, TA, Ex,...
## $ BsmtCond <fctr> TA, TA, TA, Gd, TA, TA, TA, TA, TA, TA, TA, TA,...
## $ BsmtExposure <fctr> No, Gd, Mn, No, Av, No, Av, Mn, No, No, No, No,...
## $ BsmtFinType1 <fctr> GLQ, ALQ, GLQ, ALQ, GLQ, GLQ, GLQ, ALQ, Unf, GL...
## $ BsmtFinSF1 <int> 706, 978, 486, 216, 655, 732, 1369, 859, 0, 851,...
## $ BsmtFinType2 <fctr> Unf, Unf, Unf, Unf, Unf, Unf, Unf, BLQ, Unf, Un...
## $ BsmtFinSF2 <int> 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0,...
## $ BsmtUnfSF <int> 150, 284, 434, 540, 490, 64, 317, 216, 952, 140,...
## $ TotalBsmtSF <int> 856, 1262, 920, 756, 1145, 796, 1686, 1107, 952,...
## $ Heating <fctr> GasA, GasA, GasA, GasA, GasA, GasA, GasA, GasA,...
## $ HeatingQC <fctr> Ex, Ex, Ex, Gd, Ex, Ex, Ex, Ex, Gd, Ex, Ex, Ex,...
## $ CentralAir <fctr> Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y,...
## $ Electrical <fctr> SBrkr, SBrkr, SBrkr, SBrkr, SBrkr, SBrkr, SBrkr...
## $ X1stFlrSF <int> 856, 1262, 920, 961, 1145, 796, 1694, 1107, 1022...
## $ X2ndFlrSF <int> 854, 0, 866, 756, 1053, 566, 0, 983, 752, 0, 0, ...
## $ LowQualFinSF <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
## $ GrLivArea <int> 1710, 1262, 1786, 1717, 2198, 1362, 1694, 2090, ...
## $ BsmtFullBath <int> 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, ...
## $ BsmtHalfBath <int> 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
## $ FullBath <int> 2, 2, 2, 1, 2, 1, 2, 2, 2, 1, 1, 3, 1, 2, 1, 1, ...
## $ HalfBath <int> 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, ...
## $ BedroomAbvGr <int> 3, 3, 3, 3, 4, 1, 3, 3, 2, 2, 3, 4, 2, 3, 2, 2, ...
## $ KitchenAbvGr <int> 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, ...
## $ KitchenQual <fctr> Gd, TA, Gd, Gd, Gd, TA, Gd, TA, TA, TA, TA, Ex,...
## $ TotRmsAbvGrd <int> 8, 6, 6, 7, 9, 5, 7, 7, 8, 5, 5, 11, 4, 7, 5, 5,...
## $ Functional <fctr> Typ, Typ, Typ, Typ, Typ, Typ, Typ, Typ, Min1, T...
## $ Fireplaces <int> 0, 1, 1, 1, 1, 0, 1, 2, 2, 2, 0, 2, 0, 1, 1, 0, ...
## $ FireplaceQu <fctr> NA, TA, TA, Gd, TA, NA, Gd, TA, TA, TA, NA, Gd,...
## $ GarageType <fctr> Attchd, Attchd, Attchd, Detchd, Attchd, Attchd,...
## $ GarageYrBlt <int> 2003, 1976, 2001, 1998, 2000, 1993, 2004, 1973, ...
## $ GarageFinish <fctr> RFn, RFn, RFn, Unf, RFn, Unf, RFn, RFn, Unf, RF...
## $ GarageCars <int> 2, 2, 2, 3, 3, 2, 2, 2, 2, 1, 1, 3, 1, 3, 1, 2, ...
## $ GarageArea <int> 548, 460, 608, 642, 836, 480, 636, 484, 468, 205...
## $ GarageQual <fctr> TA, TA, TA, TA, TA, TA, TA, TA, Fa, Gd, TA, TA,...
## $ GarageCond <fctr> TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA,...
## $ PavedDrive <fctr> Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y,...
## $ WoodDeckSF <int> 0, 298, 0, 0, 192, 40, 255, 235, 90, 0, 0, 147, ...
## $ OpenPorchSF <int> 61, 0, 42, 35, 84, 30, 57, 204, 0, 4, 0, 21, 0, ...
## $ EnclosedPorch <int> 0, 0, 0, 272, 0, 0, 0, 228, 205, 0, 0, 0, 0, 0, ...
## $ X3SsnPorch <int> 0, 0, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...
## $ ScreenPorch <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0...
## $ PoolArea <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
## $ PoolQC <fctr> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,...
## $ Fence <fctr> NA, NA, NA, NA, NA, MnPrv, NA, NA, NA, NA, NA, ...
## $ MiscFeature <fctr> NA, NA, NA, NA, NA, Shed, NA, Shed, NA, NA, NA,...
## $ MiscVal <int> 0, 0, 0, 0, 0, 700, 0, 350, 0, 0, 0, 0, 0, 0, 0,...
## $ MoSold <int> 2, 5, 9, 2, 12, 10, 8, 11, 4, 1, 2, 7, 9, 8, 5, ...
## $ YrSold <int> 2008, 2007, 2008, 2006, 2008, 2009, 2007, 2009, ...
## $ SaleType <fctr> WD, WD, WD, WD, WD, WD, WD, WD, WD, WD, WD, New...
## $ SaleCondition <fctr> Normal, Normal, Normal, Abnorml, Normal, Normal...
## $ SalePrice <int> 208500, 181500, 223500, 140000, 250000, 143000, ...Based on the glimpse function, the house_kaggle data set has 1,460 observations and and 81 variables.
Now we will explore the house_test data
glimpse(house_test)## Observations: 1,459
## Variables: 80
## $ Id <int> 1461, 1462, 1463, 1464, 1465, 1466, 1467, 1468, ...
## $ MSSubClass <int> 20, 20, 60, 60, 120, 60, 20, 60, 20, 20, 120, 16...
## $ MSZoning <fctr> RH, RL, RL, RL, RL, RL, RL, RL, RL, RL, RH, RM,...
## $ LotFrontage <int> 80, 81, 74, 78, 43, 75, NA, 63, 85, 70, 26, 21, ...
## $ LotArea <int> 11622, 14267, 13830, 9978, 5005, 10000, 7980, 84...
## $ Street <fctr> Pave, Pave, Pave, Pave, Pave, Pave, Pave, Pave,...
## $ Alley <fctr> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,...
## $ LotShape <fctr> Reg, IR1, IR1, IR1, IR1, IR1, IR1, IR1, Reg, Re...
## $ LandContour <fctr> Lvl, Lvl, Lvl, Lvl, HLS, Lvl, Lvl, Lvl, Lvl, Lv...
## $ Utilities <fctr> AllPub, AllPub, AllPub, AllPub, AllPub, AllPub,...
## $ LotConfig <fctr> Inside, Corner, Inside, Inside, Inside, Corner,...
## $ LandSlope <fctr> Gtl, Gtl, Gtl, Gtl, Gtl, Gtl, Gtl, Gtl, Gtl, Gt...
## $ Neighborhood <fctr> NAmes, NAmes, Gilbert, Gilbert, StoneBr, Gilber...
## $ Condition1 <fctr> Feedr, Norm, Norm, Norm, Norm, Norm, Norm, Norm...
## $ Condition2 <fctr> Norm, Norm, Norm, Norm, Norm, Norm, Norm, Norm,...
## $ BldgType <fctr> 1Fam, 1Fam, 1Fam, 1Fam, TwnhsE, 1Fam, 1Fam, 1Fa...
## $ HouseStyle <fctr> 1Story, 1Story, 2Story, 2Story, 1Story, 2Story,...
## $ OverallQual <int> 5, 6, 5, 6, 8, 6, 6, 6, 7, 4, 7, 6, 5, 6, 7, 9, ...
## $ OverallCond <int> 6, 6, 5, 6, 5, 5, 7, 5, 5, 5, 5, 5, 5, 6, 6, 5, ...
## $ YearBuilt <int> 1961, 1958, 1997, 1998, 1992, 1993, 1992, 1998, ...
## $ YearRemodAdd <int> 1961, 1958, 1998, 1998, 1992, 1994, 2007, 1998, ...
## $ RoofStyle <fctr> Gable, Hip, Gable, Gable, Gable, Gable, Gable, ...
## $ RoofMatl <fctr> CompShg, CompShg, CompShg, CompShg, CompShg, Co...
## $ Exterior1st <fctr> VinylSd, Wd Sdng, VinylSd, VinylSd, HdBoard, Hd...
## $ Exterior2nd <fctr> VinylSd, Wd Sdng, VinylSd, VinylSd, HdBoard, Hd...
## $ MasVnrType <fctr> None, BrkFace, None, BrkFace, None, None, None,...
## $ MasVnrArea <int> 0, 108, 0, 20, 0, 0, 0, 0, 0, 0, 0, 504, 492, 0,...
## $ ExterQual <fctr> TA, TA, TA, TA, Gd, TA, TA, TA, TA, TA, Gd, TA,...
## $ ExterCond <fctr> TA, TA, TA, TA, TA, TA, Gd, TA, TA, TA, TA, TA,...
## $ Foundation <fctr> CBlock, CBlock, PConc, PConc, PConc, PConc, PCo...
## $ BsmtQual <fctr> TA, TA, Gd, TA, Gd, Gd, Gd, Gd, Gd, TA, Gd, TA,...
## $ BsmtCond <fctr> TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA,...
## $ BsmtExposure <fctr> No, No, No, No, No, No, No, No, Gd, No, No, No,...
## $ BsmtFinType1 <fctr> Rec, ALQ, GLQ, GLQ, ALQ, Unf, ALQ, Unf, GLQ, AL...
## $ BsmtFinSF1 <int> 468, 923, 791, 602, 263, 0, 935, 0, 637, 804, 10...
## $ BsmtFinType2 <fctr> LwQ, Unf, Unf, Unf, Unf, Unf, Unf, Unf, Unf, Re...
## $ BsmtFinSF2 <int> 144, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, ...
## $ BsmtUnfSF <int> 270, 406, 137, 324, 1017, 763, 233, 789, 663, 0,...
## $ TotalBsmtSF <int> 882, 1329, 928, 926, 1280, 763, 1168, 789, 1300,...
## $ Heating <fctr> GasA, GasA, GasA, GasA, GasA, GasA, GasA, GasA,...
## $ HeatingQC <fctr> TA, TA, Gd, Ex, Ex, Gd, Ex, Gd, Gd, TA, Ex, TA,...
## $ CentralAir <fctr> Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y,...
## $ Electrical <fctr> SBrkr, SBrkr, SBrkr, SBrkr, SBrkr, SBrkr, SBrkr...
## $ X1stFlrSF <int> 896, 1329, 928, 926, 1280, 763, 1187, 789, 1341,...
## $ X2ndFlrSF <int> 0, 0, 701, 678, 0, 892, 0, 676, 0, 0, 0, 504, 56...
## $ LowQualFinSF <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
## $ GrLivArea <int> 896, 1329, 1629, 1604, 1280, 1655, 1187, 1465, 1...
## $ BsmtFullBath <int> 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, ...
## $ BsmtHalfBath <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
## $ FullBath <int> 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 2, 1, 1, 2, 1, 2, ...
## $ HalfBath <int> 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, ...
## $ BedroomAbvGr <int> 2, 3, 3, 3, 2, 3, 3, 3, 2, 2, 2, 2, 3, 3, 2, 3, ...
## $ KitchenAbvGr <int> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
## $ KitchenQual <fctr> TA, Gd, TA, Gd, Gd, TA, TA, TA, Gd, TA, Gd, TA,...
## $ TotRmsAbvGrd <int> 5, 6, 6, 7, 5, 7, 6, 7, 5, 4, 5, 5, 6, 6, 4, 10,...
## $ Functional <fctr> Typ, Typ, Typ, Typ, Typ, Typ, Typ, Typ, Typ, Ty...
## $ Fireplaces <int> 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, ...
## $ FireplaceQu <fctr> NA, NA, TA, Gd, NA, TA, NA, Gd, Po, NA, Fa, NA,...
## $ GarageType <fctr> Attchd, Attchd, Attchd, Attchd, Attchd, Attchd,...
## $ GarageYrBlt <int> 1961, 1958, 1997, 1998, 1992, 1993, 1992, 1998, ...
## $ GarageFinish <fctr> Unf, Unf, Fin, Fin, RFn, Fin, Fin, Fin, Unf, Fi...
## $ GarageCars <int> 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 1, 3, ...
## $ GarageArea <int> 730, 312, 482, 470, 506, 440, 420, 393, 506, 525...
## $ GarageQual <fctr> TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA,...
## $ GarageCond <fctr> TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA, TA,...
## $ PavedDrive <fctr> Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y, Y,...
## $ WoodDeckSF <int> 140, 393, 212, 360, 0, 157, 483, 0, 192, 240, 20...
## $ OpenPorchSF <int> 0, 36, 34, 36, 82, 84, 21, 75, 0, 0, 68, 0, 0, 0...
## $ EnclosedPorch <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
## $ X3SsnPorch <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
## $ ScreenPorch <int> 120, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...
## $ PoolArea <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
## $ PoolQC <fctr> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,...
## $ Fence <fctr> MnPrv, NA, MnPrv, NA, NA, NA, GdPrv, NA, NA, Mn...
## $ MiscFeature <fctr> NA, Gar2, NA, NA, NA, NA, Shed, NA, NA, NA, NA,...
## $ MiscVal <int> 0, 12500, 0, 0, 0, 0, 500, 0, 0, 0, 0, 0, 0, 0, ...
## $ MoSold <int> 6, 6, 3, 6, 1, 4, 3, 5, 2, 4, 6, 2, 3, 6, 6, 1, ...
## $ YrSold <int> 2010, 2010, 2010, 2010, 2010, 2010, 2010, 2010, ...
## $ SaleType <fctr> WD, WD, WD, WD, WD, WD, WD, WD, WD, WD, WD, COD...
## $ SaleCondition <fctr> Normal, Normal, Normal, Normal, Normal, Normal,...The house_test data set has 1,459 observations and and 80 variables.
Problem 1
Pick one of the quantitative independent variables (Xi) from the data set below, and define that variable as X. Also, pick one of the dependent variables (Yi) below, and define that as Y.
Using the tibble function from the tidyverse package, a four columns of numeric variables are assigned to the X and Y variables respectively.
variables <- tibble(
'Y1' = c(20.3,19.1,19.3,20.9,22.0,23.5,13.8,18.8,20.9,18.6,22.3,17.6,20.8,
28.7,15.2,20.9,18.4,10.3,26.3,28.1),
'Y2' = c(20.8,14.6,18.0,7.3,19.4,13.5,14.7,15.3,12.6,13.0,13.1,10.3,14.9,
14.8,16.2,15.7,16.3,11.5,12.2,11.8),
'Y3' = c(28.4,21.5,20.8,22.2,21.6,21.8,25.2,22.5,21.1,21.7,21.4,20.8,23.0,
17.4,21.3,15.1,17.8,26.4,21.6,22.5),
'Y4' = c(20.2,18.6,22.6,11.4,23.6,24.0,26.0,26.8,19.7,22.7,16.8,20.2,21.7,
20.9,26.9,16.3,19.9,15.5,26.5,21.7),
'X1' = c(9.3,4.1,22.4,9.1,15.8,7.1,15.9,6.9,16.0,6.7,8.2,16.0,6.4,11.8,3.5,
21.7,12.2,9.3,8.0,6.2),
'X2' = c(7.4,6.4,8.5,9.5,11.8,8.8,8.4,5.1,11.4,15.1,12.6,8.0,10.3,10.4,9.5,
9.5,15.1,6.6,15.4,8.2),
'X3' = c(9.5,3.7,11.7,7.4,5.3,7.4,7.4,8.6,9.1,11.4,8.4,7.3,11.3,4.4,9.3,10.9,
10.9,7.7,7.7,11.5),
'X4' = c(9.3,12.4,19.9,6.9,-1.0,10.6,6.4,10.6,1.2,7.7,15.5,6.9,13.7,3.7,4.4,
11.5,4.2,13.9,12.9,1.2)
)#view the head of the variables tibble
head(variables)## # A tibble: 6 x 8
## Y1 Y2 Y3 Y4 X1 X2 X3 X4
## <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 20.3 20.8 28.4 20.2 9.30 7.40 9.50 9.30
## 2 19.1 14.6 21.5 18.6 4.10 6.40 3.70 12.4
## 3 19.3 18.0 20.8 22.6 22.4 8.50 11.7 19.9
## 4 20.9 7.30 22.2 11.4 9.10 9.50 7.40 6.90
## 5 22.0 19.4 21.6 23.6 15.8 11.8 5.30 - 1.00
## 6 23.5 13.5 21.8 24.0 7.10 8.80 7.40 10.6When the quantile function is utilized it provides the end user with quartiles of each column. For this project, columns X3 and Y3 were chosen.
cat("Quartiles of Column X3","\n","\n")## Quartiles of Column X3
## quantile(variables$X3)## 0% 25% 50% 75% 100%
## 3.7 7.4 8.5 10.9 11.7cat("Quartiles of Column Y3","\n","\n")## Quartiles of Column Y3
## quantile(variables$Y3)## 0% 25% 50% 75% 100%
## 15.100 21.025 21.600 22.500 28.400Problem 1 - Probability
Calculate as a minimum the below probabilities a through c. Assume the small letter “x” is estimated as the 3rd quartile of the X variable, and the small letter “y” is estimated as the 1st quartile of the Y variable. Interpret the meaning of all probabilities.
(Hint: P(X>3d quartile of x values | Y> 1st quartile of y values.)
A
P(X > x | Y > y)
In order to achieve P(X > x | Y > y) we first calculate the denominator by utilizing the nrow and subset functions to identify how many variables are greater than the 1st quartile in Y3 (21.025); assigned variable will be entitled y_denom1
cat("Number of Variables Larger Than Q1 in Y3","\n","\n")## Number of Variables Larger Than Q1 in Y3
## (y_denom1 <- nrow(subset(variables, variables$Y3 > 21.025)))## [1] 15Next, using the same functions we identify which variables are larger than the 3rd quartile for X3 (10.9) and Y3 (21.025) and divide by y_denom1
cat("P(X > x | Y > y)","\n","\n")## P(X > x | Y > y)
## (prob_1 <- nrow(subset(variables, variables$X3 > 10.9 &
variables$Y3 > 21.025))/y_denom1)## [1] 0.2B
P(X > x , Y > y)
For P(X > x, Y > y) the only difference in obtaining the probability is we will divide by the total number of rows in the variables data set instead of the number of variables greater than Y3; prob_2 will host the probability calculation.
cat("P(X > x , Y > y)","\n","\n")## P(X > x , Y > y)
## (prob_2 <- nrow(subset(variables, variables$X3 > 10.9 & variables$Y3 > 21.025))/nrow(variables))## [1] 0.15C
P(X < x | Y > y)
For P(X < x | Y > y) we revert back to utilizing the y_denom1 variable, but the probability shifts for the X variable from greater than to less than x; prob_3 will host the probability calculation.
cat("P(X < x , Y > y)","\n","\n")## P(X < x , Y > y)
## (prob_3 <- nrow(subset(variables, variables$X3 < 10.9 & variables$Y3 > 21.025))/y_denom1)## [1] 0.8Problem 1 - Make a Table of Counts
Based off the probabilities calculated previously, the next step involves creating a table of counts.
Numbers for the tables will be acquired by conducting the length function which obtains the length/count of the probabilities that reside within the brackets; values will be assigned as follows (p1,p2,etc…)
cat("<=1st Quartile & <=3rd Quartile","\n","\n")## <=1st Quartile & <=3rd Quartile
## (p1 <- length(variables$X3[variables$X3 <= 10.9 & variables$Y3 <= 21.025]))## [1] 4cat("\n","<=1st Quartile & >3rd Quartile","\n","\n")##
## <=1st Quartile & >3rd Quartile
## (p2 <- length(variables$X3[variables$X3 <= 10.9 & variables$Y3 > 21.025]))## [1] 12cat("\n",">1st Quartile & <=3rd Quartile","\n","\n")##
## >1st Quartile & <=3rd Quartile
## (p3 <- length(variables$X3[variables$X3 > 10.9 & variables$Y3 <= 21.025]))## [1] 1cat("\n",">1st Quartile & >3rd Quartile","\n","\n")##
## >1st Quartile & >3rd Quartile
## (p4 <- length(variables$X3[variables$X3 > 10.9 & variables$Y3 > 21.025]))## [1] 3cat("\n","<=3rd Quartile Total Column","\n","\n")##
## <=3rd Quartile Total Column
## (p5 <- p1 + p3)## [1] 5cat("\n",">3rd Quartile Total Column","\n","\n")##
## >3rd Quartile Total Column
## (p6 <- p2 + p4)## [1] 15cat("\n","<=1st quartile Total Row","\n","\n")##
## <=1st quartile Total Row
## (p7 <- p1 + p2)## [1] 16cat("\n",">1st quartile Total Row","\n","\n")##
## >1st quartile Total Row
## (p8 <- p3 + p4)## [1] 4cat("\n","Total Column","\n","\n")##
## Total Column
## (p9 <- p7 + p8)## [1] 20Create a dataframe named counts that will house the results.
r_names <- c("<=3rd Quartile",">3rd Quartile","Total")
c_names <- c("<=1st Quartile",">1st Quartile","Total")
(counts <- matrix(c(p1,p2,p7,
p3,p4,p8,
p5,p6,p9),nrow=3,byrow=TRUE,
dimnames = list(r_names,c_names)))## <=1st Quartile >1st Quartile Total
## <=3rd Quartile 4 12 16
## >3rd Quartile 1 3 4
## Total 5 15 20Problem 1 - Splitting Data/Chi-Square
Does splitting the training data in this fashion make them independent? Let A be the new variable counting those observations above the 1st quartile for X, and let B be the new variable counting those observations above the 1st quartile for Y. Does P(AB)=P(A)P(B)? Check mathematically, and then evaluate by running a Chi Square test for association.
In terms of the training data set, we can conduct the chi-square test on two new selected variables. For this example, we will use the LotArea and SalePrice variables.
#run the quantile function in order to identify the 1st quartile
#and assign them to the variables x_n and y_n
cat("1st Quartile for the Lot Area Variable","\n","\n")## 1st Quartile for the Lot Area Variable
## (x_n <- quantile(house_kaggle$LotArea,0.25))## 25%
## 7553.5cat("\n","1st Quartile for the Sale Price Variable","\n","\n")##
## 1st Quartile for the Sale Price Variable
## (y_n <- quantile(house_kaggle$SalePrice,0.25))## 25%
## 129975cat("P(AB)","\n","\n")## P(AB)
## (PAB <- nrow(subset(house_kaggle, house_kaggle$LotArea > x_n & house_kaggle$SalePrice > y_n))/nrow(house_kaggle))## [1] 0.6150685P_A <- nrow(subset(house_kaggle, house_kaggle$LotArea > x_n)) /nrow(house_kaggle)
P_B <- nrow(subset(house_kaggle, house_kaggle$SalePrice > y_n))/ nrow(house_kaggle)
cat("P(A)P(B)","\n","\n")## P(A)P(B)
## P_A*P_B## [1] 0.5625We check to see if these two probabilites are equal to one another
cat("Equality Check","\n","\n")## Equality Check
## PAB == P_A*P_B## [1] FALSELastly, we will run the chi-square test to double check for equality
Lot_SP <- subset(house_kaggle, select=c("GrLivArea","SalePrice"))
chisq.test(Lot_SP)##
## Pearson's Chi-squared test
##
## data: Lot_SP
## X-squared = 200960, df = 1459, p-value < 2.2e-16Values are deemed independent due to low p-values and equality calculations.
Problem 2
Descriptive and Inferential Statistics
Provide univariate descriptive statistics and appropriate plots for the training data set. Provide a scatterplot matrix for at least two of the independent variables and the dependent variable. Derive a correlation matrix for any THREE quantitative variables in the dataset. Test the hypotheses that the correlations between each pairwise set of variables is 0 and provide a 80% confidence interval. Discuss the meaning of your analysis. Would you be worried about familywise error? Why or why not?
First, we will provide univariate descriptive statistics by using the summary function from Base R.
When importing the data using read_csv from the tidyverse package it was noticed that the summary function would only present frequency counts on numerical data; categorical data was bypassed
summary(house_kaggle)## Id MSSubClass MSZoning LotFrontage
## Min. : 1.0 Min. : 20.0 C (all): 10 Min. : 21.00
## 1st Qu.: 365.8 1st Qu.: 20.0 FV : 65 1st Qu.: 59.00
## Median : 730.5 Median : 50.0 RH : 16 Median : 69.00
## Mean : 730.5 Mean : 56.9 RL :1151 Mean : 70.05
## 3rd Qu.:1095.2 3rd Qu.: 70.0 RM : 218 3rd Qu.: 80.00
## Max. :1460.0 Max. :190.0 Max. :313.00
## NA's :259
## LotArea Street Alley LotShape LandContour
## Min. : 1300 Grvl: 6 Grvl: 50 IR1:484 Bnk: 63
## 1st Qu.: 7554 Pave:1454 Pave: 41 IR2: 41 HLS: 50
## Median : 9478 NA's:1369 IR3: 10 Low: 36
## Mean : 10517 Reg:925 Lvl:1311
## 3rd Qu.: 11602
## Max. :215245
##
## Utilities LotConfig LandSlope Neighborhood Condition1
## AllPub:1459 Corner : 263 Gtl:1382 NAmes :225 Norm :1260
## NoSeWa: 1 CulDSac: 94 Mod: 65 CollgCr:150 Feedr : 81
## FR2 : 47 Sev: 13 OldTown:113 Artery : 48
## FR3 : 4 Edwards:100 RRAn : 26
## Inside :1052 Somerst: 86 PosN : 19
## Gilbert: 79 RRAe : 11
## (Other):707 (Other): 15
## Condition2 BldgType HouseStyle OverallQual
## Norm :1445 1Fam :1220 1Story :726 Min. : 1.000
## Feedr : 6 2fmCon: 31 2Story :445 1st Qu.: 5.000
## Artery : 2 Duplex: 52 1.5Fin :154 Median : 6.000
## PosN : 2 Twnhs : 43 SLvl : 65 Mean : 6.099
## RRNn : 2 TwnhsE: 114 SFoyer : 37 3rd Qu.: 7.000
## PosA : 1 1.5Unf : 14 Max. :10.000
## (Other): 2 (Other): 19
## OverallCond YearBuilt YearRemodAdd RoofStyle
## Min. :1.000 Min. :1872 Min. :1950 Flat : 13
## 1st Qu.:5.000 1st Qu.:1954 1st Qu.:1967 Gable :1141
## Median :5.000 Median :1973 Median :1994 Gambrel: 11
## Mean :5.575 Mean :1971 Mean :1985 Hip : 286
## 3rd Qu.:6.000 3rd Qu.:2000 3rd Qu.:2004 Mansard: 7
## Max. :9.000 Max. :2010 Max. :2010 Shed : 2
##
## RoofMatl Exterior1st Exterior2nd MasVnrType MasVnrArea
## CompShg:1434 VinylSd:515 VinylSd:504 BrkCmn : 15 Min. : 0.0
## Tar&Grv: 11 HdBoard:222 MetalSd:214 BrkFace:445 1st Qu.: 0.0
## WdShngl: 6 MetalSd:220 HdBoard:207 None :864 Median : 0.0
## WdShake: 5 Wd Sdng:206 Wd Sdng:197 Stone :128 Mean : 103.7
## ClyTile: 1 Plywood:108 Plywood:142 NA's : 8 3rd Qu.: 166.0
## Membran: 1 CemntBd: 61 CmentBd: 60 Max. :1600.0
## (Other): 2 (Other):128 (Other):136 NA's :8
## ExterQual ExterCond Foundation BsmtQual BsmtCond BsmtExposure
## Ex: 52 Ex: 3 BrkTil:146 Ex :121 Fa : 45 Av :221
## Fa: 14 Fa: 28 CBlock:634 Fa : 35 Gd : 65 Gd :134
## Gd:488 Gd: 146 PConc :647 Gd :618 Po : 2 Mn :114
## TA:906 Po: 1 Slab : 24 TA :649 TA :1311 No :953
## TA:1282 Stone : 6 NA's: 37 NA's: 37 NA's: 38
## Wood : 3
##
## BsmtFinType1 BsmtFinSF1 BsmtFinType2 BsmtFinSF2
## ALQ :220 Min. : 0.0 ALQ : 19 Min. : 0.00
## BLQ :148 1st Qu.: 0.0 BLQ : 33 1st Qu.: 0.00
## GLQ :418 Median : 383.5 GLQ : 14 Median : 0.00
## LwQ : 74 Mean : 443.6 LwQ : 46 Mean : 46.55
## Rec :133 3rd Qu.: 712.2 Rec : 54 3rd Qu.: 0.00
## Unf :430 Max. :5644.0 Unf :1256 Max. :1474.00
## NA's: 37 NA's: 38
## BsmtUnfSF TotalBsmtSF Heating HeatingQC CentralAir
## Min. : 0.0 Min. : 0.0 Floor: 1 Ex:741 N: 95
## 1st Qu.: 223.0 1st Qu.: 795.8 GasA :1428 Fa: 49 Y:1365
## Median : 477.5 Median : 991.5 GasW : 18 Gd:241
## Mean : 567.2 Mean :1057.4 Grav : 7 Po: 1
## 3rd Qu.: 808.0 3rd Qu.:1298.2 OthW : 2 TA:428
## Max. :2336.0 Max. :6110.0 Wall : 4
##
## Electrical X1stFlrSF X2ndFlrSF LowQualFinSF
## FuseA: 94 Min. : 334 Min. : 0 Min. : 0.000
## FuseF: 27 1st Qu.: 882 1st Qu.: 0 1st Qu.: 0.000
## FuseP: 3 Median :1087 Median : 0 Median : 0.000
## Mix : 1 Mean :1163 Mean : 347 Mean : 5.845
## SBrkr:1334 3rd Qu.:1391 3rd Qu.: 728 3rd Qu.: 0.000
## NA's : 1 Max. :4692 Max. :2065 Max. :572.000
##
## GrLivArea BsmtFullBath BsmtHalfBath FullBath
## Min. : 334 Min. :0.0000 Min. :0.00000 Min. :0.000
## 1st Qu.:1130 1st Qu.:0.0000 1st Qu.:0.00000 1st Qu.:1.000
## Median :1464 Median :0.0000 Median :0.00000 Median :2.000
## Mean :1515 Mean :0.4253 Mean :0.05753 Mean :1.565
## 3rd Qu.:1777 3rd Qu.:1.0000 3rd Qu.:0.00000 3rd Qu.:2.000
## Max. :5642 Max. :3.0000 Max. :2.00000 Max. :3.000
##
## HalfBath BedroomAbvGr KitchenAbvGr KitchenQual
## Min. :0.0000 Min. :0.000 Min. :0.000 Ex:100
## 1st Qu.:0.0000 1st Qu.:2.000 1st Qu.:1.000 Fa: 39
## Median :0.0000 Median :3.000 Median :1.000 Gd:586
## Mean :0.3829 Mean :2.866 Mean :1.047 TA:735
## 3rd Qu.:1.0000 3rd Qu.:3.000 3rd Qu.:1.000
## Max. :2.0000 Max. :8.000 Max. :3.000
##
## TotRmsAbvGrd Functional Fireplaces FireplaceQu GarageType
## Min. : 2.000 Maj1: 14 Min. :0.000 Ex : 24 2Types : 6
## 1st Qu.: 5.000 Maj2: 5 1st Qu.:0.000 Fa : 33 Attchd :870
## Median : 6.000 Min1: 31 Median :1.000 Gd :380 Basment: 19
## Mean : 6.518 Min2: 34 Mean :0.613 Po : 20 BuiltIn: 88
## 3rd Qu.: 7.000 Mod : 15 3rd Qu.:1.000 TA :313 CarPort: 9
## Max. :14.000 Sev : 1 Max. :3.000 NA's:690 Detchd :387
## Typ :1360 NA's : 81
## GarageYrBlt GarageFinish GarageCars GarageArea GarageQual
## Min. :1900 Fin :352 Min. :0.000 Min. : 0.0 Ex : 3
## 1st Qu.:1961 RFn :422 1st Qu.:1.000 1st Qu.: 334.5 Fa : 48
## Median :1980 Unf :605 Median :2.000 Median : 480.0 Gd : 14
## Mean :1979 NA's: 81 Mean :1.767 Mean : 473.0 Po : 3
## 3rd Qu.:2002 3rd Qu.:2.000 3rd Qu.: 576.0 TA :1311
## Max. :2010 Max. :4.000 Max. :1418.0 NA's: 81
## NA's :81
## GarageCond PavedDrive WoodDeckSF OpenPorchSF EnclosedPorch
## Ex : 2 N: 90 Min. : 0.00 Min. : 0.00 Min. : 0.00
## Fa : 35 P: 30 1st Qu.: 0.00 1st Qu.: 0.00 1st Qu.: 0.00
## Gd : 9 Y:1340 Median : 0.00 Median : 25.00 Median : 0.00
## Po : 7 Mean : 94.24 Mean : 46.66 Mean : 21.95
## TA :1326 3rd Qu.:168.00 3rd Qu.: 68.00 3rd Qu.: 0.00
## NA's: 81 Max. :857.00 Max. :547.00 Max. :552.00
##
## X3SsnPorch ScreenPorch PoolArea PoolQC
## Min. : 0.00 Min. : 0.00 Min. : 0.000 Ex : 2
## 1st Qu.: 0.00 1st Qu.: 0.00 1st Qu.: 0.000 Fa : 2
## Median : 0.00 Median : 0.00 Median : 0.000 Gd : 3
## Mean : 3.41 Mean : 15.06 Mean : 2.759 NA's:1453
## 3rd Qu.: 0.00 3rd Qu.: 0.00 3rd Qu.: 0.000
## Max. :508.00 Max. :480.00 Max. :738.000
##
## Fence MiscFeature MiscVal MoSold
## GdPrv: 59 Gar2: 2 Min. : 0.00 Min. : 1.000
## GdWo : 54 Othr: 2 1st Qu.: 0.00 1st Qu.: 5.000
## MnPrv: 157 Shed: 49 Median : 0.00 Median : 6.000
## MnWw : 11 TenC: 1 Mean : 43.49 Mean : 6.322
## NA's :1179 NA's:1406 3rd Qu.: 0.00 3rd Qu.: 8.000
## Max. :15500.00 Max. :12.000
##
## YrSold SaleType SaleCondition SalePrice
## Min. :2006 WD :1267 Abnorml: 101 Min. : 34900
## 1st Qu.:2007 New : 122 AdjLand: 4 1st Qu.:129975
## Median :2008 COD : 43 Alloca : 12 Median :163000
## Mean :2008 ConLD : 9 Family : 20 Mean :180921
## 3rd Qu.:2009 ConLI : 5 Normal :1198 3rd Qu.:214000
## Max. :2010 ConLw : 5 Partial: 125 Max. :755000
## (Other): 9Plots
ggplot(data=house_kaggle) +
geom_point(mapping = aes(x = GarageArea, y = SalePrice,
color = GarageQual)) +
ggtitle("Housing Sale Price vs Garage Area")
By adding the GarageQual variable to the color parameter, one can see that there is a hint of Fair quality garages on the lower end of the market and the mass majority is considered typical/average. In addition, we see that between the 500 to 750 square feet garage sizes there are a sprinkle of poor quality garages.
ggplot(data = house_kaggle) +
geom_point(mapping = aes(x = YearBuilt, y = SalePrice),
color = "green")+
xlab("Lot Area") +
ylab ("Sale Price") +
ggtitle("Housing Sale Price vs Year Built")
This scatterplot shows us that the after the second world war the bulk of homes were built and the prices of homes surged from the 90’s to the 2000’s.
ggplot(data = house_kaggle) +
geom_point(mapping = aes(x = GarageArea, y = SalePrice))+
facet_wrap(~ GarageType, nrow=5) +
ggtitle("Sale Price & Garaga Area by Garage Type")
Utilizing facet_wrap we add the GarageType variable. One can determine that 2Types, Basement, and Car Ports are the least attractive. The bulk of garage types are Attached, Detached, and Built-In.
ggplot(data = house_kaggle)+
geom_bar(mapping = aes(x = Foundation))+
ggtitle("Types of Foundation")
#obtain the count of the Foundation variable
house_kaggle %>% count(Foundation)## # A tibble: 6 x 2
## Foundation n
## <fctr> <int>
## 1 BrkTil 146
## 2 CBlock 634
## 3 PConc 647
## 4 Slab 24
## 5 Stone 6
## 6 Wood 3A bar chart was conducted in order to investigate the Foundation variable.Based on our findings, one can determine that most homes were built using cinder block and poured concrete while stone & wood are the least used.
ggplot(data = house_kaggle)+
geom_bar(mapping = aes(x = CentralAir))+
ggtitle("Houses with Central Air Conditioning")
#obtain the count of the CentralAir variable
house_kaggle %>% count(CentralAir)## # A tibble: 2 x 2
## CentralAir n
## <fctr> <int>
## 1 N 95
## 2 Y 1365Another bar chart highlights the disparity between homes with and without central air conditioning.
ggplot(data = house_kaggle, mapping = aes(x = HouseStyle, y = SalePrice)) +
geom_boxplot()+
ggtitle("Sale Price vs Type of Dwelling")
Utilizing the box & whisker plot, we investigate the sale price versus the style of home. Based off our findings it is clear that 2 story homes are the most expensive in this region.
ggplot(data = house_kaggle, mapping = aes(x = RoofStyle, y = SalePrice)) +
geom_boxplot()+
coord_flip()+
ggtitle("Sale Price vs Roof Style")
Here, we investigate sale price versus roof styles, in this particular instance we flip the coordinates with the coord_flip component.
ggplot(data = house_kaggle)+
geom_histogram(mapping = aes(x = X1stFlrSF),
binwidth=200, color="red")+
ggtitle("Histogram of Houses & First Floor Square Size")+
xlab("First Floor Square Feet")
Based off this histogram, it is clear that the bulk of homes within this region linger around 1000 square feet.
ggplot(data = house_kaggle)+
geom_histogram(mapping = aes(x = YearBuilt),
binwidth=10, color="blue")+
ggtitle("Years Houses Were Built")
This histogram shows us that the count of homes dropped around the 80’s and 90’s and skyrocketed around the 2000’s
Scatterplot Matrix
Next, we will provide a scatterplot matrix for at least two of the independent variables and the dependent variable. Using the select function, we will subset the house_kaggle data set. For independent variables we will select GrLivArea (Above ground living area sq ft) and LotArea (Lot size in square feet), and the dependent variable will be SalePrice; the data set will be entitled house_scatter_m.
house_scatter_m <- subset(house_kaggle, select=
c("GrLivArea","LotArea","SalePrice"))
head(house_scatter_m)## GrLivArea LotArea SalePrice
## 1 1710 8450 208500
## 2 1262 9600 181500
## 3 1786 11250 223500
## 4 1717 9550 140000
## 5 2198 14260 250000
## 6 1362 14115 143000Using the pairs function we can create a scatterplot matrix
pairs(house_scatter_m)
Next, we will create separate scatterplots for each combination.
ggplot(data = house_scatter_m) +
geom_point(mapping = aes(x = GrLivArea, y = SalePrice),
color = "blue") +
xlab("Above Ground Living Area Sq Feet") +
ylab("Housing Data Sale Price") +
ggtitle("Housing Sale Price & Above Ground Living Area Square Feet")
This scatterplot shows a strong positive relationship with minimal outliers.
ggplot(data = house_scatter_m) +
geom_point(mapping = aes(x = LotArea, y = SalePrice),
color = "red") +
xlab("Lot Size in Square Feet") +
ylab("Housing Data Sale Price") +
ggtitle("Housing Sale Price & Lot Size Square Feet")
Correlation Matrix
Next, we will create the correlation matrix for three quantitative variables. As we did in the scatterplot matrix we will create a subset of the data using the select function. The variables we have chosen are GrLivArea (Above ground living area sq ft),X1stFlrSF (First Floor Square Feet), and SalePrice. Data will be applied to the house_correlation data set.
house_correlation <- subset(house_kaggle, select = c("GrLivArea","X1stFlrSF","SalePrice"))
head(house_correlation)## GrLivArea X1stFlrSF SalePrice
## 1 1710 856 208500
## 2 1262 1262 181500
## 3 1786 920 223500
## 4 1717 961 140000
## 5 2198 1145 250000
## 6 1362 796 143000According to http://www.sthda.com/english/wiki/correlation-matrix-a-quick-start-guide-to-analyze-format-and-visualize-a-correlation-matrix-using-r-software a correlation matrix is used to investigate the dependence between multiple variables at the same time. We will create our matrix using the cor function. We will apply our findings to a variable entitled house_cor and will round to 2 digits.
cat("Computing the Correlation Matrix House Correlation","\n","\n")## Computing the Correlation Matrix House Correlation
## house_cor <- cor(house_correlation)
round(house_cor, 2)## GrLivArea X1stFlrSF SalePrice
## GrLivArea 1.00 0.57 0.71
## X1stFlrSF 0.57 1.00 0.61
## SalePrice 0.71 0.61 1.00Next, we will test the hypotheses that the correlations between each pairwise set of variables is 0 and provide a 80% confidence interval.
cor.test(house_correlation$GrLivArea,
house_correlation$X1stFlrSF,
conf.level = 0.8)##
## Pearson's product-moment correlation
##
## data: house_correlation$GrLivArea and house_correlation$X1stFlrSF
## t = 26.217, df = 1458, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 80 percent confidence interval:
## 0.5427732 0.5884078
## sample estimates:
## cor
## 0.566024Based on the findings there is a moderate positive relationship between the two variables and you can reject the null hypothesis.
cor.test(house_correlation$GrLivArea,
house_correlation$SalePrice,
conf.level = 0.8)##
## Pearson's product-moment correlation
##
## data: house_correlation$GrLivArea and house_correlation$SalePrice
## t = 38.348, df = 1458, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 80 percent confidence interval:
## 0.6915087 0.7249450
## sample estimates:
## cor
## 0.7086245Based on the findings there is a strong positive relationship between the two variables and you can reject the null hypothesis.
cor.test(house_correlation$X1stFlrSF,
house_correlation$SalePrice,
conf.level = 0.8)##
## Pearson's product-moment correlation
##
## data: house_correlation$X1stFlrSF and house_correlation$SalePrice
## t = 29.078, df = 1458, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 80 percent confidence interval:
## 0.5841687 0.6266715
## sample estimates:
## cor
## 0.6058522Based on the findings there is a moderate-strong positive relationship between the two variables and you can reject the null hypothesis.
Linear Algebra and Correlation
Invert your 3 x 3 correlation matrix from above. (This is known as the precision matrix and contains variance inflation factors on the diagonal.) Multiply the correlation matrix by the precision matrix, and then multiply the precision matrix by the correlation matrix. Conduct LU decomposition on the matrix.
First, we will recreate the correlation matrix and name it c_matrix
cat("3 x 3 Correlation Matrix","\n","\n")## 3 x 3 Correlation Matrix
## (c_matrix <- matrix(c(1.00,0.57,0.71,
0.57,1.00,0.61,
0.71,0.61,1.00), nrow=3,byrow=TRUE))## [,1] [,2] [,3]
## [1,] 1.00 0.57 0.71
## [2,] 0.57 1.00 0.61
## [3,] 0.71 0.61 1.00Next, we will create the inverse of c_matrix using the inv function from the matlib package; the precision matrix will be entitled p_matrix
cat("3 x 3 Precision Matrix","\n","\n")## 3 x 3 Precision Matrix
## (p_matrix <- inv(c_matrix))## [,1] [,2] [,3]
## [1,] 2.1456837 -0.4678199 -1.2380653
## [2,] -0.4678199 1.6946083 -0.7015589
## [3,] -1.2380653 -0.7015589 2.3069773We will now muliply c_matrix by p_matrix
cat("Multiplication of c_matrix & p_matrix","\n","\n")## Multiplication of c_matrix & p_matrix
## (m1 <- c_matrix %*% p_matrix)## [,1] [,2] [,3]
## [1,] 1.000000e+00 1.082034e-16 2.385245e-18
## [2,] -1.013729e-17 1.000000e+00 9.801188e-17
## [3,] 0.000000e+00 1.110223e-16 1.000000e+00We will now muliply p_matrix by c_matrix
cat("Multiplication of p_matrix & c_matrix","\n","\n")## Multiplication of p_matrix & c_matrix
## (m2 <- p_matrix %*% c_matrix)## [,1] [,2] [,3]
## [1,] 1.000000e+00 1.252796e-16 2.220446e-16
## [2,] 1.082034e-16 1.000000e+00 1.110223e-16
## [3,] -2.196594e-16 9.801188e-17 1.000000e+00From the pracma package the LU component was used in order to conduct matrix factorization of the matrices that were multiplied as well as the original correlation matrix.
cat("LU Decomposition of the m2 Matrix","\n","\n")## LU Decomposition of the m2 Matrix
## (LU_m2 <- lu(m2))## $L
## [,1] [,2] [,3]
## [1,] 1.000000e+00 0.000000e+00 0
## [2,] 1.082034e-16 1.000000e+00 0
## [3,] -2.196594e-16 9.801188e-17 1
##
## $U
## [,1] [,2] [,3]
## [1,] 1 1.252796e-16 2.220446e-16
## [2,] 0 1.000000e+00 1.110223e-16
## [3,] 0 0.000000e+00 1.000000e+00cat("LU Decomposition of the Correlation Matrix","\n","\n")## LU Decomposition of the Correlation Matrix
## (LU_c_matrix <- lu(c_matrix))## $L
## [,1] [,2] [,3]
## [1,] 1.00 0.0000000 0
## [2,] 0.57 1.0000000 0
## [3,] 0.71 0.3041031 1
##
## $U
## [,1] [,2] [,3]
## [1,] 1 0.5700 0.7100000
## [2,] 0 0.6751 0.2053000
## [3,] 0 0.0000 0.4334676Calculus-Based Probability & Statistics
Select a variable in the Kaggle.com training dataset that is skewed to the right, shift it so that the minimum value is absolutely above zero if necessary.
When the mean is larger than the median it is stated that data is skewed to the right. Therefore, the X1stFlrSF was selected as the variable. To test this we can run the summary function on the variable as well as the skewness function from the momentspackage.
cat("Summary Statistic: X1stFlrSF Variable","\n","n")## Summary Statistic: X1stFlrSF Variable
## nsummary(house_kaggle$X1stFlrSF)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 334 882 1087 1163 1391 4692cat("\n","Skewness Statistic: X1stFlrSF Variable","\n","n")##
## Skewness Statistic: X1stFlrSF Variable
## nskewness(house_kaggle$X1stFlrSF)## [1] 1.375342Then load the MASS package and run fitdistr to fit an exponential probability density function.
fit <- fitdistr(house_kaggle$X1stFlrSF,"exponential")Find the optimal value of X1stFlrSF for this distribution, and then take 1000 samples from this exponential distribution using this value (e.g., rexp(1000, X1stFlrSF)).
opt_value <- fit$estimate
example <- rexp(1000,opt_value)
opt_value## rate
## 0.0008601213Plot a histogram and compare it with a histogram of your original variable.
hist(example, breaks = 50,
col = "green", xlab = "Optimal Value/Example",
main = "Exponential Distribution Histogram")
hist(house_kaggle$X1stFlrSF, breaks = 50,
col = "yellow", xlab="Original Variable Histogram",
main = "First Floor Square Feet")
Using the exponential pdf, find the 5th and 95th percentiles using the cumulative distribution function (CDF).
cat("The 5th Percentile Using the Cumulative Distribution Function","\n","\n")## The 5th Percentile Using the Cumulative Distribution Function
## qexp(0.05, rate = opt_value, lower.tail = TRUE, log.p = FALSE)## [1] 59.63495cat("The 95th Percentile Using the Cumulative Distribution Function","\n","\n")## The 95th Percentile Using the Cumulative Distribution Function
## qexp(0.95, rate = opt_value, lower.tail = TRUE, log.p = FALSE)## [1] 3482.918Also generate a 95% confidence interval from the empirical data, assuming normality. Finally, provide the empirical 5th percentile and 95th percentile of the data. Discuss.
#calculate the mean
first_flr_mean <- mean(house_kaggle$X1stFlrSF)
#calculate the standard deviation
first_flr_sd <- sd(house_kaggle$X1stFlrSF)
#use the qnorm function
qnorm(0.95,first_flr_mean,first_flr_sd)## [1] 1798.507cat("The 5th Percentile of the X1stFlrSF Variable","\n","\n")## The 5th Percentile of the X1stFlrSF Variable
## quantile(house_kaggle$X1stFlrSF,c(0.05))## 5%
## 672.95cat("The 95th Percentile of the X1stFlrSF Variable","\n","\n")## The 95th Percentile of the X1stFlrSF Variable
## quantile(house_kaggle$X1stFlrSF,c(0.95))## 95%
## 1831.25Modeling
Build some type of multiple regression model and submit your model to the competition board. Provide your complete model summary and results with analysis. Report your Kaggle.com user name and score.
For the modeling portion of the project the following variables have been chosen:
- SalePrice cost of home
- GrLivArea above ground living area square feet
- LotFrontage lot size in square feet
- FullBath size of garage in square feet
- X1stFlrsF first floor square feet
- TotRmsAbvGrd total rooms above ground
house_multi_reg <- lm(SalePrice~GrLivArea + LotFrontage + FullBath + X1stFlrSF,
data = house_kaggle)
summary(house_multi_reg)##
## Call:
## lm(formula = SalePrice ~ GrLivArea + LotFrontage + FullBath +
## X1stFlrSF, data = house_kaggle)
##
## Residuals:
## Min 1Q Median 3Q Max
## -538757 -22151 1488 20926 282615
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) -40730.391 6199.708 -6.570 7.50e-11 ***
## GrLivArea 63.079 4.285 14.721 < 2e-16 ***
## LotFrontage 33.617 73.307 0.459 0.647
## FullBath 29550.770 3597.218 8.215 5.46e-16 ***
## X1stFlrSF 66.917 5.141 13.017 < 2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 53590 on 1196 degrees of freedom
## (259 observations deleted due to missingness)
## Multiple R-squared: 0.5884, Adjusted R-squared: 0.5871
## F-statistic: 427.5 on 4 and 1196 DF, p-value: < 2.2e-16Next, we run the qqnorm and qqline
qqnorm(house_multi_reg$residuals)
qqline(house_multi_reg$residuals)
Next, we plot the data used in the multiple regression.
plot(house_kaggle$GrLivArea,house_kaggle$SalePrice, col="red",
main="GrLivArea", ylab="Sale Price", xlab="GrLivArea")
abline(lm(house_kaggle$SalePrice~house_kaggle$GrLivArea),col="blue")
plot(house_kaggle$LotFrontage,house_kaggle$SalePrice, col="green",
main="GrLivArea", ylab="Sale Price", xlab="LotFrontage")
abline(lm(house_kaggle$SalePrice~house_kaggle$LotFrontage),col="blue")
plot(house_kaggle$FullBath,house_kaggle$SalePrice, col="red",
main="GrLivArea", ylab="Sale Price", xlab="FullBath")
abline(lm(house_kaggle$SalePrice~house_kaggle$FullBath),col="blue")
plot(house_kaggle$X1stFlrSF,house_kaggle$SalePrice, col="green",
main="GrLivArea", ylab="Sale Price", xlab="X1stFlrSF")
abline(lm(house_kaggle$SalePrice~house_kaggle$X1stFlrSF),col="blue")
In order to make sure the data is full we run the complete.cases function on the test set.
house_test[complete.cases(house_test),]## [1] Id MSSubClass MSZoning LotFrontage LotArea
## [6] Street Alley LotShape LandContour Utilities
## [11] LotConfig LandSlope Neighborhood Condition1 Condition2
## [16] BldgType HouseStyle OverallQual OverallCond YearBuilt
## [21] YearRemodAdd RoofStyle RoofMatl Exterior1st Exterior2nd
## [26] MasVnrType MasVnrArea ExterQual ExterCond Foundation
## [31] BsmtQual BsmtCond BsmtExposure BsmtFinType1 BsmtFinSF1
## [36] BsmtFinType2 BsmtFinSF2 BsmtUnfSF TotalBsmtSF Heating
## [41] HeatingQC CentralAir Electrical X1stFlrSF X2ndFlrSF
## [46] LowQualFinSF GrLivArea BsmtFullBath BsmtHalfBath FullBath
## [51] HalfBath BedroomAbvGr KitchenAbvGr KitchenQual TotRmsAbvGrd
## [56] Functional Fireplaces FireplaceQu GarageType GarageYrBlt
## [61] GarageFinish GarageCars GarageArea GarageQual GarageCond
## [66] PavedDrive WoodDeckSF OpenPorchSF EnclosedPorch X3SsnPorch
## [71] ScreenPorch PoolArea PoolQC Fence MiscFeature
## [76] MiscVal MoSold YrSold SaleType SaleCondition
## <0 rows> (or 0-length row.names)Next, we remove data less than zero.
house_test[is.na(house_test)] <-0## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generated
## Warning in `[<-.factor`(`*tmp*`, thisvar, value = 0): invalid factor level,
## NA generatedThe predict statistical component is then run on the linear regression & test data.
pred <- predict(house_multi_reg,house_test)Creating a variable entitled submission we build our data frame step by step
#using cbind to connect the Id field from house_test and pred
submission <- cbind(house_test$Id,pred)
#rename the columns using colnames
colnames(submission)[1] <- "Id"
colnames(submission)[2] <- "SalePrice"
#create the data frame entitle submission
submission <- as.data.frame(submission)Lastly, we prepare the data for submission.
write.csv(submission, file="C:\\Users\\lizza\\Documents\\CUNY - Data Analytics\\Final\\house_Liles.csv", quote=FALSE, row.names=FALSE)Kaggle Score 0.26000 Snapshot of the score can be found with the submission