Assignment 12

#get the working directory 
getwd()

[1] "/cloud/project"

# import the CSV file
wbcd <- read.csv("wisc_bc_data.csv", stringsAsFactors = FALSE)

# examine the structure of the wbcd data frame
str(wbcd)

# make sure the packages for this chapter
# are installed, install if necessary
pkg <- c("ggplot2", "scales", "maptools",
              "sp", "maps", "grid", "car" )
new.pkg <- pkg[!(pkg %in% installed.packages())]
if (length(new.pkg)) {
  install.packages(new.pkg)  
}

Installing packages into ‘/cloud/lib/x86_64-pc-linux-gnu-library/4.3’
(as ‘lib’ is unspecified)

Warning in install.packages :
  package ‘maptools’ is not available for this version of R

A version of this package for your version of R might be available elsewhere,
see the ideas at
https://cran.r-project.org/doc/manuals/r-patched/R-admin.html#Installing-packages

also installing the dependencies ‘rbibutils’, ‘cowplot’, ‘Deriv’, ‘microbenchmark’, ‘Rdpack’, ‘numDeriv’, ‘doBy’, ‘SparseM’, ‘MatrixModels’, ‘minqa’, ‘nloptr’, ‘reformulas’, ‘Rcpp’, ‘RcppEigen’, ‘carData’, ‘abind’, ‘Formula’, ‘pbkrtest’, ‘quantreg’, ‘lme4’

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/rbibutils_2.3.tar.gz'
Content type 'application/x-gzip' length 1137301 bytes (1.1 MB)
==================================================
downloaded 1.1 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/cowplot_1.1.3.tar.gz'
Content type 'application/x-gzip' length 1376043 bytes (1.3 MB)
==================================================
downloaded 1.3 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/Deriv_4.1.6.tar.gz'
Content type 'application/x-gzip' length 149977 bytes (146 KB)
==================================================
downloaded 146 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/microbenchmark_1.5.0.tar.gz'
Content type 'application/x-gzip' length 64450 bytes (62 KB)
==================================================
downloaded 62 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/Rdpack_2.6.2.tar.gz'
Content type 'application/x-gzip' length 747892 bytes (730 KB)
==================================================
downloaded 730 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/numDeriv_2016.8-1.1.tar.gz'
Content type 'application/x-gzip' length 112843 bytes (110 KB)
==================================================
downloaded 110 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/doBy_4.6.25.tar.gz'
Content type 'application/x-gzip' length 4831866 bytes (4.6 MB)
==================================================
downloaded 4.6 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/SparseM_1.84-2.tar.gz'
Content type 'application/x-gzip' length 879962 bytes (859 KB)
==================================================
downloaded 859 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/MatrixModels_0.5-3.tar.gz'
Content type 'application/x-gzip' length 413182 bytes (403 KB)
==================================================
downloaded 403 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/minqa_1.2.8.tar.gz'
Content type 'application/x-gzip' length 121451 bytes (118 KB)
==================================================
downloaded 118 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/nloptr_2.1.1.tar.gz'
Content type 'application/x-gzip' length 554121 bytes (541 KB)
==================================================
downloaded 541 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/reformulas_0.4.0.tar.gz'
Content type 'application/x-gzip' length 89712 bytes (87 KB)
==================================================
downloaded 87 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/Rcpp_1.0.14.tar.gz'
Content type 'application/x-gzip' length 2176986 bytes (2.1 MB)
==================================================
downloaded 2.1 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/RcppEigen_0.3.4.0.2.tar.gz'
Content type 'application/x-gzip' length 1845612 bytes (1.8 MB)
==================================================
downloaded 1.8 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/carData_3.0-5.tar.gz'
Content type 'application/x-gzip' length 1820881 bytes (1.7 MB)
==================================================
downloaded 1.7 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/abind_1.4-8.tar.gz'
Content type 'application/x-gzip' length 64323 bytes (62 KB)
==================================================
downloaded 62 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/Formula_1.2-5.tar.gz'
Content type 'application/x-gzip' length 158399 bytes (154 KB)
==================================================
downloaded 154 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/pbkrtest_0.5.3.tar.gz'
Content type 'application/x-gzip' length 174550 bytes (170 KB)
==================================================
downloaded 170 KB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/quantreg_6.00.tar.gz'
Content type 'application/x-gzip' length 1446791 bytes (1.4 MB)
==================================================
downloaded 1.4 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/lme4_1.1-36.tar.gz'
Content type 'application/x-gzip' length 4235318 bytes (4.0 MB)
==================================================
downloaded 4.0 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/maps_3.4.2.1.tar.gz'
Content type 'application/x-gzip' length 3096278 bytes (3.0 MB)
==================================================
downloaded 3.0 MB

trying URL 'http://rspm/default/__linux__/focal/latest/src/contrib/car_3.1-3.tar.gz'
Content type 'application/x-gzip' length 1512749 bytes (1.4 MB)
==================================================
downloaded 1.4 MB

* installing *binary* package ‘rbibutils’ ...
* DONE (rbibutils)
* installing *binary* package ‘cowplot’ ...
* DONE (cowplot)
* installing *binary* package ‘Deriv’ ...
* DONE (Deriv)
* installing *binary* package ‘microbenchmark’ ...
* DONE (microbenchmark)
* installing *binary* package ‘numDeriv’ ...
* DONE (numDeriv)
* installing *binary* package ‘SparseM’ ...
* DONE (SparseM)
* installing *binary* package ‘MatrixModels’ ...
* DONE (MatrixModels)
* installing *binary* package ‘nloptr’ ...
* DONE (nloptr)
* installing *binary* package ‘Rcpp’ ...
* DONE (Rcpp)
* installing *binary* package ‘carData’ ...
* DONE (carData)
* installing *binary* package ‘abind’ ...
* DONE (abind)
* installing *binary* package ‘Formula’ ...
* DONE (Formula)
* installing *binary* package ‘maps’ ...
* DONE (maps)
* installing *binary* package ‘Rdpack’ ...
* DONE (Rdpack)
* installing *binary* package ‘doBy’ ...
* DONE (doBy)
* installing *binary* package ‘minqa’ ...
* DONE (minqa)
* installing *binary* package ‘RcppEigen’ ...
* DONE (RcppEigen)
* installing *binary* package ‘quantreg’ ...
* DONE (quantreg)
* installing *binary* package ‘reformulas’ ...
* DONE (reformulas)
* installing *binary* package ‘lme4’ ...
* DONE (lme4)
* installing *binary* package ‘pbkrtest’ ...
* DONE (pbkrtest)
* installing *binary* package ‘car’ ...
* DONE (car)

The downloaded source packages are in
    ‘/tmp/RtmpEDqZ4I/downloaded_packages’

# read the CSV with headers

regression1<-read.csv("/cloud/project/incidents (5).csv", header=T,sep =",")

View(regression1)

summary(regression1)

     area               zone            population          incidents     
 Length:16          Length:16          Length:16          Min.   : 103.0  
 Class :character   Class :character   Class :character   1st Qu.: 277.8  
 Mode  :character   Mode  :character   Mode  :character   Median : 654.0  
                                                          Mean   : 695.2  
                                                          3rd Qu.: 853.0  
                                                          Max.   :2072.0  

# summarize three numeric features
summary(wbcd[c("radius_mean", "area_mean", "smoothness_mean")])

regression1$population <- as.numeric(gsub(",","",regression1$population))
regression1$population

 [1]  107353  326534  444752  750000   64403 2744878 1600000 2333000 1572816  712091 6900000 2700000
[13] 4900000 4200000 5200000 7100000

str(regression1$population)

 num [1:16] 107353 326534 444752 750000 64403 ...

<!-- rnb-text-end -->


<!-- rnb-chunk-begin -->


<!-- rnb-source-begin eyJkYXRhIjoiYGBgclxuIyBub3JtYWxpemUgdGhlIHdiY2QgZGF0YVxud2JjZF9uIDwtIGFzLmRhdGEuZnJhbWUobGFwcGx5KHdiY2RbMjozMV0sIG5vcm1hbGl6ZSkpXG5gYGAifQ== -->

```r
# normalize the wbcd data
wbcd_n <- as.data.frame(lapply(wbcd[2:31], normalize))

# confirm that normalization worked
summary(wbcd_n$area_mean)

<!-- rnb-text-end -->


<!-- rnb-chunk-begin -->


<!-- rnb-source-begin eyJkYXRhIjoiYGBgclxucmVnLmZpdDM8LWxtKHJlZ3Jlc3Npb24xJGluY2lkZW50c35pbnRlcmFjdGlvbityZWdyZXNzaW9uMSRwb3B1bGF0aW9uK3JlZ3Jlc3Npb24xJHpvbmUpXG5gYGAifQ== -->

```r
reg.fit3<-lm(regression1$incidents~interaction+regression1$population+regression1$zone)

# create labels for training and test data

wbcd_train_labels <- wbcd[1:469, 1]
wbcd_test_labels <- wbcd[470:569, 1]

<!-- rnb-text-end -->


<!-- rnb-chunk-begin -->


<!-- rnb-source-begin eyJkYXRhIjoiYGBgclxucmVnLmZpdDQ8LWxtKHJlZ3Jlc3Npb24xJGluY2lkZW50c35pbnRlcmFjdGlvbilcbmBgYCJ9 -->

```r
reg.fit4<-lm(regression1$incidents~interaction)

wbcd_test_pred <- knn(train = wbcd_train, test = wbcd_test,
                      cl = wbcd_train_labels, k = 21)

install.packages("gmodels")

## Step 4: Evaluating model performance
# load the "gmodels" library
library(gmodels)

# Create the cross tabulation of predicted vs. actual
CrossTable(x = wbcd_test_labels, y = wbcd_test_pred,
           prop.chisq = FALSE)
           
           
           
           

## Step 5: Improving model performance

# use the scale() function to z-score standardize a data frame
wbcd_z <- as.data.frame(scale(wbcd[-1]))

# confirm that the transformation was applied correctly
summary(wbcd_z$area_mean)

# create training and test datasets
wbcd_train <- wbcd_z[1:469, ]
wbcd_test <- wbcd_z[470:569, ]

# re-classify test cases
wbcd_test_pred <- knn(train = wbcd_train, test = wbcd_test,
                      cl = wbcd_train_labels, k = 21)

# Create the cross tabulation of predicted vs. actual
CrossTable(x = wbcd_test_labels, y = wbcd_test_pred,
           prop.chisq = FALSE)

# try several different values of k
wbcd_train <- wbcd_n[1:469, ]
wbcd_test <- wbcd_n[470:569, ]

#K=1
wbcd_test_pred <- knn(train = wbcd_train, test = wbcd_test, cl = wbcd_train_labels, k=1)
CrossTable(x = wbcd_test_labels, y = wbcd_test_pred, prop.chisq=FALSE)

#k=5
wbcd_test_pred <- knn(train = wbcd_train, test = wbcd_test, cl = wbcd_train_labels, k=5)
CrossTable(x = wbcd_test_labels, y = wbcd_test_pred, prop.chisq=FALSE)

wbcd_test_pred <- knn(train = wbcd_train, test = wbcd_test, cl = wbcd_train_labels, k=11)
CrossTable(x = wbcd_test_labels, y = wbcd_test_pred, prop.chisq=FALSE)

wbcd_test_pred <- knn(train = wbcd_train, test = wbcd_test, cl = wbcd_train_labels, k=15)
CrossTable(x = wbcd_test_labels, y = wbcd_test_pred, prop.chisq=FALSE)

wbcd_test_pred <- knn(train = wbcd_train, test = wbcd_test, cl = wbcd_train_labels, k=21)
CrossTable(x = wbcd_test_labels, y = wbcd_test_pred, prop.chisq=FALSE)

wbcd_test_pred <- knn(train = wbcd_train, test = wbcd_test, cl = wbcd_train_labels, k=27)
CrossTable(x = wbcd_test_labels, y = wbcd_test_pred, prop.chisq=FALSE)