donors<- read.csv("D:\\my data\\donor.csv")
summary(donors)
## age numberChildren incomeRating wealthRating
## Min. : 1.00 Min. :1.00 Min. :1.000 Min. :0.00
## 1st Qu.:48.00 1st Qu.:1.00 1st Qu.:2.000 1st Qu.:3.00
## Median :62.00 Median :1.00 Median :4.000 Median :6.00
## Mean :61.61 Mean :1.53 Mean :3.886 Mean :5.35
## 3rd Qu.:75.00 3rd Qu.:2.00 3rd Qu.:5.000 3rd Qu.:8.00
## Max. :98.00 Max. :7.00 Max. :7.000 Max. :9.00
## NA's :23665 NA's :83026 NA's :21286 NA's :44732
## mailOrderPurchases totalGivingAmount numberGifts smallestGiftAmount
## Min. : 0.000 Min. : 13.0 Min. : 1.000 Min. : 0.000
## 1st Qu.: 0.000 1st Qu.: 40.0 1st Qu.: 3.000 1st Qu.: 3.000
## Median : 0.000 Median : 78.0 Median : 7.000 Median : 5.000
## Mean : 3.321 Mean : 104.5 Mean : 9.602 Mean : 7.934
## 3rd Qu.: 3.000 3rd Qu.: 131.0 3rd Qu.: 13.000 3rd Qu.: 10.000
## Max. :241.000 Max. :9485.0 Max. :237.000 Max. :1000.000
##
## largestGiftAmount averageGiftAmount yearsSinceFirstDonation
## Min. : 5 Min. : 1.286 Min. : 0.000
## 1st Qu.: 14 1st Qu.: 8.385 1st Qu.: 2.000
## Median : 17 Median : 11.636 Median : 5.000
## Mean : 20 Mean : 13.348 Mean : 5.596
## 3rd Qu.: 23 3rd Qu.: 15.478 3rd Qu.: 9.000
## Max. :5000 Max. :1000.000 Max. :13.000
##
## monthsSinceLastDonation inHouseDonor plannedGivingDonor sweepstakesDonor
## Min. : 0.00 Mode :logical Mode :logical Mode :logical
## 1st Qu.:12.00 FALSE:88709 FALSE:95298 FALSE:93795
## Median :14.00 TRUE :6703 TRUE :114 TRUE :1617
## Mean :14.36
## 3rd Qu.:17.00
## Max. :23.00
##
## P3Donor state urbanicity socioEconomicStatus
## Mode :logical Length:95412 Length:95412 Length:95412
## FALSE:93395 Class :character Class :character Class :character
## TRUE :2017 Mode :character Mode :character Mode :character
##
##
##
##
## isHomeowner gender respondedMailing
## Mode:logical Length:95412 Mode :logical
## TRUE:52354 Class :character FALSE:90569
## NA's:43058 Mode :character TRUE :4843
##
##
##
##
View(donors)
dplyr::glimpse(donors)
## Rows: 95,412
## Columns: 22
## $ age <int> 60, 46, NA, 70, 78, NA, 38, NA, NA, 65, NA, 75…
## $ numberChildren <int> NA, 1, NA, NA, 1, NA, 1, NA, NA, NA, NA, NA, 2…
## $ incomeRating <int> NA, 6, 3, 1, 3, NA, 4, 2, 3, NA, 2, 1, 4, NA, …
## $ wealthRating <int> NA, 9, 1, 4, 2, NA, 6, 9, 2, NA, 0, 5, 2, NA, …
## $ mailOrderPurchases <int> 0, 16, 2, 2, 60, 0, 0, 1, 0, 0, 0, 3, 16, 0, 1…
## $ totalGivingAmount <dbl> 240, 47, 202, 109, 254, 51, 107, 31, 199, 28, …
## $ numberGifts <int> 31, 3, 27, 16, 37, 4, 14, 5, 11, 3, 1, 2, 9, 1…
## $ smallestGiftAmount <dbl> 5, 10, 2, 2, 3, 10, 3, 5, 10, 3, 20, 10, 4, 5,…
## $ largestGiftAmount <dbl> 12, 25, 16, 11, 15, 16, 12, 11, 22, 15, 20, 15…
## $ averageGiftAmount <dbl> 7.741935, 15.666667, 7.481481, 6.812500, 6.864…
## $ yearsSinceFirstDonation <int> 8, 3, 7, 10, 11, 3, 10, 3, 9, 3, 1, 1, 8, 5, 4…
## $ monthsSinceLastDonation <int> 14, 14, 14, 14, 13, 20, 22, 18, 19, 22, 12, 14…
## $ inHouseDonor <lgl> FALSE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE…
## $ plannedGivingDonor <lgl> FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALS…
## $ sweepstakesDonor <lgl> FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALS…
## $ P3Donor <lgl> FALSE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE…
## $ state <chr> "IL", "CA", "NC", "CA", "FL", "AL", "IN", "LA"…
## $ urbanicity <chr> "town", "suburb", "rural", "rural", "suburb", …
## $ socioEconomicStatus <chr> "average", "highest", "average", "average", "a…
## $ isHomeowner <lgl> NA, TRUE, NA, NA, TRUE, NA, TRUE, NA, NA, NA, …
## $ gender <chr> "female", "male", "male", "female", "female", …
## $ respondedMailing <lgl> FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALS…
df<-donors
boxplot(df$incomeRating)
my_data<-is.na(df)
#which(is.na(df))
clean_df <- na.omit(df)
attach(clean_df)
##check the outliers
# Convert all columns to numeric
clean_df <- as.data.frame(lapply(clean_df, as.numeric))
## Warning in lapply(clean_df, as.numeric): NAs introduced by coercion
## Warning in lapply(clean_df, as.numeric): NAs introduced by coercion
## Warning in lapply(clean_df, as.numeric): NAs introduced by coercion
## Warning in lapply(clean_df, as.numeric): NAs introduced by coercion
# Now create the boxplot
boxplot(clean_df)
check<-clean_df[,-6]
check<-check[,-8]
boxplot(check)
summary(age)
## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 4.00 42.00 48.00 50.88 59.00 97.00
IQR_age=61-31
Upfen_age=60+1.5*IQR_age
Upfen_age
## [1] 105
summary(numberChildren)
## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 1.00 1.00 1.00 1.55 2.00 7.00
IQR_num=3-2
Upfen_num=3+1.5*IQR_num
Upfen_num
## [1] 4.5
summary(clean_df)
## age numberChildren incomeRating wealthRating
## Min. : 4.00 Min. :1.00 Min. :1.000 Min. :0.00
## 1st Qu.:42.00 1st Qu.:1.00 1st Qu.:4.000 1st Qu.:4.00
## Median :48.00 Median :1.00 Median :5.000 Median :6.00
## Mean :50.88 Mean :1.55 Mean :4.671 Mean :5.83
## 3rd Qu.:59.00 3rd Qu.:2.00 3rd Qu.:6.000 3rd Qu.:8.00
## Max. :97.00 Max. :7.00 Max. :7.000 Max. :9.00
##
## mailOrderPurchases totalGivingAmount numberGifts smallestGiftAmount
## Min. : 0.000 Min. : 15.00 Min. : 1.000 Min. : 0.000
## 1st Qu.: 1.000 1st Qu.: 36.00 1st Qu.: 3.000 1st Qu.: 3.000
## Median : 3.000 Median : 69.00 Median : 6.000 Median : 5.000
## Mean : 5.721 Mean : 88.94 Mean : 8.423 Mean : 7.885
## 3rd Qu.: 8.000 3rd Qu.: 113.00 3rd Qu.:12.000 3rd Qu.: 10.000
## Max. :240.000 Max. :1261.00 Max. :61.000 Max. :150.000
##
## largestGiftAmount averageGiftAmount yearsSinceFirstDonation
## Min. : 5.00 Min. : 1.286 Min. : 1.000
## 1st Qu.: 14.00 1st Qu.: 8.357 1st Qu.: 2.000
## Median : 17.00 Median : 11.571 Median : 5.000
## Mean : 18.91 Mean : 12.863 Mean : 5.246
## 3rd Qu.: 20.00 3rd Qu.: 15.000 3rd Qu.: 8.000
## Max. :1000.00 Max. :150.000 Max. :11.000
##
## monthsSinceLastDonation inHouseDonor plannedGivingDonor
## Min. : 0.00 Min. :0.00000 Min. :0.0000000
## 1st Qu.:12.00 1st Qu.:0.00000 1st Qu.:0.0000000
## Median :14.00 Median :0.00000 Median :0.0000000
## Mean :14.55 Mean :0.03766 Mean :0.0004312
## 3rd Qu.:17.00 3rd Qu.:0.00000 3rd Qu.:0.0000000
## Max. :23.00 Max. :1.00000 Max. :1.0000000
##
## sweepstakesDonor P3Donor state urbanicity
## Min. :0.00000 Min. :0.00000 Min. : NA Min. : NA
## 1st Qu.:0.00000 1st Qu.:0.00000 1st Qu.: NA 1st Qu.: NA
## Median :0.00000 Median :0.00000 Median : NA Median : NA
## Mean :0.01078 Mean :0.01466 Mean :NaN Mean :NaN
## 3rd Qu.:0.00000 3rd Qu.:0.00000 3rd Qu.: NA 3rd Qu.: NA
## Max. :1.00000 Max. :1.00000 Max. : NA Max. : NA
## NA's :6957 NA's :6957
## socioEconomicStatus isHomeowner gender respondedMailing
## Min. : NA Min. :1 Min. : NA Min. :0.00000
## 1st Qu.: NA 1st Qu.:1 1st Qu.: NA 1st Qu.:0.00000
## Median : NA Median :1 Median : NA Median :0.00000
## Mean :NaN Mean :1 Mean :NaN Mean :0.04657
## 3rd Qu.: NA 3rd Qu.:1 3rd Qu.: NA 3rd Qu.:0.00000
## Max. : NA Max. :1 Max. : NA Max. :1.00000
## NA's :6957 NA's :6957
clean_df2=subset(clean_df,age<=103.5 & numberChildren<=7)
##spilt data into test and training set (1)30% test data
# Assuming 'clean_df' is your dataframe
# Set seed for reproducibility
set.seed(123)
# Number of rows in the dataframe
n_rows <- nrow(clean_df)
# Generate indices for the test set
test_indices <- sample(1:n_rows, size = 0.3 * n_rows, replace = FALSE)
# Create training and testing sets
train_data <- clean_df[-test_indices, ]
test_data <- clean_df[test_indices, ]
(2)20% test data
# Assuming 'clean_df' is your dataframe
# Set seed for reproducibility
set.seed(123)
# Number of rows in the dataframe
n_rows <- nrow(clean_df)
# Calculate the number of rows for the test set (20%)
n_test <- round(0.2 * n_rows)
# Generate indices for the test set
test_indices <- sample(1:n_rows, size = n_test, replace = FALSE)
# Create training and testing sets
train_data <- clean_df[-test_indices, ]
test_data <- clean_df[test_indices, ]
Comparison:
30% Test Data: Test Data Size: 30% of the total data Training Data Size: 70% of the total data Larger proportion of data allocated for training 20% Test Data: Test Data Size: 20% of the total data Training Data Size: 80% of the total data Smaller proportion of data allocated for training
Report: 30% and 20% test data allocation depends on several factors such as the size of the dataset, the complexity of the model, and the desired trade-off between training and testing data sizes. If the dataset is large, allocating 30% for testing might be reasonable as it allows for a larger training set while still retaining a substantial amount of data for testing.
However, if the dataset is relatively small or if computational resources are limited, a 20% test data allocation may be preferred to ensure an adequate amount of data for training while still providing a reasonable amount for testing.
5 fold cross validation
library(purrr)
library(dplyr)
##
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
##
## filter, lag
## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union
head(donors)
## age numberChildren incomeRating wealthRating mailOrderPurchases
## 1 60 NA NA NA 0
## 2 46 1 6 9 16
## 3 NA NA 3 1 2
## 4 70 NA 1 4 2
## 5 78 1 3 2 60
## 6 NA NA NA NA 0
## totalGivingAmount numberGifts smallestGiftAmount largestGiftAmount
## 1 240 31 5 12
## 2 47 3 10 25
## 3 202 27 2 16
## 4 109 16 2 11
## 5 254 37 3 15
## 6 51 4 10 16
## averageGiftAmount yearsSinceFirstDonation monthsSinceLastDonation
## 1 7.741935 8 14
## 2 15.666667 3 14
## 3 7.481481 7 14
## 4 6.812500 10 14
## 5 6.864865 11 13
## 6 12.750000 3 20
## inHouseDonor plannedGivingDonor sweepstakesDonor P3Donor state urbanicity
## 1 FALSE FALSE FALSE FALSE IL town
## 2 FALSE FALSE FALSE FALSE CA suburb
## 3 FALSE FALSE FALSE FALSE NC rural
## 4 FALSE FALSE FALSE FALSE CA rural
## 5 TRUE FALSE FALSE TRUE FL suburb
## 6 FALSE FALSE FALSE FALSE AL town
## socioEconomicStatus isHomeowner gender respondedMailing
## 1 average NA female FALSE
## 2 highest TRUE male FALSE
## 3 average NA male FALSE
## 4 average NA female FALSE
## 5 average TRUE female FALSE
## 6 average NA <NA> FALSE
library(modelr)
## Warning: package 'modelr' was built under R version 4.3.3
cv<-crossv_kfold(donors,k=5)
cv
## # A tibble: 5 × 3
## train test .id
## <named list> <named list> <chr>
## 1 <resample [76,329 x 22]> <resample [19,083 x 22]> 1
## 2 <resample [76,329 x 22]> <resample [19,083 x 22]> 2
## 3 <resample [76,330 x 22]> <resample [19,082 x 22]> 3
## 4 <resample [76,330 x 22]> <resample [19,082 x 22]> 4
## 5 <resample [76,330 x 22]> <resample [19,082 x 22]> 5
cv<-crossv_kfold(donors,k=10)
cv
## # A tibble: 10 × 3
## train test .id
## <named list> <named list> <chr>
## 1 <resample [85,870 x 22]> <resample [9,542 x 22]> 01
## 2 <resample [85,870 x 22]> <resample [9,542 x 22]> 02
## 3 <resample [85,871 x 22]> <resample [9,541 x 22]> 03
## 4 <resample [85,871 x 22]> <resample [9,541 x 22]> 04
## 5 <resample [85,871 x 22]> <resample [9,541 x 22]> 05
## 6 <resample [85,871 x 22]> <resample [9,541 x 22]> 06
## 7 <resample [85,871 x 22]> <resample [9,541 x 22]> 07
## 8 <resample [85,871 x 22]> <resample [9,541 x 22]> 08
## 9 <resample [85,871 x 22]> <resample [9,541 x 22]> 09
## 10 <resample [85,871 x 22]> <resample [9,541 x 22]> 10
Comparison:
Number of Folds:
5-fold Cross-validation: Divides the data into 5 equal parts for training and testing. 10-fold Cross-validation: Divides the data into 10 equal parts for training and testing. The 10-fold cross-validation uses more folds, which may lead to a more reliable estimate of model performance. Training and Testing Sets:
5-fold Cross-validation: Each fold contains 80% training data and 20% testing data. 10-fold Cross-validation: Each fold contains 90% training data and 10% testing data. The 10-fold cross-validation uses a larger proportion of the data for training in each fold, potentially providing more information to the model. Robustness:
5-fold Cross-validation: May be less sensitive to the randomness in the data splitting process compared to higher folds. 10-fold Cross-validation: Provides a more robust estimate of model performance by averaging over more iterations, potentially reducing the impact of random variability. Report:
The choice between 5-fold and 10-fold cross-validation depends on factors such as the size of the dataset, computational resources, and the desired trade-off between bias and variance in the estimated model performance.
Use 5-fold cross-validation when:
The dataset is large. Computational resources are limited. A balance between computational efficiency and reliable performance estimation is desired. Use 10-fold cross-validation when:
The dataset is relatively small. A more robust estimate of model performance is required.