donors<- read.csv("D:\\my data\\donor.csv")
summary(donors)
##       age        numberChildren   incomeRating    wealthRating  
##  Min.   : 1.00   Min.   :1.00    Min.   :1.000   Min.   :0.00   
##  1st Qu.:48.00   1st Qu.:1.00    1st Qu.:2.000   1st Qu.:3.00   
##  Median :62.00   Median :1.00    Median :4.000   Median :6.00   
##  Mean   :61.61   Mean   :1.53    Mean   :3.886   Mean   :5.35   
##  3rd Qu.:75.00   3rd Qu.:2.00    3rd Qu.:5.000   3rd Qu.:8.00   
##  Max.   :98.00   Max.   :7.00    Max.   :7.000   Max.   :9.00   
##  NA's   :23665   NA's   :83026   NA's   :21286   NA's   :44732  
##  mailOrderPurchases totalGivingAmount  numberGifts      smallestGiftAmount
##  Min.   :  0.000    Min.   :  13.0    Min.   :  1.000   Min.   :   0.000  
##  1st Qu.:  0.000    1st Qu.:  40.0    1st Qu.:  3.000   1st Qu.:   3.000  
##  Median :  0.000    Median :  78.0    Median :  7.000   Median :   5.000  
##  Mean   :  3.321    Mean   : 104.5    Mean   :  9.602   Mean   :   7.934  
##  3rd Qu.:  3.000    3rd Qu.: 131.0    3rd Qu.: 13.000   3rd Qu.:  10.000  
##  Max.   :241.000    Max.   :9485.0    Max.   :237.000   Max.   :1000.000  
##                                                                           
##  largestGiftAmount averageGiftAmount  yearsSinceFirstDonation
##  Min.   :   5      Min.   :   1.286   Min.   : 0.000         
##  1st Qu.:  14      1st Qu.:   8.385   1st Qu.: 2.000         
##  Median :  17      Median :  11.636   Median : 5.000         
##  Mean   :  20      Mean   :  13.348   Mean   : 5.596         
##  3rd Qu.:  23      3rd Qu.:  15.478   3rd Qu.: 9.000         
##  Max.   :5000      Max.   :1000.000   Max.   :13.000         
##                                                              
##  monthsSinceLastDonation inHouseDonor    plannedGivingDonor sweepstakesDonor
##  Min.   : 0.00           Mode :logical   Mode :logical      Mode :logical   
##  1st Qu.:12.00           FALSE:88709     FALSE:95298        FALSE:93795     
##  Median :14.00           TRUE :6703      TRUE :114          TRUE :1617      
##  Mean   :14.36                                                              
##  3rd Qu.:17.00                                                              
##  Max.   :23.00                                                              
##                                                                             
##   P3Donor           state            urbanicity        socioEconomicStatus
##  Mode :logical   Length:95412       Length:95412       Length:95412       
##  FALSE:93395     Class :character   Class :character   Class :character   
##  TRUE :2017      Mode  :character   Mode  :character   Mode  :character   
##                                                                           
##                                                                           
##                                                                           
##                                                                           
##  isHomeowner       gender          respondedMailing
##  Mode:logical   Length:95412       Mode :logical   
##  TRUE:52354     Class :character   FALSE:90569     
##  NA's:43058     Mode  :character   TRUE :4843      
##                                                    
##                                                    
##                                                    
## 
View(donors)
dplyr::glimpse(donors)
## Rows: 95,412
## Columns: 22
## $ age                     <int> 60, 46, NA, 70, 78, NA, 38, NA, NA, 65, NA, 75…
## $ numberChildren          <int> NA, 1, NA, NA, 1, NA, 1, NA, NA, NA, NA, NA, 2…
## $ incomeRating            <int> NA, 6, 3, 1, 3, NA, 4, 2, 3, NA, 2, 1, 4, NA, …
## $ wealthRating            <int> NA, 9, 1, 4, 2, NA, 6, 9, 2, NA, 0, 5, 2, NA, …
## $ mailOrderPurchases      <int> 0, 16, 2, 2, 60, 0, 0, 1, 0, 0, 0, 3, 16, 0, 1…
## $ totalGivingAmount       <dbl> 240, 47, 202, 109, 254, 51, 107, 31, 199, 28, …
## $ numberGifts             <int> 31, 3, 27, 16, 37, 4, 14, 5, 11, 3, 1, 2, 9, 1…
## $ smallestGiftAmount      <dbl> 5, 10, 2, 2, 3, 10, 3, 5, 10, 3, 20, 10, 4, 5,…
## $ largestGiftAmount       <dbl> 12, 25, 16, 11, 15, 16, 12, 11, 22, 15, 20, 15…
## $ averageGiftAmount       <dbl> 7.741935, 15.666667, 7.481481, 6.812500, 6.864…
## $ yearsSinceFirstDonation <int> 8, 3, 7, 10, 11, 3, 10, 3, 9, 3, 1, 1, 8, 5, 4…
## $ monthsSinceLastDonation <int> 14, 14, 14, 14, 13, 20, 22, 18, 19, 22, 12, 14…
## $ inHouseDonor            <lgl> FALSE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE…
## $ plannedGivingDonor      <lgl> FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALS…
## $ sweepstakesDonor        <lgl> FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALS…
## $ P3Donor                 <lgl> FALSE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE…
## $ state                   <chr> "IL", "CA", "NC", "CA", "FL", "AL", "IN", "LA"…
## $ urbanicity              <chr> "town", "suburb", "rural", "rural", "suburb", …
## $ socioEconomicStatus     <chr> "average", "highest", "average", "average", "a…
## $ isHomeowner             <lgl> NA, TRUE, NA, NA, TRUE, NA, TRUE, NA, NA, NA, …
## $ gender                  <chr> "female", "male", "male", "female", "female", …
## $ respondedMailing        <lgl> FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALS…
df<-donors
boxplot(df$incomeRating)

my_data<-is.na(df)
#which(is.na(df))
clean_df <- na.omit(df)
attach(clean_df)

##check the outliers

# Convert all columns to numeric
clean_df <- as.data.frame(lapply(clean_df, as.numeric))
## Warning in lapply(clean_df, as.numeric): NAs introduced by coercion

## Warning in lapply(clean_df, as.numeric): NAs introduced by coercion

## Warning in lapply(clean_df, as.numeric): NAs introduced by coercion

## Warning in lapply(clean_df, as.numeric): NAs introduced by coercion
# Now create the boxplot
boxplot(clean_df)

check<-clean_df[,-6]
check<-check[,-8]
boxplot(check)

summary(age)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##    4.00   42.00   48.00   50.88   59.00   97.00
IQR_age=61-31
Upfen_age=60+1.5*IQR_age
Upfen_age
## [1] 105
summary(numberChildren)
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##    1.00    1.00    1.00    1.55    2.00    7.00
IQR_num=3-2
Upfen_num=3+1.5*IQR_num
Upfen_num
## [1] 4.5
summary(clean_df)
##       age        numberChildren  incomeRating    wealthRating 
##  Min.   : 4.00   Min.   :1.00   Min.   :1.000   Min.   :0.00  
##  1st Qu.:42.00   1st Qu.:1.00   1st Qu.:4.000   1st Qu.:4.00  
##  Median :48.00   Median :1.00   Median :5.000   Median :6.00  
##  Mean   :50.88   Mean   :1.55   Mean   :4.671   Mean   :5.83  
##  3rd Qu.:59.00   3rd Qu.:2.00   3rd Qu.:6.000   3rd Qu.:8.00  
##  Max.   :97.00   Max.   :7.00   Max.   :7.000   Max.   :9.00  
##                                                               
##  mailOrderPurchases totalGivingAmount  numberGifts     smallestGiftAmount
##  Min.   :  0.000    Min.   :  15.00   Min.   : 1.000   Min.   :  0.000   
##  1st Qu.:  1.000    1st Qu.:  36.00   1st Qu.: 3.000   1st Qu.:  3.000   
##  Median :  3.000    Median :  69.00   Median : 6.000   Median :  5.000   
##  Mean   :  5.721    Mean   :  88.94   Mean   : 8.423   Mean   :  7.885   
##  3rd Qu.:  8.000    3rd Qu.: 113.00   3rd Qu.:12.000   3rd Qu.: 10.000   
##  Max.   :240.000    Max.   :1261.00   Max.   :61.000   Max.   :150.000   
##                                                                          
##  largestGiftAmount averageGiftAmount yearsSinceFirstDonation
##  Min.   :   5.00   Min.   :  1.286   Min.   : 1.000         
##  1st Qu.:  14.00   1st Qu.:  8.357   1st Qu.: 2.000         
##  Median :  17.00   Median : 11.571   Median : 5.000         
##  Mean   :  18.91   Mean   : 12.863   Mean   : 5.246         
##  3rd Qu.:  20.00   3rd Qu.: 15.000   3rd Qu.: 8.000         
##  Max.   :1000.00   Max.   :150.000   Max.   :11.000         
##                                                             
##  monthsSinceLastDonation  inHouseDonor     plannedGivingDonor 
##  Min.   : 0.00           Min.   :0.00000   Min.   :0.0000000  
##  1st Qu.:12.00           1st Qu.:0.00000   1st Qu.:0.0000000  
##  Median :14.00           Median :0.00000   Median :0.0000000  
##  Mean   :14.55           Mean   :0.03766   Mean   :0.0004312  
##  3rd Qu.:17.00           3rd Qu.:0.00000   3rd Qu.:0.0000000  
##  Max.   :23.00           Max.   :1.00000   Max.   :1.0000000  
##                                                               
##  sweepstakesDonor     P3Donor            state        urbanicity  
##  Min.   :0.00000   Min.   :0.00000   Min.   : NA    Min.   : NA   
##  1st Qu.:0.00000   1st Qu.:0.00000   1st Qu.: NA    1st Qu.: NA   
##  Median :0.00000   Median :0.00000   Median : NA    Median : NA   
##  Mean   :0.01078   Mean   :0.01466   Mean   :NaN    Mean   :NaN   
##  3rd Qu.:0.00000   3rd Qu.:0.00000   3rd Qu.: NA    3rd Qu.: NA   
##  Max.   :1.00000   Max.   :1.00000   Max.   : NA    Max.   : NA   
##                                      NA's   :6957   NA's   :6957  
##  socioEconomicStatus  isHomeowner     gender     respondedMailing 
##  Min.   : NA         Min.   :1    Min.   : NA    Min.   :0.00000  
##  1st Qu.: NA         1st Qu.:1    1st Qu.: NA    1st Qu.:0.00000  
##  Median : NA         Median :1    Median : NA    Median :0.00000  
##  Mean   :NaN         Mean   :1    Mean   :NaN    Mean   :0.04657  
##  3rd Qu.: NA         3rd Qu.:1    3rd Qu.: NA    3rd Qu.:0.00000  
##  Max.   : NA         Max.   :1    Max.   : NA    Max.   :1.00000  
##  NA's   :6957                     NA's   :6957
clean_df2=subset(clean_df,age<=103.5 & numberChildren<=7)

##spilt data into test and training set (1)30% test data

# Assuming 'clean_df' is your dataframe

# Set seed for reproducibility
set.seed(123)

# Number of rows in the dataframe
n_rows <- nrow(clean_df)

# Generate indices for the test set
test_indices <- sample(1:n_rows, size = 0.3 * n_rows, replace = FALSE)

# Create training and testing sets
train_data <- clean_df[-test_indices, ]
test_data <- clean_df[test_indices, ]

(2)20% test data

# Assuming 'clean_df' is your dataframe

# Set seed for reproducibility
set.seed(123)

# Number of rows in the dataframe
n_rows <- nrow(clean_df)

# Calculate the number of rows for the test set (20%)
n_test <- round(0.2 * n_rows)

# Generate indices for the test set
test_indices <- sample(1:n_rows, size = n_test, replace = FALSE)

# Create training and testing sets
train_data <- clean_df[-test_indices, ]
test_data <- clean_df[test_indices, ]

Comparison:

30% Test Data: Test Data Size: 30% of the total data Training Data Size: 70% of the total data Larger proportion of data allocated for training 20% Test Data: Test Data Size: 20% of the total data Training Data Size: 80% of the total data Smaller proportion of data allocated for training

Report: 30% and 20% test data allocation depends on several factors such as the size of the dataset, the complexity of the model, and the desired trade-off between training and testing data sizes. If the dataset is large, allocating 30% for testing might be reasonable as it allows for a larger training set while still retaining a substantial amount of data for testing.

However, if the dataset is relatively small or if computational resources are limited, a 20% test data allocation may be preferred to ensure an adequate amount of data for training while still providing a reasonable amount for testing.

5 fold cross validation

library(purrr)
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
head(donors)
##   age numberChildren incomeRating wealthRating mailOrderPurchases
## 1  60             NA           NA           NA                  0
## 2  46              1            6            9                 16
## 3  NA             NA            3            1                  2
## 4  70             NA            1            4                  2
## 5  78              1            3            2                 60
## 6  NA             NA           NA           NA                  0
##   totalGivingAmount numberGifts smallestGiftAmount largestGiftAmount
## 1               240          31                  5                12
## 2                47           3                 10                25
## 3               202          27                  2                16
## 4               109          16                  2                11
## 5               254          37                  3                15
## 6                51           4                 10                16
##   averageGiftAmount yearsSinceFirstDonation monthsSinceLastDonation
## 1          7.741935                       8                      14
## 2         15.666667                       3                      14
## 3          7.481481                       7                      14
## 4          6.812500                      10                      14
## 5          6.864865                      11                      13
## 6         12.750000                       3                      20
##   inHouseDonor plannedGivingDonor sweepstakesDonor P3Donor state urbanicity
## 1        FALSE              FALSE            FALSE   FALSE    IL       town
## 2        FALSE              FALSE            FALSE   FALSE    CA     suburb
## 3        FALSE              FALSE            FALSE   FALSE    NC      rural
## 4        FALSE              FALSE            FALSE   FALSE    CA      rural
## 5         TRUE              FALSE            FALSE    TRUE    FL     suburb
## 6        FALSE              FALSE            FALSE   FALSE    AL       town
##   socioEconomicStatus isHomeowner gender respondedMailing
## 1             average          NA female            FALSE
## 2             highest        TRUE   male            FALSE
## 3             average          NA   male            FALSE
## 4             average          NA female            FALSE
## 5             average        TRUE female            FALSE
## 6             average          NA   <NA>            FALSE
library(modelr)
## Warning: package 'modelr' was built under R version 4.3.3
cv<-crossv_kfold(donors,k=5)
cv
## # A tibble: 5 × 3
##   train                    test                     .id  
##   <named list>             <named list>             <chr>
## 1 <resample [76,329 x 22]> <resample [19,083 x 22]> 1    
## 2 <resample [76,329 x 22]> <resample [19,083 x 22]> 2    
## 3 <resample [76,330 x 22]> <resample [19,082 x 22]> 3    
## 4 <resample [76,330 x 22]> <resample [19,082 x 22]> 4    
## 5 <resample [76,330 x 22]> <resample [19,082 x 22]> 5
cv<-crossv_kfold(donors,k=10)
cv
## # A tibble: 10 × 3
##    train                    test                    .id  
##    <named list>             <named list>            <chr>
##  1 <resample [85,870 x 22]> <resample [9,542 x 22]> 01   
##  2 <resample [85,870 x 22]> <resample [9,542 x 22]> 02   
##  3 <resample [85,871 x 22]> <resample [9,541 x 22]> 03   
##  4 <resample [85,871 x 22]> <resample [9,541 x 22]> 04   
##  5 <resample [85,871 x 22]> <resample [9,541 x 22]> 05   
##  6 <resample [85,871 x 22]> <resample [9,541 x 22]> 06   
##  7 <resample [85,871 x 22]> <resample [9,541 x 22]> 07   
##  8 <resample [85,871 x 22]> <resample [9,541 x 22]> 08   
##  9 <resample [85,871 x 22]> <resample [9,541 x 22]> 09   
## 10 <resample [85,871 x 22]> <resample [9,541 x 22]> 10

Comparison:

Number of Folds:

5-fold Cross-validation: Divides the data into 5 equal parts for training and testing. 10-fold Cross-validation: Divides the data into 10 equal parts for training and testing. The 10-fold cross-validation uses more folds, which may lead to a more reliable estimate of model performance. Training and Testing Sets:

5-fold Cross-validation: Each fold contains 80% training data and 20% testing data. 10-fold Cross-validation: Each fold contains 90% training data and 10% testing data. The 10-fold cross-validation uses a larger proportion of the data for training in each fold, potentially providing more information to the model. Robustness:

5-fold Cross-validation: May be less sensitive to the randomness in the data splitting process compared to higher folds. 10-fold Cross-validation: Provides a more robust estimate of model performance by averaging over more iterations, potentially reducing the impact of random variability. Report:

The choice between 5-fold and 10-fold cross-validation depends on factors such as the size of the dataset, computational resources, and the desired trade-off between bias and variance in the estimated model performance.

Use 5-fold cross-validation when:

The dataset is large. Computational resources are limited. A balance between computational efficiency and reliable performance estimation is desired. Use 10-fold cross-validation when:

The dataset is relatively small. A more robust estimate of model performance is required.