final

一、資料處理

library(dplyr)

## 
## Attaching package: 'dplyr'

## The following objects are masked from 'package:stats':
## 
##     filter, lag

## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union

library("pROC")

## Type 'citation("pROC")' for a citation.

## 
## Attaching package: 'pROC'

## The following objects are masked from 'package:stats':
## 
##     cov, smooth, var

load("train.rdata")
data<- train
data[,c(2:21,31:34)] <- lapply(c(2:21,31:34), function(a)factor(data[[a]]))

# summary(data)
data <- data[-1]
set.seed(0)
train_idx <- sample(1:nrow(data), size = 0.7 * nrow(data))
train_data <- data[train_idx, ]
test_data <- data[-train_idx, ]

二、挑選變數

1.用p-value排列重要性(小到大)

model <- glm(h1n1_vaccine ~ ., data = train_data, family = "binomial")
summary(model)

## 
## Call:
## glm(formula = h1n1_vaccine ~ ., family = "binomial", data = train_data)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -2.4686  -0.5138  -0.2765  -0.1445   3.1589  
## 
## Coefficients: (1 not defined because of singularities)
##                                       Estimate Std. Error z value Pr(>|z|)    
## (Intercept)                         -4.6611597  0.2570945 -18.130  < 2e-16 ***
## h1n1_concern1                        0.0223710  0.0903792   0.248 0.804503    
## h1n1_concern2                       -0.0392892  0.0910149  -0.432 0.665975    
## h1n1_concern3                       -0.1987530  0.1031681  -1.926 0.054042 .  
## h1n1_knowledge1                     -0.0484226  0.0921482  -0.525 0.599245    
## h1n1_knowledge2                      0.0412901  0.0972068   0.425 0.671007    
## behavioral_antiviral_meds1           0.1631733  0.1026373   1.590 0.111879    
## behavioral_avoidance1               -0.0631843  0.0582705  -1.084 0.278220    
## behavioral_face_mask1                0.2383210  0.0863962   2.758 0.005807 ** 
## behavioral_wash_hands1              -0.0077819  0.0729530  -0.107 0.915050    
## behavioral_large_gatherings1        -0.2507472  0.0595166  -4.213 2.52e-05 ***
## behavioral_outside_home1            -0.0304863  0.0603161  -0.505 0.613248    
## behavioral_touch_face1               0.0008523  0.0561948   0.015 0.987899    
## doctor_recc_h1n11                    2.2714437  0.0675161  33.643  < 2e-16 ***
## doctor_recc_h1n12                   -0.7941992  0.1104525  -7.190 6.46e-13 ***
## doctor_recc_seasonal1               -1.0291488  0.0666595 -15.439  < 2e-16 ***
## doctor_recc_seasonal2                       NA         NA      NA       NA    
## chronic_med_condition1               0.0512349  0.0520272   0.985 0.324737    
## child_under_6_months1                0.2535883  0.0799855   3.170 0.001522 ** 
## health_worker1                       0.6755467  0.0676711   9.983  < 2e-16 ***
## opinion_h1n1_vacc_effective3         0.5655488  0.1561244   3.622 0.000292 ***
## opinion_h1n1_vacc_effective4         0.9412257  0.1458487   6.453 1.09e-10 ***
## opinion_h1n1_vacc_effective5         1.8451998  0.1486259  12.415  < 2e-16 ***
## opinion_h1n1_risk2                   0.5110326  0.0690307   7.403 1.33e-13 ***
## opinion_h1n1_risk4                   1.2696159  0.0794092  15.988  < 2e-16 ***
## opinion_h1n1_risk5                   1.6855449  0.1076469  15.658  < 2e-16 ***
## opinion_h1n1_sick_from_vacc2        -0.3428712  0.0617139  -5.556 2.76e-08 ***
## opinion_h1n1_sick_from_vacc4        -0.0330691  0.0701861  -0.471 0.637524    
## opinion_h1n1_sick_from_vacc5        -0.0307478  0.1040346  -0.296 0.767571    
## opinion_seas_vacc_effective4        -0.1118716  0.1088141  -1.028 0.303903    
## opinion_seas_vacc_effective5        -0.3583658  0.1138199  -3.149 0.001641 ** 
## opinion_seas_risk2                   0.1809568  0.0804374   2.250 0.024470 *  
## opinion_seas_risk4                   0.0094817  0.0861716   0.110 0.912383    
## opinion_seas_risk5                  -0.0459962  0.1028933  -0.447 0.654855    
## opinion_seas_sick_from_vacc2         0.0037181  0.0589156   0.063 0.949680    
## opinion_seas_sick_from_vacc4        -0.0870321  0.0692468  -1.257 0.208812    
## opinion_seas_sick_from_vacc5        -0.2357731  0.1128045  -2.090 0.036608 *  
## age_group35 - 44 Years              -0.2236329  0.0859782  -2.601 0.009294 ** 
## age_group45 - 54 Years              -0.1741991  0.0808925  -2.153 0.031282 *  
## age_group55 - 64 Years               0.1223979  0.0826342   1.481 0.138553    
## age_group65+ Years                  -0.0082435  0.0900258  -0.092 0.927041    
## education12 Years                    0.0733119  0.0988344   0.742 0.458229    
## educationCollege Graduate            0.1292342  0.0991099   1.304 0.192251    
## educationSome College                0.0315370  0.0990303   0.318 0.750138    
## raceHispanic                         0.3488563  0.1276015   2.734 0.006258 ** 
## raceOther or Multiple                0.3552191  0.1298520   2.736 0.006227 ** 
## raceWhite                            0.2142582  0.0982327   2.181 0.029174 *  
## sexMale                              0.1819071  0.0485781   3.745 0.000181 ***
## income_poverty> $75,000             -0.0551555  0.0574496  -0.960 0.337021    
## income_povertyBelow Poverty         -0.0107401  0.0888894  -0.121 0.903829    
## marital_statusNot Married           -0.1110293  0.0641974  -1.729 0.083720 .  
## rent_or_ownRent                      0.0089923  0.0641737   0.140 0.888562    
## employment_statusNot in Labor Force  0.0056487  0.0588539   0.096 0.923538    
## employment_statusUnemployed          0.1006140  0.1110766   0.906 0.365037    
## census_msaMSA, Principle City        0.0759100  0.0547335   1.387 0.165471    
## census_msaNon-MSA                    0.0889092  0.0564080   1.576 0.114984    
## household_adults1                    0.0248465  0.0686551   0.362 0.717424    
## household_adults2                    0.0726677  0.0914408   0.795 0.426790    
## household_children1                  0.0391762  0.0786495   0.498 0.618406    
## household_children2                 -0.0723237  0.0850295  -0.851 0.395007    
## household_children3                 -0.0710714  0.1080214  -0.658 0.510578    
## seasonal_vaccine1                    2.0660939  0.0589995  35.019  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 19348  on 18693  degrees of freedom
## Residual deviance: 12710  on 18633  degrees of freedom
## AIC: 12832
## 
## Number of Fisher Scoring iterations: 6

# 結果：
#1 doctor_recc_h1n1
#2 doctor_recc_seasonal
#3 health_worker
#4 opinion_h1n1_vacc_effective
#5 opinion_h1n1_risk
#6 seasonal_vaccine
#7 opinion_h1n1_sick_from_vacc
#8 behavioral_large_gatherings
#9 sex
#10 child_under_6_months
#11 opinion_seas_vacc_effective
#12 behavioral_face_mask
#13 race
#14 age_group
#15 opinion_seas_risk
#16 opinion_seas_sick_from_vacc
#17 h1n1_concern
#18 marital_status
#19 behavioral_antiviral_meds
#20 census_msa
#21 education
#22 behavioral_avoidance
#23 chronic_med_condition
#24 income_poverty
#25 employment_status
#26 household_children
#27 household_adults
#28 h1n1_knowledge
#29 behavioral_outside_home
#30 rent_or_own
#31 behavioral_wash_hands
#32 behavioral_touch_face

2.用隨機森林看變數重要性

library(randomForest)

## randomForest 4.6-14

## Type rfNews() to see new features/changes/bug fixes.

## 
## Attaching package: 'randomForest'

## The following object is masked from 'package:dplyr':
## 
##     combine

rf <- randomForest(h1n1_vaccine ~ ., train_data)
rf_pred_train <- predict(rf, train_data)
rf_pred_test <- predict(rf, test_data)
importance(rf)

##                             MeanDecreaseGini
## h1n1_concern                       233.62608
## h1n1_knowledge                     156.09912
## behavioral_antiviral_meds           40.73726
## behavioral_avoidance                90.29804
## behavioral_face_mask                52.06175
## behavioral_wash_hands               61.89743
## behavioral_large_gatherings         96.69375
## behavioral_outside_home             95.07615
## behavioral_touch_face               94.32057
## doctor_recc_h1n1                   678.15275
## doctor_recc_seasonal               193.86674
## chronic_med_condition               98.14627
## child_under_6_months                57.74945
## health_worker                      122.48378
## opinion_h1n1_vacc_effective        363.51193
## opinion_h1n1_risk                  429.10475
## opinion_h1n1_sick_from_vacc        237.11170
## opinion_seas_vacc_effective        147.72297
## opinion_seas_risk                  258.55647
## opinion_seas_sick_from_vacc        226.34071
## age_group                          309.30970
## education                          230.05704
## race                               145.69007
## sex                                110.02518
## income_poverty                     141.08452
## marital_status                      91.72238
## rent_or_own                         77.56683
## employment_status                  131.21883
## census_msa                         206.34013
## household_adults                   150.96274
## household_children                 172.10172
## seasonal_vaccine                   549.50472

# 結果：
#1 doctor_recc_h1n1
#2 seasonal_vaccine
#3 opinion_h1n1_risk
#4 opinion_h1n1_vacc_effective
#5 age_group
#6 opinion_seas_risk
#7 opinion_h1n1_sick_from_vacc
#8 h1n1_concern
#9 education
#10 opinion_seas_sick_from_vacc
#11 census_msa
#12 doctor_recc_seasonal
#13 household_children
#14 h1n1_knowledge
#15 household_adults
#16 opinion_seas_vacc_effective
#17 race
#18 income_poverty
#19 employment_status
#20 health_worker
#21 sex
#22 chronic_med_condition
#23 behavioral_large_gatherings
#24 behavioral_outside_home
#25 behavioral_touch_face
#26 marital_status
#27 behavioral_avoidance
#28 rent_or_own
#29 behavioral_wash_hands
#30 child_under_6_months
#31 behavioral_face_mask
#32 behavioral_antiviral_meds

3.用elastic net 挑變數貢獻較小的變量的係數趨近於零

# Dumy code categorical predictor variables
x <- model.matrix(h1n1_vaccine ~ ., train_data)[,-1]
# Convert the outcome (class) to a numerical variable
y <- as.numeric(train_data$h1n1_vaccine)

library(glmnet)

## Loading required package: Matrix

## Loaded glmnet 3.0-2

# glmnet(x, y, family = "binomial", alpha = 1, lambda = NULL)
# 
# # Find the best lambda using cross-validation
# set.seed(123) 
# cv.lasso <- cv.glmnet(x, y, alpha = 1, family = "binomial")
# # Fit the final model on the training data
# model <- glmnet(x, y, alpha = 1, family = "binomial",
#                 lambda = cv.lasso$lambda.min)
# # Display regression coefficients
# coef(model)
# # Make predictions on the test data
# x.test <- model.matrix(h1n1_vaccine ~., test_data)[,-1]
# probabilities <- model %>% predict(newx = x.test)
# predicted.classes <- ifelse(probabilities > 0.5, "0", "1")
# # Model accuracy
# observed.classes <- test_data$h1n1_vaccine
# mean(predicted.classes == observed.classes)


library(glmnet)
set.seed(123)
cv.lasso <- cv.glmnet(x, y, alpha = 1, family = "binomial")
# plot(cv.lasso)

coef(cv.lasso, cv.lasso$lambda.min)

## 62 x 1 sparse Matrix of class "dgCMatrix"
##                                                 1
## (Intercept)                         -4.2891182663
## h1n1_concern1                        0.0396536632
## h1n1_concern2                        .           
## h1n1_concern3                       -0.1546387596
## h1n1_knowledge1                     -0.0277052952
## h1n1_knowledge2                      0.0542458946
## behavioral_antiviral_meds1           0.1276831287
## behavioral_avoidance1               -0.0549870663
## behavioral_face_mask1                0.2030913045
## behavioral_wash_hands1              -0.0003458214
## behavioral_large_gatherings1        -0.2436000010
## behavioral_outside_home1            -0.0263000863
## behavioral_touch_face1               .           
## doctor_recc_h1n11                    2.2256860566
## doctor_recc_h1n12                   -0.5475096081
## doctor_recc_seasonal1               -0.9828377949
## doctor_recc_seasonal2               -0.1971709402
## chronic_med_condition1               0.0321340180
## child_under_6_months1                0.2243628482
## health_worker1                       0.6583935172
## opinion_h1n1_vacc_effective3         0.2201165380
## opinion_h1n1_vacc_effective4         0.6117284145
## opinion_h1n1_vacc_effective5         1.5028898463
## opinion_h1n1_risk2                   0.4557652122
## opinion_h1n1_risk4                   1.2006770678
## opinion_h1n1_risk5                   1.5910251547
## opinion_h1n1_sick_from_vacc2        -0.3029826801
## opinion_h1n1_sick_from_vacc4         .           
## opinion_h1n1_sick_from_vacc5         .           
## opinion_seas_vacc_effective4         .           
## opinion_seas_vacc_effective5        -0.2301889533
## opinion_seas_risk2                   0.1608195887
## opinion_seas_risk4                   .           
## opinion_seas_risk5                  -0.0306915621
## opinion_seas_sick_from_vacc2         .           
## opinion_seas_sick_from_vacc4        -0.0749268135
## opinion_seas_sick_from_vacc5        -0.2172780015
## age_group35 - 44 Years              -0.2021337824
## age_group45 - 54 Years              -0.1510577433
## age_group55 - 64 Years               0.1178112899
## age_group65+ Years                   .           
## education12 Years                    0.0158972828
## educationCollege Graduate            0.0772361159
## educationSome College                .           
## raceHispanic                         0.2321002903
## raceOther or Multiple                0.2451764530
## raceWhite                            0.1280152000
## sexMale                              0.1721864558
## income_poverty> $75,000             -0.0259869989
## income_povertyBelow Poverty          .           
## marital_statusNot Married           -0.1057994628
## rent_or_ownRent                      .           
## employment_statusNot in Labor Force  .           
## employment_statusUnemployed          0.0564782461
## census_msaMSA, Principle City        0.0491123669
## census_msaNon-MSA                    0.0643672885
## household_adults1                    .           
## household_adults2                    0.0314902839
## household_children1                  0.0244918652
## household_children2                 -0.0491974842
## household_children3                 -0.0359665084
## seasonal_vaccine1                    2.0281801566

# 結果：
#1 doctor_recc_h1n1
#2 seasonal_vaccine
#3 opinion_h1n1_risk
#4 opinion_h1n1_vacc_effective
#5 doctor_recc_seasonal
#6 health_worker
#7 opinion_h1n1_sick_from_vacc
#8 race
#9 behavioral_large_gatherings
#10 opinion_seas_vacc_effective
#11 child_under_6_months
#12 opinion_seas_sick_from_vacc
#13 behavioral_face_mask
#14 age_group
#15 sex
#16 opinion_seas_risk
#17 h1n1_concern
#18 behavioral_antiviral_meds
#19 marital_status
#20 education
#21 census_msa
#22 employment_status
#23 behavioral_avoidance
#24 h1n1_knowledge
#25 household_children
#26 chronic_med_condition
#27 household_adults
#28 behavioral_outside_home
#29 income_poverty
#30 behavioral_wash_hands

三、羅吉斯回歸模型

glm_func = function(f){ 
    model <- glm(f, data = train_data, family = "binomial")
    train_pred_prob = predict(model, newdata = train_data,type = "response") 
    test_pred_prob = predict(model, newdata = test_data,type = "response") 
    model_roc_train = roc(train_data$h1n1_vaccine, train_pred_prob)
    model_roc_test = roc(test_data$h1n1_vaccine, test_pred_prob)
    
    return(paste("Train：",model_roc_train$auc,"Test：",model_roc_test$auc));
}

1. p-value + glm

p_glm_AUC <- Map(function(x) glm_func(x),
c(h1n1_vaccine ~ doctor_recc_h1n1,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children,
    h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own+behavioral_wash_hands,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own+behavioral_wash_hands+behavioral_touch_face
  )) 

p_glm_AUC

## [[1]]
## [1] "Train： 0.704736068606229 Test： 0.696843662191186"
## 
## [[2]]
## [1] "Train： 0.717503622085824 Test： 0.704409080777814"
## 
## [[3]]
## [1] "Train： 0.74316954177846 Test： 0.727352891529973"
## 
## [[4]]
## [1] "Train： 0.79922267844053 Test： 0.790085723318477"
## 
## [[5]]
## [1] "Train： 0.824063196272348 Test： 0.812563007851443"
## 
## [[6]]
## [1] "Train： 0.872103390830101 Test： 0.864892281637285"
## 
## [[7]]
## [1] "Train： 0.872977042168762 Test： 0.86543518666178"
## 
## [[8]]
## [1] "Train： 0.873702407747203 Test： 0.865318147100543"
## 
## [[9]]
## [1] "Train： 0.874373773555862 Test： 0.865840022307461"
## 
## [[10]]
## [1] "Train： 0.874524913101383 Test： 0.865936544973501"
## 
## [[11]]
## [1] "Train： 0.875318957500951 Test： 0.86654860126067"
## 
## [[12]]
## [1] "Train： 0.875547422666178 Test： 0.866762676410327"
## 
## [[13]]
## [1] "Train： 0.875708371030641 Test： 0.866765893832528"
## 
## [[14]]
## [1] "Train： 0.876254435158307 Test： 0.866177618492064"
## 
## [[15]]
## [1] "Train： 0.876561951885248 Test： 0.866248215263264"
## 
## [[16]]
## [1] "Train： 0.876785384476994 Test： 0.866150993157615"
## 
## [[17]]
## [1] "Train： 0.876650103091712 Test： 0.86590329827742"
## 
## [[18]]
## [1] "Train： 0.87675803427714 Test： 0.86612464759901"
## 
## [[19]]
## [1] "Train： 0.876848492435794 Test： 0.866303097957916"
## 
## [[20]]
## [1] "Train： 0.876890701116606 Test： 0.866596722705768"
## 
## [[21]]
## [1] "Train： 0.876952672687846 Test： 0.866888109246871"
## 
## [[22]]
## [1] "Train： 0.877012901228543 Test： 0.866740480860068"
## 
## [[23]]
## [1] "Train： 0.877014225589985 Test： 0.866790560736072"
## 
## [[24]]
## [1] "Train： 0.877027178699319 Test： 0.866760391574271"
## 
## [[25]]
## [1] "Train： 0.877022624604811 Test： 0.866717958904659"
## 
## [[26]]
## [1] "Train： 0.877064132655698 Test： 0.866802824243882"
## 
## [[27]]
## [1] "Train： 0.877111228657445 Test： 0.866785571400194"
## 
## [[28]]
## [1] "Train： 0.877237649637463 Test： 0.866716093732368"
## 
## [[29]]
## [1] "Train： 0.877235035091648 Test： 0.866720383628637"
## 
## [[30]]
## [1] "Train： 0.877236530338438 Test： 0.866717958904659"
## 
## [[31]]
## [1] "Train： 0.877235812619979 Test： 0.866716186990983"
## 
## [[32]]
## [1] "Train： 0.87723668413525 Test： 0.866714508335921"

aaaa<-as.data.frame(p_glm_AUC)
colnames(aaaa) <- c(1:32)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:32),
  AUC = aaaa
)
library(tidyr)

## 
## Attaching package: 'tidyr'

## The following objects are masked from 'package:Matrix':
## 
##     expand, pack, unpack

ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
p_glm_AUC_df <- ccc[,-c(2,4)]

plot(p_glm_AUC_df$numOfVar, p_glm_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(p_glm_AUC_df$numOfVar, p_glm_AUC_df$test, col = "blue")

p-value + glm最好的模型

# 以上圖來看最好的模型是7個變數的模型
p_glm_AUC[[7]]

## [1] "Train： 0.872977042168762 Test： 0.86543518666178"

#1 doctor_recc_h1n1
#2 doctor_recc_seasonal
#3 health_worker
#4 opinion_h1n1_vacc_effective
#5 opinion_h1n1_risk
#6 seasonal_vaccine
#7 opinion_h1n1_sick_from_vacc

2. 隨機森林 + glm

rf_glm_AUC<-Map(function(x) glm_func(x),
c(h1n1_vaccine ~  doctor_recc_h1n1,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective,  
 h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home,       
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months+behavioral_face_mask,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months+behavioral_face_mask+behavioral_antiviral_meds   
))

rf_glm_AUC

## [[1]]
## [1] "Train： 0.704736068606229 Test： 0.696843662191186"
## 
## [[2]]
## [1] "Train： 0.809366936774882 Test： 0.807125144667426"
## 
## [[3]]
## [1] "Train： 0.846480047221773 Test： 0.841189112616305"
## 
## [[4]]
## [1] "Train： 0.860438099106939 Test： 0.856671394878423"
## 
## [[5]]
## [1] "Train： 0.861855746860563 Test： 0.856799438956175"
## 
## [[6]]
## [1] "Train： 0.862896515524729 Test： 0.856925151568563"
## 
## [[7]]
## [1] "Train： 0.863887078073824 Test： 0.85800639194544"
## 
## [[8]]
## [1] "Train： 0.863836120063225 Test： 0.857801689286544"
## 
## [[9]]
## [1] "Train： 0.864171174963902 Test： 0.858692728719084"
## 
## [[10]]
## [1] "Train： 0.864388105368179 Test： 0.85853964470333"
## 
## [[11]]
## [1] "Train： 0.864425153311498 Test： 0.858818021667706"
## 
## [[12]]
## [1] "Train： 0.872372099494644 Test： 0.864668600850332"
## 
## [[13]]
## [1] "Train： 0.872437873264864 Test： 0.864752113939645"
## 
## [[14]]
## [1] "Train： 0.872688117767619 Test： 0.864620852439692"
## 
## [[15]]
## [1] "Train： 0.872855243637392 Test： 0.864714250942145"
## 
## [[16]]
## [1] "Train： 0.8736603015976 Test： 0.86496996606319"
## 
## [[17]]
## [1] "Train： 0.873877496874165 Test： 0.864911446282572"
## 
## [[18]]
## [1] "Train： 0.87385749474425 Test： 0.864898576593766"
## 
## [[19]]
## [1] "Train： 0.874020391201674 Test： 0.864929025531411"
## 
## [[20]]
## [1] "Train： 0.875684336006527 Test： 0.865725640616738"
## 
## [[21]]
## [1] "Train： 0.876160157712138 Test： 0.866144511683905"
## 
## [[22]]
## [1] "Train： 0.876165198829887 Test： 0.866162837001661"
## 
## [[23]]
## [1] "Train： 0.876690201338483 Test： 0.86620634214534"
## 
## [[24]]
## [1] "Train： 0.876691482978589 Test： 0.866192819646233"
## 
## [[25]]
## [1] "Train： 0.876692140887177 Test： 0.866200187076781"
## 
## [[26]]
## [1] "Train： 0.876722789174236 Test： 0.8663267390167"
## 
## [[27]]
## [1] "Train： 0.876776609514404 Test： 0.866172302751036"
## 
## [[28]]
## [1] "Train： 0.876778403810552 Test： 0.866168758923684"
## 
## [[29]]
## [1] "Train： 0.876777805711836 Test： 0.866172209492421"
## 
## [[30]]
## [1] "Train： 0.876914889937533 Test： 0.866377471703005"
## 
## [[31]]
## [1] "Train： 0.877146918062253 Test： 0.866544777657474"
## 
## [[32]]
## [1] "Train： 0.87723668413525 Test： 0.866714508335921"

aaaa<-as.data.frame(rf_glm_AUC)
colnames(aaaa) <- c(1:32)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:32),
  AUC = aaaa
)
library(tidyr)
ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
rf_glm_AUC_df <- ccc[,-c(2,4)]

plot(rf_glm_AUC_df$numOfVar, rf_glm_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(rf_glm_AUC_df$numOfVar, rf_glm_AUC_df$test, col = "blue")

隨機森林 + glm最好的模型

# 以上圖來看最好的模型是5個變數的模型
rf_glm_AUC[[5]]

## [1] "Train： 0.861855746860563 Test： 0.856799438956175"

#1 doctor_recc_h1n1
#2 seasonal_vaccine
#3 opinion_h1n1_risk
#4 opinion_h1n1_vacc_effective
#5 age_group

3. elastic net + glm

en_glm_AUC<-Map(function(x) glm_func(x),
c(h1n1_vaccine ~ doctor_recc_h1n1,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition,  
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults, 
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home+income_poverty,  
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home+income_poverty+behavioral_wash_hands
))
en_glm_AUC

## [[1]]
## [1] "Train： 0.704736068606229 Test： 0.696843662191186"
## 
## [[2]]
## [1] "Train： 0.809366936774882 Test： 0.807125144667426"
## 
## [[3]]
## [1] "Train： 0.846480047221773 Test： 0.841189112616305"
## 
## [[4]]
## [1] "Train： 0.860438099106939 Test： 0.856671394878423"
## 
## [[5]]
## [1] "Train： 0.869888819258806 Test： 0.86398053879232"
## 
## [[6]]
## [1] "Train： 0.872103390830101 Test： 0.864892281637285"
## 
## [[7]]
## [1] "Train： 0.872977657356013 Test： 0.86543518666178"
## 
## [[8]]
## [1] "Train： 0.8731348547871 Test： 0.865419192809388"
## 
## [[9]]
## [1] "Train： 0.873967638894927 Test： 0.865292127947089"
## 
## [[10]]
## [1] "Train： 0.874788136346274 Test： 0.865955849506709"
## 
## [[11]]
## [1] "Train： 0.874928971505347 Test： 0.866140921227246"
## 
## [[12]]
## [1] "Train： 0.875133299115251 Test： 0.8659769725829"
## 
## [[13]]
## [1] "Train： 0.875359149734663 Test： 0.866242339970549"
## 
## [[14]]
## [1] "Train： 0.875939057705384 Test： 0.865577359419633"
## 
## [[15]]
## [1] "Train： 0.876501236321311 Test： 0.866042160354457"
## 
## [[16]]
## [1] "Train： 0.876785384476994 Test： 0.866150993157615"
## 
## [[17]]
## [1] "Train： 0.876650103091712 Test： 0.86590329827742"
## 
## [[18]]
## [1] "Train： 0.876733144826289 Test： 0.866083240774158"
## 
## [[19]]
## [1] "Train： 0.876848492435794 Test： 0.866303097957916"
## 
## [[20]]
## [1] "Train： 0.876907883638289 Test： 0.86656958444894"
## 
## [[21]]
## [1] "Train： 0.876952672687846 Test： 0.866888109246871"
## 
## [[22]]
## [1] "Train： 0.876949383144908 Test： 0.866841246793069"
## 
## [[23]]
## [1] "Train： 0.877006236699993 Test： 0.866687276820478"
## 
## [[24]]
## [1] "Train： 0.87712612985974 Test： 0.866636684022095"
## 
## [[25]]
## [1] "Train： 0.877164126216738 Test： 0.866704716181395"
## 
## [[26]]
## [1] "Train： 0.877166450257463 Test： 0.866767292711746"
## 
## [[27]]
## [1] "Train： 0.877209803870102 Test： 0.866743791540884"
## 
## [[28]]
## [1] "Train： 0.877217263015517 Test： 0.866757127522762"
## 
## [[29]]
## [1] "Train： 0.877235035091648 Test： 0.866720383628637"
## 
## [[30]]
## [1] "Train： 0.877236367997358 Test： 0.866717399352972"

aaaa<-as.data.frame(en_glm_AUC)
colnames(aaaa) <- c(1:30)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:30),
  AUC = aaaa
)
library(tidyr)
ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
en_glm_AUC_df <- ccc[,-c(2,4)]

plot(en_glm_AUC_df$numOfVar, en_glm_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(en_glm_AUC_df$numOfVar, en_glm_AUC_df$test, col = "blue")

elastic net + glm最好的模型

# 以上圖來看最好的模型是5個變數的模型
en_glm_AUC[[5]]

## [1] "Train： 0.869888819258806 Test： 0.86398053879232"

#1 doctor_recc_h1n1
#2 seasonal_vaccine
#3 opinion_h1n1_risk
#4 opinion_h1n1_vacc_effective
#5 doctor_recc_seasonal

四、CART決策樹演算法-rpart()

library(rpart); library(rpart.plot);library(partykit)

## Loading required package: grid

## Loading required package: libcoin

## Loading required package: mvtnorm

cart_func = function(f){ 
    model <- rpart(f,train_data,na.action = na.omit,control = rpart.control(maxdepth = 2))
    
    train_pred_prob = predict(model , newdata = train_data,type = "prob")[, "1"];
    test_pred_prob = predict(model , newdata = test_data,type = "prob")[, "1"];
    
    model_roc_train = roc(train_data$h1n1_vaccine, train_pred_prob)
    model_roc_test = roc(test_data$h1n1_vaccine, test_pred_prob)
    
    return(paste("Train：",model_roc_train$auc,"Test：",model_roc_test$auc));
}

1. p-value + CART

p_cart_AUC <- Map(function(x) cart_func(x),
c(h1n1_vaccine ~ doctor_recc_h1n1,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children,
    h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own+behavioral_wash_hands,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own+behavioral_wash_hands+behavioral_touch_face
  )) 

p_cart_AUC

## [[1]]
## [1] "Train： 0.695225743645184 Test： 0.689736842841515"
## 
## [[2]]
## [1] "Train： 0.695225743645184 Test： 0.689736842841515"
## 
## [[3]]
## [1] "Train： 0.695225743645184 Test： 0.689736842841515"
## 
## [[4]]
## [1] "Train： 0.703379879767804 Test： 0.698271171803817"
## 
## [[5]]
## [1] "Train： 0.703379879767804 Test： 0.698271171803817"
## 
## [[6]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[7]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[8]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[9]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[10]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[11]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[12]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[13]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[14]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[15]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[16]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[17]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[18]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[19]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[20]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[21]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[22]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[23]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[24]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[25]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[26]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[27]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[28]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[29]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[30]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[31]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[32]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"

aaaa<-as.data.frame(p_cart_AUC)
colnames(aaaa) <- c(1:32)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:32),
  AUC = aaaa
)
library(tidyr)
ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
p_cart_AUC_df <- ccc[,-c(2,4)]

plot(p_cart_AUC_df$numOfVar, p_cart_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(p_cart_AUC_df$numOfVar, p_cart_AUC_df$test, col = "blue")

2. 隨機森林 + CART

rf_cart_AUC<-Map(function(x) cart_func(x),
c(h1n1_vaccine ~  doctor_recc_h1n1,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective,  
 h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home,       
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months+behavioral_face_mask,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months+behavioral_face_mask+behavioral_antiviral_meds   
))

rf_cart_AUC

## [[1]]
## [1] "Train： 0.695225743645184 Test： 0.689736842841515"
## 
## [[2]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[3]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[4]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[5]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[6]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[7]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[8]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[9]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[10]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[11]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[12]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[13]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[14]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[15]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[16]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[17]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[18]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[19]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[20]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[21]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[22]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[23]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[24]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[25]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[26]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[27]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[28]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[29]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[30]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[31]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[32]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"

aaaa<-as.data.frame(rf_cart_AUC)
colnames(aaaa) <- c(1:32)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:32),
  AUC = aaaa
)
library(tidyr)
ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
rf_cart_AUC_df <- ccc[,-c(2,4)]

plot(rf_cart_AUC_df$numOfVar, rf_cart_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(rf_cart_AUC_df$numOfVar, rf_cart_AUC_df$test, col = "blue")

3. elastic net + CART

en_cart_AUC<-Map(function(x) cart_func(x),
c(h1n1_vaccine ~ doctor_recc_h1n1,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition,  
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults, 
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home+income_poverty,  
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home+income_poverty+behavioral_wash_hands
))
en_cart_AUC

## [[1]]
## [1] "Train： 0.695225743645184 Test： 0.689736842841515"
## 
## [[2]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[3]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[4]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[5]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[6]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[7]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[8]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[9]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[10]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[11]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[12]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[13]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[14]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[15]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[16]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[17]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[18]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[19]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[20]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[21]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[22]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[23]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[24]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[25]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[26]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[27]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[28]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[29]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"
## 
## [[30]]
## [1] "Train： 0.707512015632318 Test： 0.702414465530217"

aaaa<-as.data.frame(en_cart_AUC)
colnames(aaaa) <- c(1:30)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:30),
  AUC = aaaa
)
library(tidyr)
ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
en_cart_AUC_df <- ccc[,-c(2,4)]

plot(en_cart_AUC_df$numOfVar, en_cart_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(en_cart_AUC_df$numOfVar, en_cart_AUC_df$test, col = "blue")

五、隨機森林模型

library(randomForest)

rf_func = function(f){ 
    model <- randomForest(f,train_data,ntree=100, mtry=2, importance=TRUE)
    
    train_pred_prob = predict(model , newdata = train_data,type = "prob")[, "1"];
    test_pred_prob = predict(model , newdata = test_data,type = "prob")[, "1"];
    
    model_roc_train = roc(train_data$h1n1_vaccine, train_pred_prob)
    model_roc_test = roc(test_data$h1n1_vaccine, test_pred_prob)
    
    return(paste("Train：",model_roc_train$auc,"Test：",model_roc_test$auc));
}

1. p-value + 隨機森林

p_rf_AUC <- Map(function(x) rf_func(x),
c(h1n1_vaccine ~ doctor_recc_h1n1,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children,
    h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own+behavioral_wash_hands,
  h1n1_vaccine ~  doctor_recc_h1n1 + doctor_recc_seasonal +health_worker + opinion_h1n1_vacc_effective + opinion_h1n1_risk + seasonal_vaccine + opinion_h1n1_sick_from_vacc + behavioral_large_gatherings +sex +child_under_6_months +opinion_seas_vacc_effective + behavioral_face_mask +race +age_group + opinion_seas_risk+opinion_seas_sick_from_vacc + h1n1_concern + marital_status + behavioral_antiviral_meds + census_msa+education +behavioral_avoidance+chronic_med_condition+income_poverty+employment_status+household_children+household_adults+h1n1_knowledge+behavioral_outside_home+rent_or_own+behavioral_wash_hands+behavioral_touch_face
  )) 

p_rf_AUC

## [[1]]
## [1] "Train： 0.695225743645184 Test： 0.689736842841515"
## 
## [[2]]
## [1] "Train： 0.695952407952266 Test： 0.690952422252625"
## 
## [[3]]
## [1] "Train： 0.697850106823848 Test： 0.693005603910147"
## 
## [[4]]
## [1] "Train： 0.732534868129828 Test： 0.724761048114917"
## 
## [[5]]
## [1] "Train： 0.775996155626516 Test： 0.769499443712364"
## 
## [[6]]
## [1] "Train： 0.833532773622302 Test： 0.82660020125209"
## 
## [[7]]
## [1] "Train： 0.820326762518309 Test： 0.81051971160706"
## 
## [[8]]
## [1] "Train： 0.840698286744519 Test： 0.82733843644472"
## 
## [[9]]
## [1] "Train： 0.84831663749312 Test： 0.828667138555256"
## 
## [[10]]
## [1] "Train： 0.849206924520352 Test： 0.82837342054879"
## 
## [[11]]
## [1] "Train： 0.862154608244657 Test： 0.832196977115269"
## 
## [[12]]
## [1] "Train： 0.865797687333404 Test： 0.833913774950177"
## 
## [[13]]
## [1] "Train： 0.877305311690772 Test： 0.838740467803862"
## 
## [[14]]
## [1] "Train： 0.902733205182994 Test： 0.840847226535433"
## 
## [[15]]
## [1] "Train： 0.921069578908428 Test： 0.84795479195402"
## 
## [[16]]
## [1] "Train： 0.934834376554202 Test： 0.853258968914106"
## 
## [[17]]
## [1] "Train： 0.948952462567223 Test： 0.85345415919432"
## 
## [[18]]
## [1] "Train： 0.956270593393217 Test： 0.854767660150687"
## 
## [[19]]
## [1] "Train： 0.951527337349276 Test： 0.854783234339314"
## 
## [[20]]
## [1] "Train： 0.964132985506462 Test： 0.853367521941421"
## 
## [[21]]
## [1] "Train： 0.973689774193469 Test： 0.857547046639566"
## 
## [[22]]
## [1] "Train： 0.97521668603823 Test： 0.856566012644003"
## 
## [[23]]
## [1] "Train： 0.977673060376117 Test： 0.854690815052313"
## 
## [[24]]
## [1] "Train： 0.980869667317671 Test： 0.856633018958544"
## 
## [[25]]
## [1] "Train： 0.983272135872717 Test： 0.855155522728523"
## 
## [[26]]
## [1] "Train： 0.985234463582692 Test： 0.855794577384599"
## 
## [[27]]
## [1] "Train： 0.987054255824388 Test： 0.853016543145632"
## 
## [[28]]
## [1] "Train： 0.989831287972433 Test： 0.853517388534972"
## 
## [[29]]
## [1] "Train： 0.991140944730757 Test： 0.853736033356741"
## 
## [[30]]
## [1] "Train： 0.991116209076719 Test： 0.852460954949561"
## 
## [[31]]
## [1] "Train： 0.991881279865632 Test： 0.853899375820093"
## 
## [[32]]
## [1] "Train： 0.992727880053797 Test： 0.851164007397273"

aaaa<-as.data.frame(p_rf_AUC)
colnames(aaaa) <- c(1:32)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:32),
  AUC = aaaa
)
library(tidyr)
ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
p_rf_AUC_df <- ccc[,-c(2,4)]

plot(p_rf_AUC_df$numOfVar, p_rf_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(p_rf_AUC_df$numOfVar, p_rf_AUC_df$test, col = "blue")

p-value + 隨機森林最好的模型

後面的training的AUC有增加，但是testing的沒有，推測已經overfitting了

# 以上圖來看最好的模型是5個變數的模型
p_rf_AUC[[6]]

## [1] "Train： 0.833532773622302 Test： 0.82660020125209"

#1 doctor_recc_h1n1
#2 doctor_recc_seasonal
#3 health_worker
#4 opinion_h1n1_vacc_effective
#5 opinion_h1n1_risk
#6 seasonal_vaccine

2. 隨機森林 + 隨機森林

rf_rf_AUC<-Map(function(x) rf_func(x),
c(h1n1_vaccine ~  doctor_recc_h1n1,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group,
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective,  
 h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home,       
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance,   
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months, 
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months+behavioral_face_mask,  
  h1n1_vaccine ~  doctor_recc_h1n1 + seasonal_vaccine +opinion_h1n1_risk+opinion_h1n1_vacc_effective+age_group+opinion_seas_risk+opinion_h1n1_sick_from_vacc+h1n1_concern+education+opinion_seas_sick_from_vacc+census_msa+doctor_recc_seasonal+household_children+h1n1_knowledge+household_adults+opinion_seas_vacc_effective+race+income_poverty+employment_status+health_worker+sex+chronic_med_condition+behavioral_large_gatherings+behavioral_outside_home+behavioral_touch_face+marital_status+behavioral_avoidance+rent_or_own+behavioral_wash_hands+child_under_6_months+behavioral_face_mask+behavioral_antiviral_meds   
))

rf_rf_AUC

## [[1]]
## [1] "Train： 0.695225743645184 Test： 0.689736842841515"
## 
## [[2]]
## [1] "Train： 0.690911144950967 Test： 0.686056671394878"
## 
## [[3]]
## [1] "Train： 0.727274598467281 Test： 0.721827411877604"
## 
## [[4]]
## [1] "Train： 0.793613178937738 Test： 0.793955489528457"
## 
## [[5]]
## [1] "Train： 0.819836979479814 Test： 0.810950706294117"
## 
## [[6]]
## [1] "Train： 0.826489298270941 Test： 0.810930236028228"
## 
## [[7]]
## [1] "Train： 0.852576629774571 Test： 0.826649861464328"
## 
## [[8]]
## [1] "Train： 0.880326342482125 Test： 0.837092261679942"
## 
## [[9]]
## [1] "Train： 0.908659594474033 Test： 0.841731318201191"
## 
## [[10]]
## [1] "Train： 0.930578254825871 Test： 0.845068344575659"
## 
## [[11]]
## [1] "Train： 0.950550625057588 Test： 0.847880511467546"
## 
## [[12]]
## [1] "Train： 0.95376940437297 Test： 0.855318911821182"
## 
## [[13]]
## [1] "Train： 0.961844044631972 Test： 0.858061228010784"
## 
## [[14]]
## [1] "Train： 0.968695829344869 Test： 0.858327294838042"
## 
## [[15]]
## [1] "Train： 0.975558029519692 Test： 0.855735917716059"
## 
## [[16]]
## [1] "Train： 0.980305566241586 Test： 0.856778502397213"
## 
## [[17]]
## [1] "Train： 0.982237954838694 Test： 0.858964950614901"
## 
## [[18]]
## [1] "Train： 0.984424441403141 Test： 0.860442446844921"
## 
## [[19]]
## [1] "Train： 0.986695986149264 Test： 0.857141884588734"
## 
## [[20]]
## [1] "Train： 0.987452709188956 Test： 0.857671453631351"
## 
## [[21]]
## [1] "Train： 0.989194816951717 Test： 0.86025061387483"
## 
## [[22]]
## [1] "Train： 0.989660376992216 Test： 0.855710178338449"
## 
## [[23]]
## [1] "Train： 0.990297001809744 Test： 0.855796069522432"
## 
## [[24]]
## [1] "Train： 0.992572041161222 Test： 0.856501337794825"
## 
## [[25]]
## [1] "Train： 0.99282061953184 Test： 0.858400689367679"
## 
## [[26]]
## [1] "Train： 0.99333069520534 Test： 0.856723479814639"
## 
## [[27]]
## [1] "Train： 0.994231560035577 Test： 0.854161758932077"
## 
## [[28]]
## [1] "Train： 0.993832867431522 Test： 0.856144996628701"
## 
## [[29]]
## [1] "Train： 0.994598596129023 Test： 0.85436156551371"
## 
## [[30]]
## [1] "Train： 0.993593320351515 Test： 0.852997285241731"
## 
## [[31]]
## [1] "Train： 0.993283872620148 Test： 0.851444016387404"
## 
## [[32]]
## [1] "Train： 0.991986271823084 Test： 0.854971849887204"

aaaa<-as.data.frame(rf_rf_AUC)
colnames(aaaa) <- c(1:32)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:32),
  AUC = aaaa
)
library(tidyr)
ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
rf_rf_AUC_df <- ccc[,-c(2,4)]

plot(rf_rf_AUC_df$numOfVar, rf_rf_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(rf_rf_AUC_df$numOfVar, rf_rf_AUC_df$test, col = "blue")

隨機森林 + 隨機森林最好的模型

後面的training的AUC有增加，但是testing的沒有，推測已經overfitting了

# 以上圖來看最好的模型是7個變數的模型
p_rf_AUC[[7]]

## [1] "Train： 0.820326762518309 Test： 0.81051971160706"

#1 doctor_recc_h1n1
#2 doctor_recc_seasonal
#3 health_worker
#4 opinion_h1n1_vacc_effective
#5 opinion_h1n1_risk
#6 seasonal_vaccine
#7 opinion_h1n1_sick_from_vacc

3. elastic net + 隨機森林

en_rf_AUC<-Map(function(x) rf_func(x),
c(h1n1_vaccine ~ doctor_recc_h1n1,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition,  
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults, 
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home,
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home+income_poverty,  
  h1n1_vaccine ~ doctor_recc_h1n1+seasonal_vaccine+opinion_h1n1_risk+opinion_h1n1_vacc_effective+doctor_recc_seasonal+health_worker+opinion_h1n1_sick_from_vacc+race+behavioral_large_gatherings+opinion_seas_vacc_effective+child_under_6_months+opinion_seas_sick_from_vacc+behavioral_face_mask+age_group+sex+opinion_seas_risk+h1n1_concern+behavioral_antiviral_meds+marital_status+education+census_msa+employment_status+behavioral_avoidance+h1n1_knowledge+household_children+chronic_med_condition+household_adults+behavioral_outside_home+income_poverty+behavioral_wash_hands
))
en_rf_AUC

## [[1]]
## [1] "Train： 0.695225743645184 Test： 0.689736842841515"
## 
## [[2]]
## [1] "Train： 0.690911144950967 Test： 0.686056671394878"
## 
## [[3]]
## [1] "Train： 0.727385229641198 Test： 0.721972802057658"
## 
## [[4]]
## [1] "Train： 0.790843520492434 Test： 0.781269007271374"
## 
## [[5]]
## [1] "Train： 0.824169803096333 Test： 0.814722923993297"
## 
## [[6]]
## [1] "Train： 0.833720183582812 Test： 0.821618465951746"
## 
## [[7]]
## [1] "Train： 0.837536745476255 Test： 0.826724654873182"
## 
## [[8]]
## [1] "Train： 0.840318058302253 Test： 0.821387044699787"
## 
## [[9]]
## [1] "Train： 0.849305038342571 Test： 0.825102887566482"
## 
## [[10]]
## [1] "Train： 0.862441200060808 Test： 0.829704780529839"
## 
## [[11]]
## [1] "Train： 0.867885402167045 Test： 0.833636656977097"
## 
## [[12]]
## [1] "Train： 0.884456402089668 Test： 0.838961723866838"
## 
## [[13]]
## [1] "Train： 0.884484683614665 Test： 0.835785615231743"
## 
## [[14]]
## [1] "Train： 0.908275999590422 Test： 0.842200735437434"
## 
## [[15]]
## [1] "Train： 0.91860264385607 Test： 0.846603101595002"
## 
## [[16]]
## [1] "Train： 0.934951151056359 Test： 0.851769582210733"
## 
## [[17]]
## [1] "Train： 0.949187182136165 Test： 0.855626618619828"
## 
## [[18]]
## [1] "Train： 0.948969089711526 Test： 0.850129629474199"
## 
## [[19]]
## [1] "Train： 0.954093727673829 Test： 0.854328878369317"
## 
## [[20]]
## [1] "Train： 0.963913551631846 Test： 0.857494308893048"
## 
## [[21]]
## [1] "Train： 0.972018771823768 Test： 0.857207911687822"
## 
## [[22]]
## [1] "Train： 0.97535697436419 Test： 0.858603200449134"
## 
## [[23]]
## [1] "Train： 0.978809029267328 Test： 0.855227285232405"
## 
## [[24]]
## [1] "Train： 0.982503203074952 Test： 0.85394922254956"
## 
## [[25]]
## [1] "Train： 0.984696191826868 Test： 0.853595632512564"
## 
## [[26]]
## [1] "Train： 0.985920149583464 Test： 0.857499951039227"
## 
## [[27]]
## [1] "Train： 0.987878717815795 Test： 0.854286352441091"
## 
## [[28]]
## [1] "Train： 0.98890191092198 Test： 0.852834688847296"
## 
## [[29]]
## [1] "Train： 0.990758289716557 Test： 0.853542381843667"
## 
## [[30]]
## [1] "Train： 0.99155429929762 Test： 0.852146999823741"

aaaa<-as.data.frame(en_rf_AUC)
colnames(aaaa) <- c(1:30)
aaaa <- t(aaaa)
bbb<-data.frame(
  numOfVar = c(1:30),
  AUC = aaaa
)
library(tidyr)
ccc <- separate(bbb, AUC, c("del","trainAUC","del2", "test"), " ")
en_rf_AUC_df <- ccc[,-c(2,4)]

plot(en_rf_AUC_df$numOfVar, en_rf_AUC_df$trainAUC ,type = "l", ylim = c(0.5, 1))
lines(en_rf_AUC_df$numOfVar, en_rf_AUC_df$test, col = "blue")

隨機森林 + 隨機森林最好的模型

後面的training的AUC有增加，但是testing的沒有，推測已經overfitting了

# 以上圖來看最好的模型是6個變數的模型
p_rf_AUC[[6]]

## [1] "Train： 0.833532773622302 Test： 0.82660020125209"

#1 doctor_recc_h1n1
#2 seasonal_vaccine
#3 opinion_h1n1_risk
#4 opinion_h1n1_vacc_effective
#5 doctor_recc_seasonal
#6 health_worker

六、最終模型

1. AUC

2. Confusion matrix