Sample data
See how many full rows do we have
sum(complete.cases(sample_dirty_dataset))
## [1] 732
nrow(sample_dirty_dataset[complete.cases(sample_dirty_dataset), ])/nrow(sample_dirty_dataset)*100
## [1] 73.2
Visualsing missing data
vis_miss(sample_dirty_dataset)
Correlationbetween missing data
gg_miss_upset(sample_dirty_dataset)
miss_var_summary(sample_dirty_dataset)
## # A tibble: 14 × 3
## variable n_miss pct_miss
## <chr> <int> <num>
## 1 workclass 50 5
## 2 marital_status 50 5
## 3 relationship 50 5
## 4 sex 50 5
## 5 hours_per_week 50 5
## 6 native_country 50 5
## 7 age 0 0
## 8 education 0 0
## 9 education_num 0 0
## 10 occupation 0 0
## 11 race 0 0
## 12 capital_gain 0 0
## 13 capital_loss 0 0
## 14 income 0 0
# Convert the variable dose from a numeric to a factor variable
#tooth$dose <- as.factor(tooth$dose)
# Change box plot line colors by groups
p<-ggplot(sample_dirty_dataset, aes(y=capital_gain)) +
geom_boxplot()
p
Repleacing invalid data with NA
#cr <- correctionRules(expression(
# if (!is.na(capital_gain) && capital_gain <=0){capital_gain = NA}
# else if (!is.na(capital_loss) && capital_loss <=0){capital_loss = NA}
# ))
RULE <- editset(c("capital_loss >= 0", "capital_gain >= 0"))
sample_dirty_dataset[localizeErrors(RULE, sample_dirty_dataset)$adapt] <- NA
Imputing data
sample_dirty_dataset <- mutate(sample_dirty_dataset,
capital_gain = imputate_na(sample_dirty_dataset, capital_gain, method="median"),
capital_loss = imputate_na(sample_dirty_dataset, capital_loss, method = "median"),
hours_per_week = imputate_na(sample_dirty_dataset, hours_per_week, method = "mean"))
sample_dirty_dataset <- sample_dirty_dataset[!(sample_dirty_dataset$age %in% "InvalidAge"),]
sample_dirty_dataset <- sample_dirty_dataset[!(sample_dirty_dataset$occupation %in% "InvalidOccupation"),]
sample_dirty_dataset <- sample_dirty_dataset[!(sample_dirty_dataset$race %in% "InvalidRace"),]
sample_dirty_dataset <- sample_dirty_dataset[!(sample_dirty_dataset$education %in% "InvalidEducation"),]
sample_dirty_dataset <- sample_dirty_dataset[complete.cases(sample_dirty_dataset$native_country), ]
sample_dirty_dataset <- sample_dirty_dataset[complete.cases(sample_dirty_dataset$sex), ]
sample_dirty_dataset <- sample_dirty_dataset[complete.cases(sample_dirty_dataset$workclass), ]
sample_dirty_dataset <- mutate(sample_dirty_dataset, relationship = ifelse(is.na(relationship), "Other-relative", relationship))
sample_dirty_dataset <- mutate(sample_dirty_dataset, marital_status = ifelse(is.na(marital_status), "Never-married", marital_status))
#/sample_dirty_dataset <- mutate(sample_dirty_dataset, education = imputate_na(sample_dirty_dataset, education, education_num, method="knn"))
#sample_dirty_dataset$workclass <- imputate_na(sample_dirty_dataset, workclass, education, method="knn")
Checking for outliers in capital gain
zscore <- abs(scale(sample_dirty_dataset$capital_gain))
plot(zscore, type='n')
abline(h=2, col="red")
text(1:length(sample_dirty_dataset$capital_gain), zscore)
# every score is under 2 so there should be no outliers in capital gain
boxplot(sample_dirty_dataset$capital_loss)
# capital loss has many outliers, but they are not so far from the 25/75 percentile, and are distributed evenly
# so I think there is no need to delete them
Checking for outliers in hours per week
par(mfrow=c(2, 2))
plot(covMcd(sample_dirty_dataset$hours_per_week))
# again we can see that there are no significant outliers in hours_per_week
Binding
sample_dirty_dataset$age <- as.numeric(sample_dirty_dataset$age)
bin <- binning(sample_dirty_dataset$age)
plot(bin)
## Don't know how to automatically pick scale for object of type <table>.
## Defaulting to continuous.
sample_dirty_dataset %>%
mutate(Age_bin = binning(sample_dirty_dataset$age)) %>%
group_by(Age_bin) %>%
summarise(freq = n(),
avg_cap_gain = mean(capital_gain),
avg_cap_loss = mean(capital_loss),
avg_hours_per_week = mean(hours_per_week)) %>%
arrange(desc(freq)) %>%
head(10)
## # A tibble: 10 × 5
## Age_bin freq avg_cap_gain avg_cap_loss avg_hours_per_week
## <bins> <int> <dbl> <dbl> <dbl>
## 1 [18,25] 70 51570. 47976. 47.3
## 2 (38,44] 68 53363. 48195. 47.1
## 3 (50,56] 67 49257. 48959. 51.7
## 4 (44,50] 66 50750. 44275. 52.2
## 5 (25,31] 65 51554. 49596. 52.8
## 6 (69,77] 65 51525. 51456. 47.6
## 7 (77,84] 64 47006. 45950. 50.0
## 8 (63,69] 61 47060. 50361. 50.0
## 9 (31,38] 59 53726. 49229. 47.8
## 10 (56,63] 59 50588. 42543. 42.6
