Sample data
# Load necessary libraries
library(tidyverse)## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr 1.1.4 ✔ readr 2.1.5
## ✔ forcats 1.0.0 ✔ stringr 1.5.1
## ✔ ggplot2 3.5.0 ✔ tibble 3.2.1
## ✔ lubridate 1.9.3 ✔ tidyr 1.3.1
## ✔ purrr 1.0.2
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errorslibrary(flextable)##
## Attaching package: 'flextable'
##
## The following object is masked from 'package:purrr':
##
## composelibrary(dlookr)## Registered S3 methods overwritten by 'dlookr':
## method from
## plot.transform scales
## print.transform scales
##
## Attaching package: 'dlookr'
##
## The following object is masked from 'package:tidyr':
##
## extract
##
## The following object is masked from 'package:base':
##
## transform# Set seed for reproducibility
set.seed(124)
# Generate sample data
n <- 1000 # Number of observations
# Age between 18 and 90
age <- sample(18:90, n, replace = TRUE)
# Introduce errors in age column
age[sample(1:n, 50)] <- "InvalidAge"
# Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
# Introduce missing values in workclass column
workclass[sample(1:n, 50)] <- NA
# Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
# Introduce inconsistencies in education column
education[sample(1:n, 50)] <- "InvalidEducation"
# Education Number: 1-16
education_num <- sample(1:16, n, replace = TRUE)
# Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
# Introduce missing values in marital_status column
marital_status[sample(1:n, 50)] <- NA
# Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
# Introduce errors in occupation column
occupation[sample(1:n, 50)] <- "InvalidOccupation"
# Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
# Introduce missing values in relationship column
relationship[sample(1:n, 50)] <- NA
# Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
# Introduce inconsistencies in race column
race[sample(1:n, 50)] <- "InvalidRace"
# Sex: Female, Male
sex <- sample(c("Female", "Male"), n, replace = TRUE)
# Introduce missing values in sex column
sex[sample(1:n, 50)] <- NA
# Capital Gain: 0-99999
capital_gain <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_gain column
capital_gain[sample(1:n, 50)] <- -999
# Capital Loss: 0-99999
capital_loss <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_loss column
capital_loss[sample(1:n, 50)] <- -999
# Hours per week: 1-99
hours_per_week <- sample(1:99, n, replace = TRUE)
# Introduce missing values in hours_per_week column
hours_per_week[sample(1:n, 50)] <- NA
# Native country: United-States, Cambodia, England, ...
native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
# Introduce missing values in native_country column
native_country[sample(1:n, 50)] <- NA
# Income: <=50K, >50K
income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
# Create dataframe
sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)
# Print first few rows of the dataset
head(sample_dirty_dataset)## age workclass education education_num marital_status
## 1 82 Federal-gov Doctorate 9 Divorced
## 2 56 Self-emp-inc 7th-8th 11 Divorced
## 3 44 Private Some-college 8 Never-married
## 4 22 Federal-gov 7th-8th 5 Married-AF-spouse
## 5 23 Self-emp-not-inc Assoc-acdm 15 Widowed
## 6 62 Self-emp-not-inc 5th-6th 6 Divorced
## occupation relationship race sex capital_gain
## 1 Sales Not-in-family InvalidRace Female -999
## 2 Tech-support Husband Amer-Indian-Eskimo Male 51355
## 3 Adm-clerical Unmarried InvalidRace Male 32620
## 4 Machine-op-inspct Wife White Female 18335
## 5 Priv-house-serv Wife Amer-Indian-Eskimo Male 3143
## 6 Sales Husband Other <NA> 49465
## capital_loss hours_per_week native_country income
## 1 68148 66 <NA> <=50K
## 2 86057 18 Poland <=50K
## 3 56436 20 Taiwan <=50K
## 4 60665 78 Haiti <=50K
## 5 58774 31 Cuba <=50K
## 6 43755 30 Cambodia >50Kdiagnose(sample_dirty_dataset)## # A tibble: 14 × 6
## variables types missing_count missing_percent unique_count unique_rate
## <chr> <chr> <int> <dbl> <int> <dbl>
## 1 age charac… 0 0 74 0.074
## 2 workclass charac… 50 5 9 0.009
## 3 education charac… 0 0 17 0.017
## 4 education_num integer 0 0 16 0.016
## 5 marital_status charac… 50 5 8 0.008
## 6 occupation charac… 0 0 15 0.015
## 7 relationship charac… 50 5 7 0.007
## 8 race charac… 0 0 6 0.006
## 9 sex charac… 50 5 3 0.003
## 10 capital_gain numeric 0 0 944 0.944
## 11 capital_loss numeric 0 0 951 0.951
## 12 hours_per_week integer 50 5 100 0.1
## 13 native_country charac… 50 5 42 0.042
## 14 income charac… 0 0 2 0.002# Convert age column to integer
sample_dirty_dataset$age <- as.integer(sample_dirty_dataset$age)## Warning: NAs introduced by coercion#task 1 Examine the variables in the dataset and their data types.Identify which variables contain errors, inconsistencies, or missing values.
# Examine the structure of the dataset and their data types
str(sample_dirty_dataset)## 'data.frame': 1000 obs. of 14 variables:
## $ age : int 82 56 44 22 23 62 50 32 37 63 ...
## $ workclass : chr "Federal-gov" "Self-emp-inc" "Private" "Federal-gov" ...
## $ education : chr "Doctorate" "7th-8th" "Some-college" "7th-8th" ...
## $ education_num : int 9 11 8 5 15 6 13 14 14 9 ...
## $ marital_status: chr "Divorced" "Divorced" "Never-married" "Married-AF-spouse" ...
## $ occupation : chr "Sales" "Tech-support" "Adm-clerical" "Machine-op-inspct" ...
## $ relationship : chr "Not-in-family" "Husband" "Unmarried" "Wife" ...
## $ race : chr "InvalidRace" "Amer-Indian-Eskimo" "InvalidRace" "White" ...
## $ sex : chr "Female" "Male" "Male" "Female" ...
## $ capital_gain : num -999 51355 32620 18335 3143 ...
## $ capital_loss : num 68148 86057 56436 60665 58774 ...
## $ hours_per_week: int 66 18 20 78 31 30 16 9 19 97 ...
## $ native_country: chr NA "Poland" "Taiwan" "Haiti" ...
## $ income : chr "<=50K" "<=50K" "<=50K" "<=50K" ...# Check for missing values in each variable
colSums(is.na(sample_dirty_dataset))## age workclass education education_num marital_status
## 50 50 0 0 50
## occupation relationship race sex capital_gain
## 0 50 0 50 0
## capital_loss hours_per_week native_country income
## 0 50 50 0# Summary statistics for numeric variables
summary(sample_dirty_dataset[, sapply(sample_dirty_dataset, is.numeric)])## age education_num capital_gain capital_loss
## Min. :18.0 Min. : 1.000 Min. : -999 Min. : -999
## 1st Qu.:38.0 1st Qu.: 5.000 1st Qu.:21586 1st Qu.:22277
## Median :56.0 Median : 8.000 Median :46921 Median :48667
## Mean :55.2 Mean : 8.267 Mean :46757 Mean :48056
## 3rd Qu.:73.0 3rd Qu.:12.000 3rd Qu.:71237 3rd Qu.:72606
## Max. :90.0 Max. :16.000 Max. :99890 Max. :99784
## NA's :50
## hours_per_week
## Min. : 1.00
## 1st Qu.:25.00
## Median :49.50
## Mean :49.19
## 3rd Qu.:72.00
## Max. :99.00
## NA's :50# Identify unique values and their frequencies for categorical variables
unique_values <- lapply(sample_dirty_dataset[, sapply(sample_dirty_dataset, is.character)], table)
print(unique_values)## $workclass
##
## Federal-gov Local-gov Never-worked Private
## 113 116 115 129
## Self-emp-inc Self-emp-not-inc State-gov Without-pay
## 121 108 129 119
##
## $education
##
## 10th 11th 12th 1st-4th
## 63 55 44 52
## 5th-6th 7th-8th 9th Assoc-acdm
## 71 58 64 67
## Assoc-voc Bachelors Doctorate HS-grad
## 68 56 67 46
## InvalidEducation Masters Preschool Prof-school
## 50 65 63 54
## Some-college
## 57
##
## $marital_status
##
## Divorced Married-AF-spouse Married-civ-spouse
## 147 134 123
## Married-spouse-absent Never-married Separated
## 147 133 138
## Widowed
## 128
##
## $occupation
##
## Adm-clerical Armed-Forces Craft-repair Exec-managerial
## 75 75 58 78
## Farming-fishing Handlers-cleaners InvalidOccupation Machine-op-inspct
## 77 66 50 70
## Other-service Priv-house-serv Prof-specialty Protective-serv
## 63 58 52 63
## Sales Tech-support Transport-moving
## 68 73 74
##
## $relationship
##
## Husband Not-in-family Other-relative Own-child Unmarried
## 150 151 181 162 151
## Wife
## 155
##
## $race
##
## Amer-Indian-Eskimo Asian-Pac-Islander Black InvalidRace
## 209 199 181 50
## Other White
## 159 202
##
## $sex
##
## Female Male
## 457 493
##
## $native_country
##
## Cambodia Canada
## 29 20
## China Columbia
## 32 18
## Cuba Dominican-Republic
## 20 26
## Ecuador El-Salvador
## 28 27
## England France
## 20 32
## Germany Greece
## 19 27
## Guatemala Haiti
## 18 20
## Holand-Netherlands Honduras
## 22 21
## Hong Hungary
## 26 17
## India Iran
## 20 19
## Ireland Italy
## 25 26
## Jamaica Japan
## 24 24
## Laos Mexico
## 17 32
## Nicaragua Outlying-US(Guam-USVI-etc)
## 19 21
## Peru Philippines
## 22 28
## Poland Portugal
## 24 27
## Puerto-Rico Scotland
## 32 19
## South Taiwan
## 19 26
## Thailand Trinadad&Tobago
## 19 13
## United-States Vietnam
## 24 24
## Yugoslavia
## 24
##
## $income
##
## <=50K >50K
## 765 235# conclusion Missing Values:workclass marital_status race sex native_country
# capital_gain and capital_loss contain negative values
invalid_age_rows <- sample_dirty_dataset[sample_dirty_dataset$age == "InvalidAge", ]
#print(invalid_age_rows)
plot_na_pareto(sample_dirty_dataset)
bla <- imputate_na(sample_dirty_dataset, age, method = "mean")
plot(bla)
bla2 <- imputate_na(sample_dirty_dataset, age, method = "median")
plot(bla2)
bla3 <- imputate_na(sample_dirty_dataset, age, method = "rpart")
plot(bla3)
#Replace age by best option # Replace "InvalidEducation" and "InvalidRace" with NA
sample_dirty_dataset$education[sample_dirty_dataset$education == "InvalidEducation"] <- NA
sample_dirty_dataset$race[sample_dirty_dataset$race == "InvalidRace"] <- NA
#correct capital_gain and capital_loss
sample_dirty_dataset$capital_gain[sample_dirty_dataset$capital_gain == -999] <- NA
sample_dirty_dataset$capital_loss[sample_dirty_dataset$capital_loss == -999] <- NA
# Replace "InvalidEducation" and "InvalidRace" with NA
sample_dirty_dataset$education[sample_dirty_dataset$education == "InvalidEducation"] <- NA
sample_dirty_dataset$race[sample_dirty_dataset$race == "InvalidRace"] <- NA
#correct capital_gain and capital_loss
sample_dirty_dataset$capital_gain[sample_dirty_dataset$capital_gain == -999] <- NA
sample_dirty_dataset$capital_loss[sample_dirty_dataset$capital_loss == -999] <- NA
#task 5
library(dlookr)
library(mice)##
## Attaching package: 'mice'## The following object is masked from 'package:stats':
##
## filter## The following objects are masked from 'package:base':
##
## cbind, rbind#changing NA values for mean vaues
#Mean to corect capital gain
data_multiple <- complete(mice(sample_dirty_dataset, m = 5), action = "broad", include = TRUE)##
## iter imp variable
## 1 1 age capital_gain capital_loss hours_per_week
## 1 2 age capital_gain capital_loss hours_per_week
## 1 3 age capital_gain capital_loss hours_per_week
## 1 4 age capital_gain capital_loss hours_per_week
## 1 5 age capital_gain capital_loss hours_per_week
## 2 1 age capital_gain capital_loss hours_per_week
## 2 2 age capital_gain capital_loss hours_per_week
## 2 3 age capital_gain capital_loss hours_per_week
## 2 4 age capital_gain capital_loss hours_per_week
## 2 5 age capital_gain capital_loss hours_per_week
## 3 1 age capital_gain capital_loss hours_per_week
## 3 2 age capital_gain capital_loss hours_per_week
## 3 3 age capital_gain capital_loss hours_per_week
## 3 4 age capital_gain capital_loss hours_per_week
## 3 5 age capital_gain capital_loss hours_per_week
## 4 1 age capital_gain capital_loss hours_per_week
## 4 2 age capital_gain capital_loss hours_per_week
## 4 3 age capital_gain capital_loss hours_per_week
## 4 4 age capital_gain capital_loss hours_per_week
## 4 5 age capital_gain capital_loss hours_per_week
## 5 1 age capital_gain capital_loss hours_per_week
## 5 2 age capital_gain capital_loss hours_per_week
## 5 3 age capital_gain capital_loss hours_per_week
## 5 4 age capital_gain capital_loss hours_per_week
## 5 5 age capital_gain capital_loss hours_per_week## Warning: Number of logged events: 9## New names:
## • `age` -> `age...1`
## • `workclass` -> `workclass...2`
## • `education` -> `education...3`
## • `education_num` -> `education_num...4`
## • `marital_status` -> `marital_status...5`
## • `occupation` -> `occupation...6`
## • `relationship` -> `relationship...7`
## • `race` -> `race...8`
## • `sex` -> `sex...9`
## • `capital_gain` -> `capital_gain...10`
## • `capital_loss` -> `capital_loss...11`
## • `hours_per_week` -> `hours_per_week...12`
## • `native_country` -> `native_country...13`
## • `income` -> `income...14`
## • `age` -> `age...15`
## • `workclass` -> `workclass...16`
## • `education` -> `education...17`
## • `education_num` -> `education_num...18`
## • `marital_status` -> `marital_status...19`
## • `occupation` -> `occupation...20`
## • `relationship` -> `relationship...21`
## • `race` -> `race...22`
## • `sex` -> `sex...23`
## • `capital_gain` -> `capital_gain...24`
## • `capital_loss` -> `capital_loss...25`
## • `hours_per_week` -> `hours_per_week...26`
## • `native_country` -> `native_country...27`
## • `income` -> `income...28`
## • `age` -> `age...29`
## • `workclass` -> `workclass...30`
## • `education` -> `education...31`
## • `education_num` -> `education_num...32`
## • `marital_status` -> `marital_status...33`
## • `occupation` -> `occupation...34`
## • `relationship` -> `relationship...35`
## • `race` -> `race...36`
## • `sex` -> `sex...37`
## • `capital_gain` -> `capital_gain...38`
## • `capital_loss` -> `capital_loss...39`
## • `hours_per_week` -> `hours_per_week...40`
## • `native_country` -> `native_country...41`
## • `income` -> `income...42`
## • `age` -> `age...43`
## • `workclass` -> `workclass...44`
## • `education` -> `education...45`
## • `education_num` -> `education_num...46`
## • `marital_status` -> `marital_status...47`
## • `occupation` -> `occupation...48`
## • `relationship` -> `relationship...49`
## • `race` -> `race...50`
## • `sex` -> `sex...51`
## • `capital_gain` -> `capital_gain...52`
## • `capital_loss` -> `capital_loss...53`
## • `hours_per_week` -> `hours_per_week...54`
## • `native_country` -> `native_country...55`
## • `income` -> `income...56`
## • `age` -> `age...57`
## • `workclass` -> `workclass...58`
## • `education` -> `education...59`
## • `education_num` -> `education_num...60`
## • `marital_status` -> `marital_status...61`
## • `occupation` -> `occupation...62`
## • `relationship` -> `relationship...63`
## • `race` -> `race...64`
## • `sex` -> `sex...65`
## • `capital_gain` -> `capital_gain...66`
## • `capital_loss` -> `capital_loss...67`
## • `hours_per_week` -> `hours_per_week...68`
## • `native_country` -> `native_country...69`
## • `income` -> `income...70`
## • `age` -> `age...71`
## • `workclass` -> `workclass...72`
## • `education` -> `education...73`
## • `education_num` -> `education_num...74`
## • `marital_status` -> `marital_status...75`
## • `occupation` -> `occupation...76`
## • `relationship` -> `relationship...77`
## • `race` -> `race...78`
## • `sex` -> `sex...79`
## • `capital_gain` -> `capital_gain...80`
## • `capital_loss` -> `capital_loss...81`
## • `hours_per_week` -> `hours_per_week...82`
## • `native_country` -> `native_country...83`
## • `income` -> `income...84`mean_multiple <- c(mean(data_multiple$capital_gain.1),
mean(data_multiple$capital_gain.2),
mean(data_multiple$capital_gain.3),
mean(data_multiple$capital_gain.4),
mean(data_multiple$capital_gain.5))
sample_dirty_dataset$capital_gain[is.na(sample_dirty_dataset$capital_gain)]<-mean(mean_multiple)
#Mean to corect capital loss
data_multiple <- complete(mice(sample_dirty_dataset, m = 5), action = "broad", include = TRUE)##
## iter imp variable
## 1 1 age capital_loss hours_per_week
## 1 2 age capital_loss hours_per_week
## 1 3 age capital_loss hours_per_week
## 1 4 age capital_loss hours_per_week
## 1 5 age capital_loss hours_per_week
## 2 1 age capital_loss hours_per_week
## 2 2 age capital_loss hours_per_week
## 2 3 age capital_loss hours_per_week
## 2 4 age capital_loss hours_per_week
## 2 5 age capital_loss hours_per_week
## 3 1 age capital_loss hours_per_week
## 3 2 age capital_loss hours_per_week
## 3 3 age capital_loss hours_per_week
## 3 4 age capital_loss hours_per_week
## 3 5 age capital_loss hours_per_week
## 4 1 age capital_loss hours_per_week
## 4 2 age capital_loss hours_per_week
## 4 3 age capital_loss hours_per_week
## 4 4 age capital_loss hours_per_week
## 4 5 age capital_loss hours_per_week
## 5 1 age capital_loss hours_per_week
## 5 2 age capital_loss hours_per_week
## 5 3 age capital_loss hours_per_week
## 5 4 age capital_loss hours_per_week
## 5 5 age capital_loss hours_per_week## Warning: Number of logged events: 9## New names:
## • `age` -> `age...1`
## • `workclass` -> `workclass...2`
## • `education` -> `education...3`
## • `education_num` -> `education_num...4`
## • `marital_status` -> `marital_status...5`
## • `occupation` -> `occupation...6`
## • `relationship` -> `relationship...7`
## • `race` -> `race...8`
## • `sex` -> `sex...9`
## • `capital_gain` -> `capital_gain...10`
## • `capital_loss` -> `capital_loss...11`
## • `hours_per_week` -> `hours_per_week...12`
## • `native_country` -> `native_country...13`
## • `income` -> `income...14`
## • `age` -> `age...15`
## • `workclass` -> `workclass...16`
## • `education` -> `education...17`
## • `education_num` -> `education_num...18`
## • `marital_status` -> `marital_status...19`
## • `occupation` -> `occupation...20`
## • `relationship` -> `relationship...21`
## • `race` -> `race...22`
## • `sex` -> `sex...23`
## • `capital_gain` -> `capital_gain...24`
## • `capital_loss` -> `capital_loss...25`
## • `hours_per_week` -> `hours_per_week...26`
## • `native_country` -> `native_country...27`
## • `income` -> `income...28`
## • `age` -> `age...29`
## • `workclass` -> `workclass...30`
## • `education` -> `education...31`
## • `education_num` -> `education_num...32`
## • `marital_status` -> `marital_status...33`
## • `occupation` -> `occupation...34`
## • `relationship` -> `relationship...35`
## • `race` -> `race...36`
## • `sex` -> `sex...37`
## • `capital_gain` -> `capital_gain...38`
## • `capital_loss` -> `capital_loss...39`
## • `hours_per_week` -> `hours_per_week...40`
## • `native_country` -> `native_country...41`
## • `income` -> `income...42`
## • `age` -> `age...43`
## • `workclass` -> `workclass...44`
## • `education` -> `education...45`
## • `education_num` -> `education_num...46`
## • `marital_status` -> `marital_status...47`
## • `occupation` -> `occupation...48`
## • `relationship` -> `relationship...49`
## • `race` -> `race...50`
## • `sex` -> `sex...51`
## • `capital_gain` -> `capital_gain...52`
## • `capital_loss` -> `capital_loss...53`
## • `hours_per_week` -> `hours_per_week...54`
## • `native_country` -> `native_country...55`
## • `income` -> `income...56`
## • `age` -> `age...57`
## • `workclass` -> `workclass...58`
## • `education` -> `education...59`
## • `education_num` -> `education_num...60`
## • `marital_status` -> `marital_status...61`
## • `occupation` -> `occupation...62`
## • `relationship` -> `relationship...63`
## • `race` -> `race...64`
## • `sex` -> `sex...65`
## • `capital_gain` -> `capital_gain...66`
## • `capital_loss` -> `capital_loss...67`
## • `hours_per_week` -> `hours_per_week...68`
## • `native_country` -> `native_country...69`
## • `income` -> `income...70`
## • `age` -> `age...71`
## • `workclass` -> `workclass...72`
## • `education` -> `education...73`
## • `education_num` -> `education_num...74`
## • `marital_status` -> `marital_status...75`
## • `occupation` -> `occupation...76`
## • `relationship` -> `relationship...77`
## • `race` -> `race...78`
## • `sex` -> `sex...79`
## • `capital_gain` -> `capital_gain...80`
## • `capital_loss` -> `capital_loss...81`
## • `hours_per_week` -> `hours_per_week...82`
## • `native_country` -> `native_country...83`
## • `income` -> `income...84`mean_multiple <- c(mean(data_multiple$capital_loss.1),
mean(data_multiple$capital_loss.2),
mean(data_multiple$capital_loss.3),
mean(data_multiple$capital_loss.4),
mean(data_multiple$capital_loss.5))
sample_dirty_dataset$capital_loss[is.na(sample_dirty_dataset$capital_loss)]<-mean(mean_multiple)
#Mean to corect age
data_multiple <- complete(mice(sample_dirty_dataset, m = 5), action = "broad", include = TRUE)##
## iter imp variable
## 1 1 age hours_per_week
## 1 2 age hours_per_week
## 1 3 age hours_per_week
## 1 4 age hours_per_week
## 1 5 age hours_per_week
## 2 1 age hours_per_week
## 2 2 age hours_per_week
## 2 3 age hours_per_week
## 2 4 age hours_per_week
## 2 5 age hours_per_week
## 3 1 age hours_per_week
## 3 2 age hours_per_week
## 3 3 age hours_per_week
## 3 4 age hours_per_week
## 3 5 age hours_per_week
## 4 1 age hours_per_week
## 4 2 age hours_per_week
## 4 3 age hours_per_week
## 4 4 age hours_per_week
## 4 5 age hours_per_week
## 5 1 age hours_per_week
## 5 2 age hours_per_week
## 5 3 age hours_per_week
## 5 4 age hours_per_week
## 5 5 age hours_per_week## Warning: Number of logged events: 9## New names:
## • `age` -> `age...1`
## • `workclass` -> `workclass...2`
## • `education` -> `education...3`
## • `education_num` -> `education_num...4`
## • `marital_status` -> `marital_status...5`
## • `occupation` -> `occupation...6`
## • `relationship` -> `relationship...7`
## • `race` -> `race...8`
## • `sex` -> `sex...9`
## • `capital_gain` -> `capital_gain...10`
## • `capital_loss` -> `capital_loss...11`
## • `hours_per_week` -> `hours_per_week...12`
## • `native_country` -> `native_country...13`
## • `income` -> `income...14`
## • `age` -> `age...15`
## • `workclass` -> `workclass...16`
## • `education` -> `education...17`
## • `education_num` -> `education_num...18`
## • `marital_status` -> `marital_status...19`
## • `occupation` -> `occupation...20`
## • `relationship` -> `relationship...21`
## • `race` -> `race...22`
## • `sex` -> `sex...23`
## • `capital_gain` -> `capital_gain...24`
## • `capital_loss` -> `capital_loss...25`
## • `hours_per_week` -> `hours_per_week...26`
## • `native_country` -> `native_country...27`
## • `income` -> `income...28`
## • `age` -> `age...29`
## • `workclass` -> `workclass...30`
## • `education` -> `education...31`
## • `education_num` -> `education_num...32`
## • `marital_status` -> `marital_status...33`
## • `occupation` -> `occupation...34`
## • `relationship` -> `relationship...35`
## • `race` -> `race...36`
## • `sex` -> `sex...37`
## • `capital_gain` -> `capital_gain...38`
## • `capital_loss` -> `capital_loss...39`
## • `hours_per_week` -> `hours_per_week...40`
## • `native_country` -> `native_country...41`
## • `income` -> `income...42`
## • `age` -> `age...43`
## • `workclass` -> `workclass...44`
## • `education` -> `education...45`
## • `education_num` -> `education_num...46`
## • `marital_status` -> `marital_status...47`
## • `occupation` -> `occupation...48`
## • `relationship` -> `relationship...49`
## • `race` -> `race...50`
## • `sex` -> `sex...51`
## • `capital_gain` -> `capital_gain...52`
## • `capital_loss` -> `capital_loss...53`
## • `hours_per_week` -> `hours_per_week...54`
## • `native_country` -> `native_country...55`
## • `income` -> `income...56`
## • `age` -> `age...57`
## • `workclass` -> `workclass...58`
## • `education` -> `education...59`
## • `education_num` -> `education_num...60`
## • `marital_status` -> `marital_status...61`
## • `occupation` -> `occupation...62`
## • `relationship` -> `relationship...63`
## • `race` -> `race...64`
## • `sex` -> `sex...65`
## • `capital_gain` -> `capital_gain...66`
## • `capital_loss` -> `capital_loss...67`
## • `hours_per_week` -> `hours_per_week...68`
## • `native_country` -> `native_country...69`
## • `income` -> `income...70`
## • `age` -> `age...71`
## • `workclass` -> `workclass...72`
## • `education` -> `education...73`
## • `education_num` -> `education_num...74`
## • `marital_status` -> `marital_status...75`
## • `occupation` -> `occupation...76`
## • `relationship` -> `relationship...77`
## • `race` -> `race...78`
## • `sex` -> `sex...79`
## • `capital_gain` -> `capital_gain...80`
## • `capital_loss` -> `capital_loss...81`
## • `hours_per_week` -> `hours_per_week...82`
## • `native_country` -> `native_country...83`
## • `income` -> `income...84`mean_multiple <- c(mean(data_multiple$age.1),
mean(data_multiple$age.2),
mean(data_multiple$age.3),
mean(data_multiple$age.4),
mean(data_multiple$age.5))
sample_dirty_dataset$age[is.na(sample_dirty_dataset$age)]<-mean(mean_multiple)
#Mean to corect hours per week
data_multiple <- complete(mice(sample_dirty_dataset, m = 5), action = "broad", include = TRUE)##
## iter imp variable
## 1 1 hours_per_week
## 1 2 hours_per_week
## 1 3 hours_per_week
## 1 4 hours_per_week
## 1 5 hours_per_week
## 2 1 hours_per_week
## 2 2 hours_per_week
## 2 3 hours_per_week
## 2 4 hours_per_week
## 2 5 hours_per_week
## 3 1 hours_per_week
## 3 2 hours_per_week
## 3 3 hours_per_week
## 3 4 hours_per_week
## 3 5 hours_per_week
## 4 1 hours_per_week
## 4 2 hours_per_week
## 4 3 hours_per_week
## 4 4 hours_per_week
## 4 5 hours_per_week
## 5 1 hours_per_week
## 5 2 hours_per_week
## 5 3 hours_per_week
## 5 4 hours_per_week
## 5 5 hours_per_week## Warning: Number of logged events: 9## New names:
## • `age` -> `age...1`
## • `workclass` -> `workclass...2`
## • `education` -> `education...3`
## • `education_num` -> `education_num...4`
## • `marital_status` -> `marital_status...5`
## • `occupation` -> `occupation...6`
## • `relationship` -> `relationship...7`
## • `race` -> `race...8`
## • `sex` -> `sex...9`
## • `capital_gain` -> `capital_gain...10`
## • `capital_loss` -> `capital_loss...11`
## • `hours_per_week` -> `hours_per_week...12`
## • `native_country` -> `native_country...13`
## • `income` -> `income...14`
## • `age` -> `age...15`
## • `workclass` -> `workclass...16`
## • `education` -> `education...17`
## • `education_num` -> `education_num...18`
## • `marital_status` -> `marital_status...19`
## • `occupation` -> `occupation...20`
## • `relationship` -> `relationship...21`
## • `race` -> `race...22`
## • `sex` -> `sex...23`
## • `capital_gain` -> `capital_gain...24`
## • `capital_loss` -> `capital_loss...25`
## • `hours_per_week` -> `hours_per_week...26`
## • `native_country` -> `native_country...27`
## • `income` -> `income...28`
## • `age` -> `age...29`
## • `workclass` -> `workclass...30`
## • `education` -> `education...31`
## • `education_num` -> `education_num...32`
## • `marital_status` -> `marital_status...33`
## • `occupation` -> `occupation...34`
## • `relationship` -> `relationship...35`
## • `race` -> `race...36`
## • `sex` -> `sex...37`
## • `capital_gain` -> `capital_gain...38`
## • `capital_loss` -> `capital_loss...39`
## • `hours_per_week` -> `hours_per_week...40`
## • `native_country` -> `native_country...41`
## • `income` -> `income...42`
## • `age` -> `age...43`
## • `workclass` -> `workclass...44`
## • `education` -> `education...45`
## • `education_num` -> `education_num...46`
## • `marital_status` -> `marital_status...47`
## • `occupation` -> `occupation...48`
## • `relationship` -> `relationship...49`
## • `race` -> `race...50`
## • `sex` -> `sex...51`
## • `capital_gain` -> `capital_gain...52`
## • `capital_loss` -> `capital_loss...53`
## • `hours_per_week` -> `hours_per_week...54`
## • `native_country` -> `native_country...55`
## • `income` -> `income...56`
## • `age` -> `age...57`
## • `workclass` -> `workclass...58`
## • `education` -> `education...59`
## • `education_num` -> `education_num...60`
## • `marital_status` -> `marital_status...61`
## • `occupation` -> `occupation...62`
## • `relationship` -> `relationship...63`
## • `race` -> `race...64`
## • `sex` -> `sex...65`
## • `capital_gain` -> `capital_gain...66`
## • `capital_loss` -> `capital_loss...67`
## • `hours_per_week` -> `hours_per_week...68`
## • `native_country` -> `native_country...69`
## • `income` -> `income...70`
## • `age` -> `age...71`
## • `workclass` -> `workclass...72`
## • `education` -> `education...73`
## • `education_num` -> `education_num...74`
## • `marital_status` -> `marital_status...75`
## • `occupation` -> `occupation...76`
## • `relationship` -> `relationship...77`
## • `race` -> `race...78`
## • `sex` -> `sex...79`
## • `capital_gain` -> `capital_gain...80`
## • `capital_loss` -> `capital_loss...81`
## • `hours_per_week` -> `hours_per_week...82`
## • `native_country` -> `native_country...83`
## • `income` -> `income...84`mean_multiple <- c(mean(data_multiple$hours_per_week.1),
mean(data_multiple$hours_per_week.2),
mean(data_multiple$hours_per_week.3),
mean(data_multiple$hours_per_week.4),
mean(data_multiple$hours_per_week.5))
sample_dirty_dataset$hours_per_week[is.na(sample_dirty_dataset$hours_per_week)]<-mean(mean_multiple)
# Use summary() to see which variables have missing (NA) values
summary(sample_dirty_dataset)## age workclass education education_num
## Min. :18.00 Length:1000 Length:1000 Min. : 1.000
## 1st Qu.:39.00 Class :character Class :character 1st Qu.: 5.000
## Median :55.18 Mode :character Mode :character Median : 8.000
## Mean :55.20 Mean : 8.267
## 3rd Qu.:72.00 3rd Qu.:12.000
## Max. :90.00 Max. :16.000
## marital_status occupation relationship race
## Length:1000 Length:1000 Length:1000 Length:1000
## Class :character Class :character Class :character Class :character
## Mode :character Mode :character Mode :character Mode :character
##
##
##
## sex capital_gain capital_loss hours_per_week
## Length:1000 Min. : 133 Min. : 11 Min. : 1.00
## Class :character 1st Qu.:25630 1st Qu.:27828 1st Qu.:26.00
## Mode :character Median :49320 Median :50674 Median :49.26
## Mean :49273 Mean :50639 Mean :49.19
## 3rd Qu.:71237 3rd Qu.:72606 3rd Qu.:71.00
## Max. :99890 Max. :99784 Max. :99.00
## native_country income
## Length:1000 Length:1000
## Class :character Class :character
## Mode :character Mode :character
##
##
## diagnose(sample_dirty_dataset)## # A tibble: 14 × 6
## variables types missing_count missing_percent unique_count unique_rate
## <chr> <chr> <int> <dbl> <int> <dbl>
## 1 age numeric 0 0 74 0.074
## 2 workclass charac… 50 5 9 0.009
## 3 education charac… 50 5 17 0.017
## 4 education_num integer 0 0 16 0.016
## 5 marital_status charac… 50 5 8 0.008
## 6 occupation charac… 0 0 15 0.015
## 7 relationship charac… 50 5 7 0.007
## 8 race charac… 50 5 6 0.006
## 9 sex charac… 50 5 3 0.003
## 10 capital_gain numeric 0 0 944 0.944
## 11 capital_loss numeric 0 0 951 0.951
## 12 hours_per_week numeric 0 0 100 0.1
## 13 native_country charac… 50 5 42 0.042
## 14 income charac… 0 0 2 0.002# Number of rows in full dataset
nrow(sample_dirty_dataset)## [1] 1000clean_dataset <- na.omit(sample_dirty_dataset)
summary(clean_dataset)## age workclass education education_num
## Min. :18.00 Length:698 Length:698 Min. : 1.000
## 1st Qu.:39.00 Class :character Class :character 1st Qu.: 4.250
## Median :55.18 Mode :character Mode :character Median : 8.000
## Mean :55.00 Mean : 8.248
## 3rd Qu.:71.75 3rd Qu.:12.000
## Max. :90.00 Max. :16.000
## marital_status occupation relationship race
## Length:698 Length:698 Length:698 Length:698
## Class :character Class :character Class :character Class :character
## Mode :character Mode :character Mode :character Mode :character
##
##
##
## sex capital_gain capital_loss hours_per_week
## Length:698 Min. : 133 Min. : 402 Min. : 1.00
## Class :character 1st Qu.:25404 1st Qu.:27529 1st Qu.:26.25
## Mode :character Median :49320 Median :50674 Median :49.26
## Mean :49136 Mean :51001 Mean :49.76
## 3rd Qu.:71109 3rd Qu.:74721 3rd Qu.:72.00
## Max. :99890 Max. :99723 Max. :99.00
## native_country income
## Length:698 Length:698
## Class :character Class :character
## Mode :character Mode :character
##
##
## nrow(clean_dataset)## [1] 698###OUTLINERS TASK ### ZSCORE OUTLIERS CAPITAL LOSS
zscore <- abs(scale(clean_dataset$capital_loss))
plot(zscore, type = "n", ylab = "Z-score")
abline(h = 2, col = "blue")
text(1:length(clean_dataset$capital_loss), zscore, labels = round(zscore, 2))
title(main = "Capital loss")
### BOX PLOT AGE
boxplot(clean_dataset$age)
boxplot(clean_dataset$age)$out
## numeric(0)OUTLIERS HOURS PER WEEK
library(dlookr)
library(mice)
library(robustbase)##
## Attaching package: 'robustbase'## The following object is masked _by_ '.GlobalEnv':
##
## education### OUTLIERS HOURS PER WEEK
par(mfrow=c(2,2))
plot(covMcd(clean_dataset$hours_per_week))
library(dlookr)
# Standardization
summary(clean_dataset)## age workclass education education_num
## Min. :18.00 Length:698 Length:698 Min. : 1.000
## 1st Qu.:39.00 Class :character Class :character 1st Qu.: 4.250
## Median :55.18 Mode :character Mode :character Median : 8.000
## Mean :55.00 Mean : 8.248
## 3rd Qu.:71.75 3rd Qu.:12.000
## Max. :90.00 Max. :16.000
## marital_status occupation relationship race
## Length:698 Length:698 Length:698 Length:698
## Class :character Class :character Class :character Class :character
## Mode :character Mode :character Mode :character Mode :character
##
##
##
## sex capital_gain capital_loss hours_per_week
## Length:698 Min. : 133 Min. : 402 Min. : 1.00
## Class :character 1st Qu.:25404 1st Qu.:27529 1st Qu.:26.25
## Mode :character Median :49320 Median :50674 Median :49.26
## Mean :49136 Mean :51001 Mean :49.76
## 3rd Qu.:71109 3rd Qu.:74721 3rd Qu.:72.00
## Max. :99890 Max. :99723 Max. :99.00
## native_country income
## Length:698 Length:698
## Class :character Class :character
## Mode :character Mode :character
##
##
## #We think that these variables doesn't need transformations
head(sample_dirty_dataset)## age workclass education education_num marital_status
## 1 82 Federal-gov Doctorate 9 Divorced
## 2 56 Self-emp-inc 7th-8th 11 Divorced
## 3 44 Private Some-college 8 Never-married
## 4 22 Federal-gov 7th-8th 5 Married-AF-spouse
## 5 23 Self-emp-not-inc Assoc-acdm 15 Widowed
## 6 62 Self-emp-not-inc 5th-6th 6 Divorced
## occupation relationship race sex capital_gain
## 1 Sales Not-in-family <NA> Female 49320.47
## 2 Tech-support Husband Amer-Indian-Eskimo Male 51355.00
## 3 Adm-clerical Unmarried <NA> Male 32620.00
## 4 Machine-op-inspct Wife White Female 18335.00
## 5 Priv-house-serv Wife Amer-Indian-Eskimo Male 3143.00
## 6 Sales Husband Other <NA> 49465.00
## capital_loss hours_per_week native_country income
## 1 68148 66 <NA> <=50K
## 2 86057 18 Poland <=50K
## 3 56436 20 Taiwan <=50K
## 4 60665 78 Haiti <=50K
## 5 58774 31 Cuba <=50K
## 6 43755 30 Cambodia >50K###Task 10
#diagnose_web_report(clean_dataset)
#diagnose_web_report(sample_dirty_dataset)
#eda_web_report(clean_dataset)
#transformation_web_report(clean_dataset)#Sumary:
#Generate data
#We performed Data Cleaning / missing values / correccting errors
#we performed Analysis