# Generate sample data
n <- 1000 # Number of observations
# Age between 18 and 90
age <- sample(18:90, n, replace = TRUE)
# Introduce errors in age column
age[sample(1:n, 50)] <- "InvalidAge"
# Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
# Introduce missing values in workclass column
workclass[sample(1:n, 50)] <- NA
# Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
# Introduce inconsistencies in education column
education[sample(1:n, 50)] <- "InvalidEducation"
# Education Number: 1-16
education_num <- sample(1:16, n, replace = TRUE)
# Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
# Introduce missing values in marital_status column
marital_status[sample(1:n, 50)] <- NA
# Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
# Introduce errors in occupation column
occupation[sample(1:n, 50)] <- "InvalidOccupation"
# Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
# Introduce missing values in relationship column
relationship[sample(1:n, 50)] <- NA
# Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
# Introduce inconsistencies in race column
race[sample(1:n, 50)] <- "InvalidRace"
# Sex: Female, Male
sex <- sample(c("Female", "Male"), n, replace = TRUE)
# Introduce missing values in sex column
sex[sample(1:n, 50)] <- NA
# Capital Gain: 0-99999
capital_gain <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_gain column
capital_gain[sample(1:n, 50)] <- -999
# Capital Loss: 0-99999
capital_loss <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_loss column
capital_loss[sample(1:n, 50)] <- -999
# Hours per week: 1-99
hours_per_week <- sample(1:99, n, replace = TRUE)
# Introduce missing values in hours_per_week column
hours_per_week[sample(1:n, 50)] <- NA
# Native country: United-States, Cambodia, England, ...
native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
# Introduce missing values in native_country column
native_country[sample(1:n, 50)] <- NA
# Income: <=50K, >50K
income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
# Create dataframe
sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)# Print first few rows of the dataset
head(sample_dirty_dataset)## age workclass education education_num marital_status
## 1 41 Self-emp-not-inc 7th-8th 5 Married-civ-spouse
## 2 18 <NA> Bachelors 2 Separated
## 3 77 State-gov Bachelors 6 Widowed
## 4 79 Self-emp-not-inc Assoc-voc 7 Widowed
## 5 29 Self-emp-inc Assoc-acdm 11 Married-AF-spouse
## 6 27 Without-pay Bachelors 15 Never-married
## occupation relationship race sex capital_gain
## 1 Handlers-cleaners Husband Amer-Indian-Eskimo Male 73933
## 2 Priv-house-serv Other-relative Amer-Indian-Eskimo Male 71604
## 3 Handlers-cleaners Husband Amer-Indian-Eskimo Male 8053
## 4 Machine-op-inspct Own-child Black Female 23425
## 5 InvalidOccupation Other-relative Asian-Pac-Islander Female 13494
## 6 Protective-serv Other-relative Amer-Indian-Eskimo Female 36865
## capital_loss hours_per_week native_country income
## 1 88732 11 Holand-Netherlands >50K
## 2 39423 65 Scotland <=50K
## 3 61667 23 Ecuador <=50K
## 4 42428 66 <NA> <=50K
## 5 28937 48 Holand-Netherlands <=50K
## 6 25581 23 Germany <=50Kdiagnose(sample_dirty_dataset)## # A tibble: 14 × 6
## variables types missing_count missing_percent unique_count unique_rate
## <chr> <chr> <int> <dbl> <int> <dbl>
## 1 age charac… 0 0 74 0.074
## 2 workclass charac… 50 5 9 0.009
## 3 education charac… 0 0 17 0.017
## 4 education_num integer 0 0 16 0.016
## 5 marital_status charac… 50 5 8 0.008
## 6 occupation charac… 0 0 15 0.015
## 7 relationship charac… 50 5 7 0.007
## 8 race charac… 0 0 6 0.006
## 9 sex charac… 50 5 3 0.003
## 10 capital_gain numeric 0 0 946 0.946
## 11 capital_loss numeric 0 0 947 0.947
## 12 hours_per_week integer 50 5 100 0.1
## 13 native_country charac… 50 5 42 0.042
## 14 income charac… 0 0 2 0.002sample_dirty_dataset %>%
diagnose() %>%
select(-unique_count, -unique_rate) %>%
filter(missing_count > 0) %>%
arrange(desc(missing_count))## # A tibble: 6 × 4
## variables types missing_count missing_percent
## <chr> <chr> <int> <dbl>
## 1 workclass character 50 5
## 2 marital_status character 50 5
## 3 relationship character 50 5
## 4 sex character 50 5
## 5 hours_per_week integer 50 5
## 6 native_country character 50 5diagnose_numeric(sample_dirty_dataset)## variables min Q1 mean median Q3 max zero minus
## 1 education_num 1 4.00 8.43500 8 13.00 16 0 0
## 2 capital_gain -999 21115.25 47448.76400 47408 74185.75 99693 0 50
## 3 capital_loss -999 20014.00 47397.87600 47389 74612.25 99978 0 50
## 4 hours_per_week 1 24.00 49.39789 49 74.75 99 0 0
## outlier
## 1 0
## 2 0
## 3 0
## 4 0diagnose_outlier(sample_dirty_dataset)## variables outliers_cnt outliers_ratio outliers_mean with_mean
## 1 education_num 0 0 NaN 8.43500
## 2 capital_gain 0 0 NaN 47448.76400
## 3 capital_loss 0 0 NaN 47397.87600
## 4 hours_per_week 0 0 NaN 49.39789
## without_mean
## 1 8.43500
## 2 47448.76400
## 3 47397.87600
## 4 49.39789There are no outliers.
# Identify rows with "InvalidAge"
invalid_age_rows <- which(sample_dirty_dataset$age == "InvalidAge")
# Generate new ages for replacement
new_ages <- sample(18:90, length(invalid_age_rows), replace = TRUE)
# Replace "InvalidAge" with new ages
sample_dirty_dataset$age[invalid_age_rows] <- new_ages
# Identify rows with "InvalidEducation"
invalid_education_rows <- which(sample_dirty_dataset$education == "InvalidEducation")
# Generate new education for replacement
new_education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), length(invalid_education_rows), replace = TRUE)
# Replace "InvalidEducation" with new education
sample_dirty_dataset$education[invalid_education_rows] <- new_education
# Identify rows with "InvalidOccupation"
invalid_occupation_rows <- which(sample_dirty_dataset$occupation == "InvalidOccupation")
# Generate new occupation for replacement
new_occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), length(invalid_occupation_rows), replace = TRUE)
# Replace "InvalidOccupation" with new occupation
sample_dirty_dataset$occupation[invalid_occupation_rows] <- new_occupation
# Identify rows with "InvalidRace"
invalid_race_rows <- which(sample_dirty_dataset$race == "InvalidRace")
# Generate new race for replacement
new_race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), length(invalid_race_rows), replace = TRUE)
# Replace "InvalidRace" with new race
sample_dirty_dataset$race[invalid_race_rows] <- new_race
# Identify rows with negative capital_gain
invalid_gain_rows <- which(sample_dirty_dataset$capital_gain <0)
# Generate new gain for replacement
new_gain <- sample(0:99999, length(invalid_gain_rows), replace = TRUE)
# Replace negative capital_gain with new gain
sample_dirty_dataset$capital_gain[invalid_gain_rows] <- new_gain
# Identify rows with negative capital_loss
invalid_loss_rows <- which(sample_dirty_dataset$capital_loss <0)
# Generate new loss for replacement
new_loss <- sample(0:99999, length(invalid_loss_rows), replace = TRUE)
# Replace negative capital_loss with new loss
sample_dirty_dataset$capital_loss[invalid_loss_rows] <- new_loss
#removing NA entries
clean_dataset <- na.omit(sample_dirty_dataset)# Print updated dataset
head(clean_dataset)## age workclass education education_num marital_status
## 1 41 Self-emp-not-inc 7th-8th 5 Married-civ-spouse
## 3 77 State-gov Bachelors 6 Widowed
## 5 29 Self-emp-inc Assoc-acdm 11 Married-AF-spouse
## 6 27 Without-pay Bachelors 15 Never-married
## 7 42 Never-worked 7th-8th 10 Married-AF-spouse
## 8 26 Local-gov Assoc-voc 1 Separated
## occupation relationship race sex capital_gain
## 1 Handlers-cleaners Husband Amer-Indian-Eskimo Male 73933
## 3 Handlers-cleaners Husband Amer-Indian-Eskimo Male 8053
## 5 Sales Other-relative Asian-Pac-Islander Female 13494
## 6 Protective-serv Other-relative Amer-Indian-Eskimo Female 36865
## 7 Armed-Forces Not-in-family Amer-Indian-Eskimo Female 62910
## 8 Farming-fishing Other-relative White Female 89640
## capital_loss hours_per_week native_country income
## 1 88732 11 Holand-Netherlands >50K
## 3 61667 23 Ecuador <=50K
## 5 28937 48 Holand-Netherlands <=50K
## 6 25581 23 Germany <=50K
## 7 46573 21 Mexico <=50K
## 8 34015 58 Peru <=50Kdiagnose(clean_dataset)## # A tibble: 14 × 6
## variables types missing_count missing_percent unique_count unique_rate
## <chr> <chr> <int> <dbl> <int> <dbl>
## 1 age charac… 0 0 73 0.0999
## 2 workclass charac… 0 0 8 0.0109
## 3 education charac… 0 0 16 0.0219
## 4 education_num integer 0 0 16 0.0219
## 5 marital_status charac… 0 0 7 0.00958
## 6 occupation charac… 0 0 14 0.0192
## 7 relationship charac… 0 0 6 0.00821
## 8 race charac… 0 0 5 0.00684
## 9 sex charac… 0 0 2 0.00274
## 10 capital_gain numeric 0 0 727 0.995
## 11 capital_loss numeric 0 0 727 0.995
## 12 hours_per_week integer 0 0 99 0.135
## 13 native_country charac… 0 0 41 0.0561
## 14 income charac… 0 0 2 0.00274clean_dataset %>%
diagnose() %>%
select(-unique_count, -unique_rate) %>%
filter(missing_count > 0) %>%
arrange(desc(missing_count))## # A tibble: 0 × 4
## # ℹ 4 variables: variables <chr>, types <chr>, missing_count <int>,
## # missing_percent <dbl>There are no missing values anymore.
diagnose_numeric(clean_dataset)## variables min Q1 mean median Q3 max zero minus
## 1 education_num 1 4.0 8.329685 8 13.0 16 0 0
## 2 capital_gain 121 25523.5 50188.094391 49906 75176.0 99622 0 0
## 3 capital_loss 57 23890.0 50957.154583 50947 77675.5 99978 0 0
## 4 hours_per_week 1 23.0 49.004104 48 73.5 99 0 0
## outlier
## 1 0
## 2 0
## 3 0
## 4 0diagnose_outlier(clean_dataset)## variables outliers_cnt outliers_ratio outliers_mean with_mean
## 1 education_num 0 0 NaN 8.329685
## 2 capital_gain 0 0 NaN 50188.094391
## 3 capital_loss 0 0 NaN 50957.154583
## 4 hours_per_week 0 0 NaN 49.004104
## without_mean
## 1 8.329685
## 2 50188.094391
## 3 50957.154583
## 4 49.004104There are no outliers still.
#diagnose_web_report(sample_dirty_dataset)
#eda_web_report(sample_dirty_dataset)
#transformation_web_report(sample_dirty_dataset)