# Set seed for reproducibility
set.seed(10)
# Generate sample data
n <- 1000 # Number of observations
# Age between 18 and 90
age <- sample(18:90, n, replace = TRUE)
# Introduce errors in age column
age[sample(1:n, 50)] <- "InvalidAge"
# Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
# Introduce missing values in workclass column
workclass[sample(1:n, 50)] <- NA
# Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
# Introduce inconsistencies in education column
education[sample(1:n, 50)] <- "InvalidEducation"
# Education Number: 1-16
education_num <- sample(1:16, n, replace = TRUE)
# Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
# Introduce missing values in marital_status column
marital_status[sample(1:n, 50)] <- NA
# Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
# Introduce errors in occupation column
occupation[sample(1:n, 50)] <- "InvalidOccupation"
# Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
# Introduce missing values in relationship column
relationship[sample(1:n, 50)] <- NA
# Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
# Introduce inconsistencies in race column
race[sample(1:n, 50)] <- "InvalidRace"
# Sex: Female, Male
sex <- sample(c("Female", "Male"), n, replace = TRUE)
# Introduce missing values in sex column
sex[sample(1:n, 50)] <- NA
# Capital Gain: 0-99999
capital_gain <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_gain column
capital_gain[sample(1:n, 50)] <- -999
# Capital Loss: 0-99999
capital_loss <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_loss column
capital_loss[sample(1:n, 50)] <- -999
# Hours per week: 1-99
hours_per_week <- sample(1:99, n, replace = TRUE)
# Introduce missing values in hours_per_week column
hours_per_week[sample(1:n, 50)] <- NA
# Native country: United-States, Cambodia, England, ...
native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
# Introduce missing values in native_country column
native_country[sample(1:n, 50)] <- NA
# Income: <=50K, >50K
income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
# Create dataframe
sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)
# Print first few rows of the dataset
head(sample_dirty_dataset)## age workclass education education_num marital_status
## 1 26 Federal-gov Assoc-acdm 2 Separated
## 2 72 Private HS-grad 16 Married-civ-spouse
## 3 89 Federal-gov HS-grad 14 Married-spouse-absent
## 4 71 Without-pay 12th 7 <NA>
## 5 56 Without-pay 5th-6th 5 Separated
## 6 32 Self-emp-not-inc Assoc-acdm 4 Married-AF-spouse
## occupation relationship race sex capital_gain
## 1 Craft-repair Not-in-family Other Female 95729
## 2 Machine-op-inspct Unmarried Asian-Pac-Islander Female 16548
## 3 Armed-Forces Not-in-family Black Male 26865
## 4 Machine-op-inspct Wife Asian-Pac-Islander Female 80193
## 5 Priv-house-serv Other-relative Other Male 97828
## 6 InvalidOccupation Husband Black Female 76074
## capital_loss hours_per_week native_country income
## 1 78233 71 Laos <=50K
## 2 91089 81 Italy <=50K
## 3 5355 51 Ecuador >50K
## 4 32043 64 Taiwan >50K
## 5 17804 47 Italy <=50K
## 6 28165 31 China <=50K# Count mean of hours_per_week and change NA into this value
mean_hours_per_week <- mean(sample_dirty_dataset$hours_per_week, na.rm = TRUE)
mean_hours_per_week <- round(mean_hours_per_week,digits = 0)
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(hours_per_week = ifelse(is.na(hours_per_week), mean_hours_per_week, hours_per_week))
# Count mean of age and change NA into this value
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(age = ifelse(as.character(age) == "InvalidAge", NA_integer_, as.integer(age))) %>%
mutate(age = ifelse(is.na(age), mean(age, na.rm = TRUE), age))## Warning: There was 1 warning in `mutate()`.
## ℹ In argument: `age = ifelse(as.character(age) == "InvalidAge", NA_integer_,
## as.integer(age))`.
## Caused by warning in `ifelse()`:
## ! NAs introduced by coercionmean_age <- mean(sample_dirty_dataset$age, na.rm = TRUE)
mean_age <- round(mean_age, digits = 0)
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(age = ifelse(is.na(age), as.integer(mean_age), as.integer(age)))
# Change InvalidEducation into NA
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(education = ifelse(as.character(education) == "InvalidEducation", NA_character_, as.character(education)))
# Change InvalidOccupation into NA
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(occupation = ifelse(occupation == "InvalidOccupation", NA_character_, occupation))
# Change InvalidRace into NA
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(race = ifelse(occupation == "InvalidRace", NA_character_, race))
# Deleting dataset from observations that contains NA
clean_sample_dataset <- na.omit(sample_dirty_dataset)Data Cleaning Steps:
Imputing Missing Values:
For the ‘hours_per_week’ column, missing values were replaced with the mean value of the column. For the ‘age’ column, invalid values (e.g., “InvalidAge”) were first converted to NA, then missing values were imputed with the mean age.
Handling Invalid Entries:
‘InvalidEducation’, ‘InvalidOccupation’, and ‘InvalidRace’ entries were converted to NA in their respective columns.
Removing Rows with Missing Values. Insights Gained:
The dataset is more consistant, without errors, and missing values, making it more reliable for analysis. By examining the dataset before and after cleaning, insights about errors and missing data can be gained. Analysis on the cleaned dataset can provide more accurate and trustworthy results. Detecting and rectifying errors can save significant amounts of time. It prevents incorrect conclusions or potential reworks.
# Summary statistics before cleaning
summary(sample_dirty_dataset)## age workclass education education_num
## Min. :18.00 Length:1000 Length:1000 Min. : 1.000
## 1st Qu.:36.00 Class :character Class :character 1st Qu.: 5.000
## Median :53.00 Mode :character Mode :character Median : 9.000
## Mean :53.78 Mean : 8.697
## 3rd Qu.:71.00 3rd Qu.:13.000
## Max. :90.00 Max. :16.000
## marital_status occupation relationship race
## Length:1000 Length:1000 Length:1000 Length:1000
## Class :character Class :character Class :character Class :character
## Mode :character Mode :character Mode :character Mode :character
##
##
##
## sex capital_gain capital_loss hours_per_week
## Length:1000 Min. : -999 Min. : -999 Min. : 1.00
## Class :character 1st Qu.:21208 1st Qu.:21453 1st Qu.:25.00
## Mode :character Median :45534 Median :46901 Median :50.00
## Mean :47103 Mean :47848 Mean :49.76
## 3rd Qu.:74322 3rd Qu.:74949 3rd Qu.:76.00
## Max. :99692 Max. :99804 Max. :99.00
## native_country income
## Length:1000 Length:1000
## Class :character Class :character
## Mode :character Mode :character
##
##
## # Summary statistics after cleaning
summary(clean_sample_dataset)## age workclass education education_num
## Min. :18.00 Length:701 Length:701 Min. : 1.000
## 1st Qu.:36.00 Class :character Class :character 1st Qu.: 5.000
## Median :53.00 Mode :character Mode :character Median : 9.000
## Mean :54.19 Mean : 8.716
## 3rd Qu.:73.00 3rd Qu.:13.000
## Max. :90.00 Max. :16.000
## marital_status occupation relationship race
## Length:701 Length:701 Length:701 Length:701
## Class :character Class :character Class :character Class :character
## Mode :character Mode :character Mode :character Mode :character
##
##
##
## sex capital_gain capital_loss hours_per_week
## Length:701 Min. : -999 Min. : -999 Min. : 1.00
## Class :character 1st Qu.:20206 1st Qu.:20268 1st Qu.:26.00
## Mode :character Median :44187 Median :45594 Median :50.00
## Mean :46160 Mean :47093 Mean :50.23
## 3rd Qu.:72953 3rd Qu.:75199 3rd Qu.:77.00
## Max. :99692 Max. :99727 Max. :99.00
## native_country income
## Length:701 Length:701
## Class :character Class :character
## Mode :character Mode :character
##
##
## # Plotting age distribution before and after cleaning
par(mfrow=c(1,2))
hist(sample_dirty_dataset$age, main="Age Distribution Before Cleaning", xlab="Age", col="skyblue", border="white")
hist(clean_sample_dataset$age, main="Age Distribution After Cleaning", xlab="Age", col="red", border="white")
# Plotting income distribution before and after cleaning
par(mfrow=c(1,2))
barplot(table(sample_dirty_dataset$income), main="Income Distribution Before Cleaning", xlab="Income", col="skyblue")
barplot(table(clean_sample_dataset$income), main="Income Distribution After Cleaning", xlab="Income", col="red")
# As we can see, the results do not differ too much, which suggests that the data cleansing was successful # (Reports had to be commented to knit the RMarkdown)
# Diagnostic Report
#diagnose_web_report(sample_dirty_dataset)
# EDA Report
# eda_web_report(sample_dirty_dataset)
# Transformation Paged Report
# transformation_paged_report(sample_dirty_dataset, target = "age")
# Last one dosen't work I don't know why.