# Load necessary libraries
library(tidyverse)
#;-) ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
#;-) ✔ dplyr 1.1.4 ✔ readr 2.1.5
#;-) ✔ forcats 1.0.0 ✔ stringr 1.5.1
#;-) ✔ ggplot2 3.4.4 ✔ tibble 3.2.1
#;-) ✔ lubridate 1.9.3 ✔ tidyr 1.3.1
#;-) ✔ purrr 1.0.2
#;-) ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
#;-) ✖ dplyr::filter() masks stats::filter()
#;-) ✖ dplyr::lag() masks stats::lag()
#;-) ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(dlookr)
#;-) Warning: pakiet 'dlookr' został zbudowany w wersji R 4.3.3
#;-) Registered S3 methods overwritten by 'dlookr':
#;-) method from
#;-) plot.transform scales
#;-) print.transform scales
#;-)
#;-) Dołączanie pakietu: 'dlookr'
#;-)
#;-) Następujący obiekt został zakryty z 'package:tidyr':
#;-)
#;-) extract
#;-)
#;-) Następujący obiekt został zakryty z 'package:base':
#;-)
#;-) transform
# Set seed for reproducibility
set.seed(1273)
# Generate sample data
n <- 1000 # Number of observations
# Age between 18 and 90
age <- sample(18:90, n, replace = TRUE)
# Introduce errors in age column
age[sample(1:n, 50)] <- "InvalidAge"
# Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
# Introduce missing values in workclass column
workclass[sample(1:n, 50)] <- NA
# Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
# Introduce inconsistencies in education column
education[sample(1:n, 50)] <- "InvalidEducation"
# Education Number: 1-16
education_num <- sample(1:16, n, replace = TRUE)
# Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
# Introduce missing values in marital_status column
marital_status[sample(1:n, 50)] <- NA
# Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
# Introduce errors in occupation column
occupation[sample(1:n, 50)] <- "InvalidOccupation"
# Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
# Introduce missing values in relationship column
relationship[sample(1:n, 50)] <- NA
# Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
# Introduce inconsistencies in race column
race[sample(1:n, 50)] <- "InvalidRace"
# Sex: Female, Male
sex <- sample(c("Female", "Male"), n, replace = TRUE)
# Introduce missing values in sex column
sex[sample(1:n, 50)] <- NA
# Capital Gain: 0-99999
capital_gain <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_gain column
capital_gain[sample(1:n, 50)] <- -999
# Capital Loss: 0-99999
capital_loss <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_loss column
capital_loss[sample(1:n, 50)] <- -999
# Hours per week: 1-99
hours_per_week <- sample(1:99, n, replace = TRUE)
# Introduce missing values in hours_per_week column
hours_per_week[sample(1:n, 50)] <- NA
# Native country: United-States, Cambodia, England, ...
native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
# Introduce missing values in native_country column
native_country[sample(1:n, 50)] <- NA
# Income: <=50K, >50K
income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
# Create dataframe
sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)
# Print first few rows of the dataset
head(sample_dirty_dataset)
#;-) age workclass education education_num marital_status occupation
#;-) 1 75 Self-emp-inc Assoc-acdm 11 Married-civ-spouse Adm-clerical
#;-) 2 52 Federal-gov 9th 6 Widowed Transport-moving
#;-) 3 38 State-gov HS-grad 3 Divorced Tech-support
#;-) 4 46 State-gov Assoc-voc 5 Widowed Farming-fishing
#;-) 5 80 <NA> 11th 4 <NA> Transport-moving
#;-) 6 18 Local-gov Preschool 14 Never-married Other-service
#;-) relationship race sex capital_gain capital_loss
#;-) 1 Other-relative Other Female 3174 50010
#;-) 2 Wife Amer-Indian-Eskimo Female 58902 63469
#;-) 3 Own-child White Male 5289 12089
#;-) 4 Husband Amer-Indian-Eskimo Male 49393 1591
#;-) 5 Husband Other <NA> 55337 88144
#;-) 6 Own-child Other Male 40036 10540
#;-) hours_per_week native_country income
#;-) 1 98 Honduras <=50K
#;-) 2 30 <NA> >50K
#;-) 3 63 Puerto-Rico >50K
#;-) 4 92 Scotland <=50K
#;-) 5 78 Jamaica <=50K
#;-) 6 28 Cuba <=50K
summary(sample_dirty_dataset)
#;-) age workclass education education_num
#;-) Length:1000 Length:1000 Length:1000 Min. : 1.00
#;-) Class :character Class :character Class :character 1st Qu.: 4.00
#;-) Mode :character Mode :character Mode :character Median : 8.00
#;-) Mean : 8.38
#;-) 3rd Qu.:12.25
#;-) Max. :16.00
#;-)
#;-) marital_status occupation relationship race
#;-) Length:1000 Length:1000 Length:1000 Length:1000
#;-) Class :character Class :character Class :character Class :character
#;-) Mode :character Mode :character Mode :character Mode :character
#;-)
#;-)
#;-)
#;-)
#;-) sex capital_gain capital_loss hours_per_week
#;-) Length:1000 Min. : -999 Min. : -999 Min. : 1.00
#;-) Class :character 1st Qu.:22162 1st Qu.:18133 1st Qu.:25.00
#;-) Mode :character Median :47386 Median :46415 Median :47.00
#;-) Mean :47727 Mean :46295 Mean :48.38
#;-) 3rd Qu.:75284 3rd Qu.:72687 3rd Qu.:73.00
#;-) Max. :99681 Max. :99976 Max. :99.00
#;-) NA's :50
#;-) native_country income
#;-) Length:1000 Length:1000
#;-) Class :character Class :character
#;-) Mode :character Mode :character
#;-)
#;-)
#;-)
#;-)
sample_dirty_dataset$age[sample_dirty_dataset$age == "InvalidAge"] <- NA
sample_dirty_dataset$capital_gain[sample_dirty_dataset$capital_gain < 0] <- NA
sample_dirty_dataset$capital_loss[sample_dirty_dataset$capital_loss < 0] <- NA
sample_dirty_dataset$occupation[sample_dirty_dataset$occupation == "InvalidOccupation"] <- NA
sample_dirty_dataset$race[sample_dirty_dataset$race == "InvalidRace"] <- NA
sample_dirty_dataset$education[sample_dirty_dataset$education == "InvalidEducation"] <- NA
head(sample_dirty_dataset)
#;-) age workclass education education_num marital_status occupation
#;-) 1 75 Self-emp-inc Assoc-acdm 11 Married-civ-spouse Adm-clerical
#;-) 2 52 Federal-gov 9th 6 Widowed Transport-moving
#;-) 3 38 State-gov HS-grad 3 Divorced Tech-support
#;-) 4 46 State-gov Assoc-voc 5 Widowed Farming-fishing
#;-) 5 80 <NA> 11th 4 <NA> Transport-moving
#;-) 6 18 Local-gov Preschool 14 Never-married Other-service
#;-) relationship race sex capital_gain capital_loss
#;-) 1 Other-relative Other Female 3174 50010
#;-) 2 Wife Amer-Indian-Eskimo Female 58902 63469
#;-) 3 Own-child White Male 5289 12089
#;-) 4 Husband Amer-Indian-Eskimo Male 49393 1591
#;-) 5 Husband Other <NA> 55337 88144
#;-) 6 Own-child Other Male 40036 10540
#;-) hours_per_week native_country income
#;-) 1 98 Honduras <=50K
#;-) 2 30 <NA> >50K
#;-) 3 63 Puerto-Rico >50K
#;-) 4 92 Scotland <=50K
#;-) 5 78 Jamaica <=50K
#;-) 6 28 Cuba <=50Ksample_dirty_dataset$age<-as.numeric(sample_dirty_dataset$age)
library(mice)
#;-) Warning: pakiet 'mice' został zbudowany w wersji R 4.3.3
#;-) Warning in check_dep_version(): ABI version mismatch:
#;-) lme4 was built with Matrix ABI version 1
#;-) Current Matrix ABI version is 0
#;-) Please re-install lme4 from source or restore original 'Matrix' package
#;-)
#;-) Dołączanie pakietu: 'mice'
#;-) Następujący obiekt został zakryty z 'package:stats':
#;-)
#;-) filter
#;-) Następujące obiekty zostały zakryte z 'package:base':
#;-)
#;-) cbind, rbind
library(dlookr)
library(tidyverse)
imp <- mice(sample_dirty_dataset %>% select(age, capital_gain, capital_loss), m = 5)
#;-)
#;-) iter imp variable
#;-) 1 1 age capital_gain capital_loss
#;-) 1 2 age capital_gain capital_loss
#;-) 1 3 age capital_gain capital_loss
#;-) 1 4 age capital_gain capital_loss
#;-) 1 5 age capital_gain capital_loss
#;-) 2 1 age capital_gain capital_loss
#;-) 2 2 age capital_gain capital_loss
#;-) 2 3 age capital_gain capital_loss
#;-) 2 4 age capital_gain capital_loss
#;-) 2 5 age capital_gain capital_loss
#;-) 3 1 age capital_gain capital_loss
#;-) 3 2 age capital_gain capital_loss
#;-) 3 3 age capital_gain capital_loss
#;-) 3 4 age capital_gain capital_loss
#;-) 3 5 age capital_gain capital_loss
#;-) 4 1 age capital_gain capital_loss
#;-) 4 2 age capital_gain capital_loss
#;-) 4 3 age capital_gain capital_loss
#;-) 4 4 age capital_gain capital_loss
#;-) 4 5 age capital_gain capital_loss
#;-) 5 1 age capital_gain capital_loss
#;-) 5 2 age capital_gain capital_loss
#;-) 5 3 age capital_gain capital_loss
#;-) 5 4 age capital_gain capital_loss
#;-) 5 5 age capital_gain capital_loss
completed_data <- complete(imp)
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(
age = completed_data$age,
capital_gain = completed_data$capital_gain,
capital_loss = completed_data$capital_loss
)
sample_dirty_dataset <- sample_dirty_dataset %>% drop_na()
head(sample_dirty_dataset)
#;-) age workclass education education_num marital_status occupation
#;-) 1 75 Self-emp-inc Assoc-acdm 11 Married-civ-spouse Adm-clerical
#;-) 2 38 State-gov HS-grad 3 Divorced Tech-support
#;-) 3 46 State-gov Assoc-voc 5 Widowed Farming-fishing
#;-) 4 18 Local-gov Preschool 14 Never-married Other-service
#;-) 5 69 Never-worked 10th 14 Married-AF-spouse Protective-serv
#;-) 6 61 Never-worked Assoc-acdm 3 Married-civ-spouse Priv-house-serv
#;-) relationship race sex capital_gain capital_loss
#;-) 1 Other-relative Other Female 3174 50010
#;-) 2 Own-child White Male 5289 12089
#;-) 3 Husband Amer-Indian-Eskimo Male 49393 1591
#;-) 4 Own-child Other Male 40036 10540
#;-) 5 Wife Asian-Pac-Islander Female 5832 93479
#;-) 6 Not-in-family White Female 71686 70135
#;-) hours_per_week native_country income
#;-) 1 98 Honduras <=50K
#;-) 2 63 Puerto-Rico >50K
#;-) 3 92 Scotland <=50K
#;-) 4 28 Cuba <=50K
#;-) 5 76 Puerto-Rico <=50K
#;-) 6 30 France <=50Klibrary(ggplot2)
boxplot(sample_dirty_dataset$capital_gain)
boxplot(sample_dirty_dataset$capital_gain)$out
#;-) numeric(0)
# No Outlierslibrary(rrcov)
#;-) Warning: pakiet 'rrcov' został zbudowany w wersji R 4.3.3
#;-) Ładowanie wymaganego pakietu: robustbase
#;-) Warning: pakiet 'robustbase' został zbudowany w wersji R 4.3.3
#;-)
#;-) Dołączanie pakietu: 'robustbase'
#;-) Następujący obiekt został zakryty _przez_ '.GlobalEnv':
#;-)
#;-) education
#;-) Scalable Robust Estimators with High Breakdown Point (version 1.7-5)
par(mfrow=c(2,2))
plot(covMcd(sample_dirty_dataset$capital_loss))
zscore<-abs(scale(sample_dirty_dataset$age))
plot(zscore, type = "l", main = "Z-scores for Age", xlab = "Observation", ylab = "Z-score", col = "blue")
abline(h = 2, col = "red")
diagnose_web_report(sample_dirty_dataset)
#;-) processing file: diagnosis_temp.Rmd
#;-) output file: diagnosis_temp.knit.md
#;-) "C:/Program Files/RStudio/resources/app/bin/quarto/bin/tools/pandoc" +RTS -K512m -RTS diagnosis_temp.knit.md --to html4 --from markdown+autolink_bare_uris+tex_math_single_backslash --output pandoc40903b74598c.html --lua-filter "C:\Users\M\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\pagebreak.lua" --lua-filter "C:\Users\M\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\latex-div.lua" --embed-resources --standalone --variable bs3=TRUE --section-divs --template "C:\Users\M\AppData\Local\R\win-library\4.3\rmarkdown\rmd\h\default.html" --no-highlight --variable highlightjs=1 --variable theme=bootstrap --css "C:/Users/M/AppData/Local/R/win-library/4.3/dlookr/resources/dlookr-bootstrap.css" --mathjax --variable "mathjax-url=https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" --include-in-header "C:\Users\M\AppData\Local\Temp\RtmpKQ6dN6\rmarkdown-str40902335714.html" --variable code_folding=show --variable code_menu=1 --include-in-header header_temp.html --include-after-body "C:\Users\M\AppData\Local\R\win-library\4.3\dlookr\resources\footer.html"
#;-)
#;-) Output created: C:\Users\M\AppData\Local\Temp\RtmpKQ6dN6/Diagnosis_Report.htmleda_web_report(sample_dirty_dataset)
#;-) processing file: eda_temp.Rmd
#;-) Error in parse_block(g[-1], g[1], params.src, markdown_mode): Duplicate chunk label 'unnamed-chunk-2', which has been used for the chunk:
#;-)
#;-) sample_dirty_dataset$age[sample_dirty_dataset$age == "InvalidAge"] <- NA
#;-) sample_dirty_dataset$capital_gain[sample_dirty_dataset$capital_gain < 0] <- NA
#;-) sample_dirty_dataset$capital_loss[sample_dirty_dataset$capital_loss < 0] <- NA
#;-) sample_dirty_dataset$occupation[sample_dirty_dataset$occupation == "InvalidOccupation"] <- NA
#;-) sample_dirty_dataset$race[sample_dirty_dataset$race == "InvalidRace"] <- NA
#;-) sample_dirty_dataset$education[sample_dirty_dataset$education == "InvalidEducation"] <- NA
#;-) head(sample_dirty_dataset)
#transformation_paged_report(sample_dirty_dataset)