Study group:
str(sample_dirty_dataset)
#;-) 'data.frame': 1000 obs. of 14 variables:
#;-) $ age : chr "59" "48" "83" "32" ...
#;-) $ workclass : chr "Without-pay" "State-gov" "State-gov" "Never-worked" ...
#;-) $ education : chr "10th" "Assoc-acdm" "Bachelors" "Prof-school" ...
#;-) $ education_num : int 15 6 1 16 9 9 3 4 15 12 ...
#;-) $ marital_status: chr "Separated" "Married-civ-spouse" "Married-civ-spouse" "Married-AF-spouse" ...
#;-) $ occupation : chr "Priv-house-serv" "Machine-op-inspct" "Priv-house-serv" "Transport-moving" ...
#;-) $ relationship : chr "Wife" "Other-relative" NA NA ...
#;-) $ race : chr "Black" "Amer-Indian-Eskimo" "Black" "Asian-Pac-Islander" ...
#;-) $ sex : chr "Male" "Female" "Male" "Female" ...
#;-) $ capital_gain : num 35050 -999 84587 24715 16599 ...
#;-) $ capital_loss : num 96892 20459 76544 92227 -999 ...
#;-) $ hours_per_week: int 34 40 52 8 85 43 92 94 NA 44 ...
#;-) $ native_country: chr "Poland" "Iran" "Poland" "Vietnam" ...
#;-) $ income : chr "<=50K" "<=50K" ">50K" "<=50K" ...summary(sample_dirty_dataset)
#;-) age workclass education education_num
#;-) Length:1000 Length:1000 Length:1000 Min. : 1.000
#;-) Class :character Class :character Class :character 1st Qu.: 5.000
#;-) Mode :character Mode :character Mode :character Median : 8.000
#;-) Mean : 8.459
#;-) 3rd Qu.:13.000
#;-) Max. :16.000
#;-)
#;-) marital_status occupation relationship race
#;-) Length:1000 Length:1000 Length:1000 Length:1000
#;-) Class :character Class :character Class :character Class :character
#;-) Mode :character Mode :character Mode :character Mode :character
#;-)
#;-)
#;-)
#;-)
#;-) sex capital_gain capital_loss hours_per_week
#;-) Length:1000 Min. : -999 Min. : -999 Min. : 1.0
#;-) Class :character 1st Qu.:21115 1st Qu.:23962 1st Qu.:24.0
#;-) Mode :character Median :46646 Median :49625 Median :49.0
#;-) Mean :46891 Mean :48749 Mean :49.4
#;-) 3rd Qu.:72898 3rd Qu.:73750 3rd Qu.:75.0
#;-) Max. :99935 Max. :99772 Max. :99.0
#;-) NA's :50
#;-) native_country income
#;-) Length:1000 Length:1000
#;-) Class :character Class :character
#;-) Mode :character Mode :character
#;-)
#;-)
#;-)
#;-) # Correct errors in the Dataset
# 1. Invalid ages ("InvalidAge")
sample_dirty_dataset$age <- ifelse(sample_dirty_dataset$age == "InvalidAge", NA, sample_dirty_dataset$age)
# cheange to numerical
sample_dirty_dataset$age <- as.numeric(as.character(sample_dirty_dataset$age))
# 2. Negative capital gains or losses
sample_dirty_dataset$capital_gain <- ifelse(sample_dirty_dataset$capital_gain <= 0, NA, sample_dirty_dataset$capital_gain)
sample_dirty_dataset$capital_loss <- ifelse(sample_dirty_dataset$capital_loss <= 0, NA, sample_dirty_dataset$capital_loss)
# 3. Incorrectly spelled or invalid values in categorical variables
sample_dirty_dataset$occupation <- ifelse(sample_dirty_dataset$occupation == "InvalidOccupation", NA, sample_dirty_dataset$occupation)
sample_dirty_dataset$race <- ifelse(sample_dirty_dataset$race == "InvalidRace", NA, sample_dirty_dataset$race)
sample_dirty_dataset$education <- ifelse(sample_dirty_dataset$education == "InvalidEducation", NA, sample_dirty_dataset$education)# Load the Dataset
data <- sample_dirty_dataset
# Detect errors and inconsistencies using dlookr library
data_report <- diagnose(data)For the first report
sample_dirty_dataset %>%
diagnose_web_report(subtitle = "report1", output_dir = ".", output_file = "Diagn.html", theme = "blue")
#;-) processing file: diagnosis_temp.Rmd
#;-) output file: diagnosis_temp.knit.md
#;-) /Applications/RStudio.app/Contents/Resources/app/quarto/bin/tools/pandoc +RTS -K512m -RTS diagnosis_temp.knit.md --to html4 --from markdown+autolink_bare_uris+tex_math_single_backslash --output Diagn.html --lua-filter /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/rmarkdown/rmarkdown/lua/pagebreak.lua --lua-filter /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/rmarkdown/rmarkdown/lua/latex-div.lua --embed-resources --standalone --variable bs3=TRUE --section-divs --template /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/rmarkdown/rmd/h/default.html --no-highlight --variable highlightjs=1 --variable theme=bootstrap --css /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/dlookr/resources/dlookr-bootstrap.css --mathjax --variable 'mathjax-url=https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML' --include-in-header /var/folders/99/w3lm9dwd2q929kjsdm9dyy7c0000gn/T//RtmpQWNM3j/rmarkdown-str141d14a92a47f.html --variable code_folding=show --variable code_menu=1 --include-in-header header_temp.html --include-after-body /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/dlookr/resources/footer.html
#;-)
#;-) Output created: Diagn.html
# When there are numerical missing values we can simply use meadian (not mean becouse every value in dataser is rounded to 0, simple mean would give us N.01234) out of the other data to populate missing cells
sample_dirty_dataset$age[is.na(sample_dirty_dataset$age)] <- median(sample_dirty_dataset$age, na.rm = TRUE) # Age
sample_dirty_dataset$capital_gain[is.na(sample_dirty_dataset$capital_gain)] <- median(sample_dirty_dataset$capital_gain, na.rm = TRUE) # Capital gain
sample_dirty_dataset$capital_loss[is.na(sample_dirty_dataset$capital_loss)] <- median(sample_dirty_dataset$capital_loss, na.rm = TRUE) # Capital loss
sample_dirty_dataset$hours_per_week[is.na(sample_dirty_dataset$hours_per_week)] <- median(sample_dirty_dataset$hours_per_week, na.rm = TRUE) # Capital loss
# Now the rest being string values for simplicity lets just remove them
sample_dirty_dataset <- na.omit(sample_dirty_dataset)
sample_dirty_dataset %>%
eda_web_report(target = "capital_loss", subtitle = "capital_loss", output_dir = ".", output_file = "Report1_EDA.html", theme = "blue")
#;-) processing file: eda_temp.Rmd
#;-) Error in parse_block(g[-1], g[1], params.src, markdown_mode): Duplicate chunk label 'unnamed-chunk-3', which has been used for the chunk:
#;-) str(sample_dirty_dataset)
# For example, assuming 'capital_loss' is the problematic column:
sample_dirty_dataset$capital_loss <- as.numeric(sample_dirty_dataset$capital_loss) # Convert to numeric if not already
sample_dirty_dataset <- sample_dirty_dataset[is.finite(sample_dirty_dataset$capital_loss), ] # Keep only rows with finite values
# this does not work and it should, but it doesn't
# sample_dirty_dataset %>%
# transformation_paged_report(target = "capital_loss", subtitle = "occupation", output_dir = ".", output_file = "Report1_trans.pdf", theme = "blue")sessionInfo()
#;-) R version 4.2.1 (2022-06-23)
#;-) Platform: aarch64-apple-darwin20 (64-bit)
#;-) Running under: macOS 14.3
#;-)
#;-) Matrix products: default
#;-) BLAS: /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/lib/libRblas.0.dylib
#;-) LAPACK: /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/lib/libRlapack.dylib
#;-)
#;-) locale:
#;-) [1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
#;-)
#;-) attached base packages:
#;-) [1] stats graphics grDevices utils datasets methods base
#;-)
#;-) other attached packages:
#;-) [1] htmltools_0.5.7 reactable_0.4.4 kableExtra_1.4.0 naniar_1.1.0
#;-) [5] mice_3.16.0 dlookr_0.6.3 lubridate_1.9.3 forcats_1.0.0
#;-) [9] stringr_1.5.1 dplyr_1.1.4 purrr_1.0.2 readr_2.1.5
#;-) [13] tidyr_1.3.1 tibble_3.2.1 ggplot2_3.5.0 tidyverse_2.0.0
#;-)
#;-) loaded via a namespace (and not attached):
#;-) [1] minqa_1.2.6 colorspace_2.0-3 ellipsis_0.3.2
#;-) [4] visdat_0.6.0 showtext_0.9-7 base64enc_0.1-3
#;-) [7] fs_1.6.3 httpcode_0.3.0 rstudioapi_0.15.0
#;-) [10] showtextdb_3.0 farver_2.1.1 fansi_1.0.3
#;-) [13] xml2_1.3.6 codetools_0.2-18 splines_4.2.1
#;-) [16] extrafont_0.19 cachem_1.0.8 knitr_1.45
#;-) [19] jsonlite_1.8.8 nloptr_2.0.3 broom_1.0.5
#;-) [22] Rttf2pt1_1.3.12 shiny_1.8.1.1 compiler_4.2.1
#;-) [25] backports_1.4.1 Matrix_1.4-1 fastmap_1.1.1
#;-) [28] cli_3.6.2 later_1.3.2 hrbrthemes_0.8.7
#;-) [31] tools_4.2.1 gtable_0.3.0 glue_1.6.2
#;-) [34] Rcpp_1.0.12 jquerylib_0.1.4 fontquiver_0.2.1
#;-) [37] vctrs_0.6.5 crul_1.4.0 svglite_2.1.3
#;-) [40] nlme_3.1-157 extrafontdb_1.0 iterators_1.0.14
#;-) [43] xfun_0.42 lme4_1.1-35.2 timechange_0.3.0
#;-) [46] mime_0.12 lifecycle_1.0.4 pan_1.9
#;-) [49] MASS_7.3-57 scales_1.3.0 hms_1.1.3
#;-) [52] promises_1.2.1 fontLiberation_0.1.0 yaml_2.3.8
#;-) [55] curl_5.2.1 gridExtra_2.3 gdtools_0.3.7
#;-) [58] sass_0.4.8 rpart_4.1.16 stringi_1.7.8
#;-) [61] fontBitstreamVera_0.1.1 highr_0.10 foreach_1.5.2
#;-) [64] boot_1.3-28 shape_1.4.6.1 rlang_1.1.3
#;-) [67] pkgconfig_2.0.3 systemfonts_1.0.6 evaluate_0.23
#;-) [70] lattice_0.20-45 fontawesome_0.5.2 htmlwidgets_1.6.4
#;-) [73] labeling_0.4.2 tidyselect_1.2.1 magrittr_2.0.3
#;-) [76] R6_2.5.1 generics_0.1.3 mitml_0.4-5
#;-) [79] pillar_1.9.0 withr_2.5.0 survival_3.3-1
#;-) [82] nnet_7.3-17 crayon_1.5.2 gfonts_0.2.0
#;-) [85] jomo_2.7-6 utf8_1.2.2 tzdb_0.4.0
#;-) [88] rmarkdown_2.25 sysfonts_0.8.9 grid_4.2.1
#;-) [91] reactR_0.5.0 pagedown_0.20 reprex_2.1.0
#;-) [94] digest_0.6.35 xtable_1.8-4 httpuv_1.6.15
#;-) [97] munsell_0.5.0 glmnet_4.1-8 viridisLite_0.4.0
#;-) [100] bslib_0.6.1