Report 2
Sample data
## Warning: pakiet 'tidyr' został zbudowany w wersji R 4.3.3## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr 1.1.4 ✔ readr 2.1.5
## ✔ forcats 1.0.0 ✔ stringr 1.5.1
## ✔ ggplot2 3.5.0 ✔ tibble 3.2.1
## ✔ lubridate 1.9.3 ✔ tidyr 1.3.1
## ✔ purrr 1.0.2
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors# Set seed for reproducibility
set.seed(123)
# Generate sample data
n <- 1000 # Number of observations
# Age between 18 and 90
age <- sample(18:90, n, replace = TRUE)
# Introduce errors in age column
age[sample(1:n, 50)] <- "InvalidAge"
# Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
# Introduce missing values in workclass column
workclass[sample(1:n, 50)] <- NA
# Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
# Introduce inconsistencies in education column
education[sample(1:n, 50)] <- "InvalidEducation"
# Education Number: 1-16
education_num <- sample(1:16, n, replace = TRUE)
# Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
# Introduce missing values in marital_status column
marital_status[sample(1:n, 50)] <- NA
# Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
# Introduce errors in occupation column
occupation[sample(1:n, 50)] <- "InvalidOccupation"
# Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
# Introduce missing values in relationship column
relationship[sample(1:n, 50)] <- NA
# Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
# Introduce inconsistencies in race column
race[sample(1:n, 50)] <- "InvalidRace"
# Sex: Female, Male
sex <- sample(c("Female", "Male"), n, replace = TRUE)
# Introduce missing values in sex column
sex[sample(1:n, 50)] <- NA
# Capital Gain: 0-99999
capital_gain <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_gain column
capital_gain[sample(1:n, 50)] <- -999
# Capital Loss: 0-99999
capital_loss <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_loss column
capital_loss[sample(1:n, 50)] <- -999
# Hours per week: 1-99
hours_per_week <- sample(1:99, n, replace = TRUE)
# Introduce missing values in hours_per_week column
hours_per_week[sample(1:n, 50)] <- NA
# Native country: United-States, Cambodia, England, ...
native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
# Introduce missing values in native_country column
native_country[sample(1:n, 50)] <- NA
# Income: <=50K, >50K
income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
# Create dataframe
sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)
# Print first few rows of the dataset
head(sample_dirty_dataset)## age workclass education education_num marital_status
## 1 48 Local-gov 7th-8th 1 <NA>
## 2 68 <NA> Prof-school 1 Divorced
## 3 31 State-gov Bachelors 2 Married-spouse-absent
## 4 84 Never-worked 7th-8th 10 Separated
## 5 59 Private 12th 16 Married-civ-spouse
## 6 67 Self-emp-not-inc InvalidEducation 12 Married-spouse-absent
## occupation relationship race sex capital_gain
## 1 Prof-specialty Other-relative Black Male 7826
## 2 Priv-house-serv Unmarried Asian-Pac-Islander Female 84532
## 3 Protective-serv Wife Black Male 35297
## 4 Handlers-cleaners Other-relative Asian-Pac-Islander Male 27074
## 5 Machine-op-inspct Other-relative White Female 73785
## 6 Adm-clerical Husband Black Male 2553
## capital_loss hours_per_week native_country income
## 1 77249 60 Poland >50K
## 2 60981 73 <NA> <=50K
## 3 15093 98 Hungary <=50K
## 4 40203 NA South <=50K
## 5 2056 4 Haiti <=50K
## 6 17031 70 Scotland <=50K## Warning: pakiet 'mice' został zbudowany w wersji R 4.3.3## Warning in check_dep_version(): ABI version mismatch:
## lme4 was built with Matrix ABI version 1
## Current Matrix ABI version is 0
## Please re-install lme4 from source or restore original 'Matrix' package##
## Dołączanie pakietu: 'mice'## Następujący obiekt został zakryty z 'package:stats':
##
## filter## Następujące obiekty zostały zakryte z 'package:base':
##
## cbind, rbind## Warning: pakiet 'dlookr' został zbudowany w wersji R 4.3.3## Registered S3 methods overwritten by 'dlookr':
## method from
## plot.transform scales
## print.transform scales##
## Dołączanie pakietu: 'dlookr'## Następujący obiekt został zakryty z 'package:tidyr':
##
## extract## Następujący obiekt został zakryty z 'package:base':
##
## transform## Warning: pakiet 'naniar' został zbudowany w wersji R 4.3.3##
## Dołączanie pakietu: 'magrittr'## Następujący obiekt został zakryty z 'package:dlookr':
##
## extract## Następujący obiekt został zakryty z 'package:purrr':
##
## set_names## Następujący obiekt został zakryty z 'package:tidyr':
##
## extract## Warning: pakiet 'rrcov' został zbudowany w wersji R 4.3.3## Ładowanie wymaganego pakietu: robustbase## Warning: pakiet 'robustbase' został zbudowany w wersji R 4.3.3##
## Dołączanie pakietu: 'robustbase'## Następujący obiekt został zakryty _przez_ '.GlobalEnv':
##
## education## Scalable Robust Estimators with High Breakdown Point (version 1.7-5)Handling inconsistencies
Errors to NAs
sample_dirty_dataset <- replace_with_na(sample_dirty_dataset, replace = list(occupation="InvalidOccupation", age = "InvalidAge", race = "InvalidRace" , education = "InvalidEducation"))sample_dirty_dataset$capital_gain <- as.character(sample_dirty_dataset$capital_gain)
sample_dirty_dataset$capital_loss <- as.character(sample_dirty_dataset$capital_loss)
sample_dirty_dataset$age <- as.numeric(sample_dirty_dataset$age)
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(
capital_gain = ifelse(capital_gain=="-999", NA, capital_gain),
capital_loss = ifelse(capital_loss=="-999", NA, capital_loss),
hours_per_week = ifelse(hours_per_week<0, NA, hours_per_week)
)
sample_dirty_dataset$capital_gain <- as.numeric(sample_dirty_dataset$capital_gain)
sample_dirty_dataset$capital_loss <- as.numeric(sample_dirty_dataset$capital_loss)
head(sample_dirty_dataset)## age workclass education education_num marital_status
## 1 48 Local-gov 7th-8th 1 <NA>
## 2 68 <NA> Prof-school 1 Divorced
## 3 31 State-gov Bachelors 2 Married-spouse-absent
## 4 84 Never-worked 7th-8th 10 Separated
## 5 59 Private <NA> 16 Married-civ-spouse
## 6 67 Self-emp-not-inc <NA> 12 Married-spouse-absent
## occupation relationship race sex capital_gain
## 1 Prof-specialty Other-relative Black Male 7826
## 2 Priv-house-serv Unmarried Asian-Pac-Islander Female 84532
## 3 Protective-serv Wife Black Male 35297
## 4 Handlers-cleaners Other-relative Asian-Pac-Islander Male 27074
## 5 Machine-op-inspct Other-relative White Female 73785
## 6 Adm-clerical Husband Black Male 2553
## capital_loss hours_per_week native_country income
## 1 77249 60 Poland >50K
## 2 60981 73 <NA> <=50K
## 3 15093 98 Hungary <=50K
## 4 40203 NA South <=50K
## 5 2056 4 Haiti <=50K
## 6 17031 70 Scotland <=50KHandling NAs
## count
## 1 834## [1] 60 98 132 145 163 167 179 202 213 230 256 261
## [13] 287 290 306 309 310 323 352 366 374 384 407 424
## [25] 467 468 477 484 492 504 529 583 589 603 608 611
## [37] 645 659 683 706 737 751 752 857 862 882 912 915
## [49] 925 933 1002 1037 1087 1108 1133 1179 1184 1192 1224 1253
## [61] 1275 1289 1320 1340 1367 1369 1374 1456 1470 1473 1497 1504
## [73] 1520 1543 1569 1585 1666 1675 1677 1686 1695 1728 1744 1768
## [85] 1773 1780 1781 1785 1818 1820 1827 1828 1829 1858 1863 1899
## [97] 1905 1949 1974 1993 2005 2006 2009 2019 2024 2025 2028 2031
## [109] 2032 2033 2035 2037 2040 2043 2047 2054 2056 2064 2065 2066
## [121] 2067 2069 2078 2079 2089 2092 2093 2099 2102 2106 2108 2109
## [133] 2113 2115 2116 2118 2119 2120 2121 2124 2129 2131 2134 2137
## [145] 2138 2144 2146 2148 2152 2164 2166 2169 2171 2173 2183 2187
## [157] 2188 2190 2197 2199 2203 2205 2206 2215 2217 2222 2223 2227
## [169] 2229 2237 2248 2251 2254 2260 2261 2268 2269 2291 2293 2295
## [181] 2305 2307 2308 2312 2313 2315 2316 2319 2323 2326 2334 2344
## [193] 2345 2348 2350 2351 2355 2356 2361 2362 2369 2373 2385 2386
## [205] 2387 2396 2397 2400 2410 2412 2418 2420 2421 2422 2429 2435
## [217] 2436 2441 2452 2454 2456 2457 2458 2459 2463 2466 2467 2471
## [229] 2477 2480 2489 2490 2492 2494 2496 2504 2506 2509 2514 2519
## [241] 2520 2523 2530 2533 2535 2536 2539 2542 2544 2546 2559 2566
## [253] 2568 2569 2570 2576 2577 2579 2581 2582 2583 2584 2586 2593
## [265] 2603 2611 2613 2616 2617 2620 2624 2625 2633 2647 2654 2656
## [277] 2658 2659 2660 2663 2667 2669 2677 2678 2680 2681 2684 2691
## [289] 2692 2694 2696 2697 2702 2703 2708 2710 2712 2713 2716 2717
## [301] 2719 2726 2729 2732 2737 2738 2742 2744 2746 2750 2752 2762
## [313] 2768 2770 2773 2779 2783 2784 2785 2788 2789 2794 2795 2799
## [325] 2800 2801 2804 2805 2807 2808 2811 2813 2819 2822 2827 2830
## [337] 2831 2839 2840 2843 2844 2846 2848 2851 2859 2866 2872 2875
## [349] 2880 2883 2885 2886 2892 2894 2896 2900 2905 2913 2920 2925
## [361] 2926 2931 2936 2939 2940 2941 2942 2945 2946 2954 2959 2960
## [373] 2972 2974 2975 2979 2980 2983 2984 2990 2991 2993 2998 2999
## [385] 4001 4016 4022 4038 4044 4056 4057 4115 4141 4149 4160 4189
## [397] 4249 4273 4291 4292 4293 4346 4361 4365 4366 4387 4465 4466
## [409] 4522 4524 4533 4534 4554 4574 4576 4589 4591 4600 4610 4691
## [421] 4711 4724 4730 4747 4781 4821 4894 4907 4922 4931 4943 4963
## [433] 4964 4990 5009 5017 5036 5044 5082 5100 5125 5155 5156 5161
## [445] 5172 5192 5194 5200 5208 5214 5240 5252 5256 5260 5267 5275
## [457] 5292 5298 5305 5370 5463 5469 5507 5528 5559 5573 5611 5639
## [469] 5732 5753 5760 5764 5799 5824 5828 5842 5853 5920 5924 5949
## [481] 5955 5957 5982 5990 6028 6075 6079 6100 6135 6139 6164 6172
## [493] 6190 6245 6248 6266 6268 6277 6288 6310 6315 6336 6338 6341
## [505] 6391 6439 6444 6484 6517 6537 6625 6633 6636 6657 6677 6691
## [517] 6720 6726 6727 6754 6774 6782 6786 6788 6796 6801 6831 6835
## [529] 6836 6885 6919 6938 6943 6994 7013 7014 7030 7033 7103 7110
## [541] 7117 7135 7145 7163 7178 7244 7272 7277 7283 7294 7298 7312
## [553] 7323 7324 7380 7395 7461 7488 7491 7513 7522 7541 7564 7595
## [565] 7604 7619 7635 7709 7711 7713 7714 7739 7754 7787 7789 7823
## [577] 7837 7862 7883 7924 7964 7973 7980 7986 8031 8074 8078 8099
## [589] 8123 8138 8173 8174 8217 8224 8291 8324 8330 8342 8360 8382
## [601] 8410 8416 8419 8440 8466 8475 8476 8549 8580 8624 8635 8640
## [613] 8666 8684 8737 8744 8750 8758 8791 8814 8822 8841 8853 8868
## [625] 8876 8877 8913 8918 8925 8973 8975 8978 8986 9000 9038 9098
## [637] 9115 9122 9129 9169 9176 9192 9214 9270 9278 9289 9328 9346
## [649] 9385 9391 9417 9422 9424 9431 9438 9457 9466 9472 9500 9508
## [661] 9538 9568 9571 9574 9600 9601 9607 9657 9678 9744 9758 9777
## [673] 9804 9814 9833 9881 9915 9919 9925 9946 9947 9948 9956 9958
## [685] 10060 10133 10136 10147 10157 10171 10186 10217 10230 10233 10249 10259
## [697] 10268 10285 10299 10308 10329 10332 10366 10373 10375 10378 10393 10400
## [709] 10457 10481 10531 10551 10619 10657 10660 10662 10684 10708 10728 10734
## [721] 10752 10763 10768 10770 10789 10806 10817 10836 10848 10867 10924 10925
## [733] 10952 10958 11004 11018 11022 11037 11116 11123 11129 11149 11169 11170
## [745] 11177 11185 11203 11222 11228 11235 11245 11321 11340 11375 11385 11412
## [757] 11483 11502 11532 11549 11573 11579 11608 11629 11639 11669 11685 11704
## [769] 11723 11730 11738 11742 11745 11791 11796 11816 11832 11855 11870 11876
## [781] 11917 11943 11983 11999 12002 12014 12020 12037 12039 12053 12082 12095
## [793] 12119 12179 12192 12194 12198 12202 12233 12249 12250 12252 12271 12274
## [805] 12345 12354 12362 12366 12367 12446 12466 12508 12511 12571 12580 12604
## [817] 12636 12642 12697 12698 12743 12771 12791 12792 12800 12803 12871 12880
## [829] 12881 12935 12946 12976 12987 12996#Imputing NAs of numerical types with the average value for the type
sample_dirty_dataset$age[is.na(sample_dirty_dataset$age)] <- as.integer(mean(sample_dirty_dataset$age, na.rm = TRUE))
sample_dirty_dataset$capital_gain[is.na(sample_dirty_dataset$capital_gain)] <- (mean(sample_dirty_dataset$capital_gain, na.rm = TRUE))
sample_dirty_dataset$capital_loss[is.na(sample_dirty_dataset$capital_loss)] <- (mean(sample_dirty_dataset$capital_loss, na.rm = TRUE))
#Removing non-numerical rows with NAs
sample_dirty_dataset <- sample_dirty_dataset %>% drop_na()Outliers
# IQR方法检测和处理age的异常值Outliers
age_IQR <- IQR(sample_dirty_dataset$age, na.rm = TRUE)
age_Q1 <- quantile(sample_dirty_dataset$age, 0.25, na.rm = TRUE)
age_Q3 <- quantile(sample_dirty_dataset$age, 0.75, na.rm = TRUE)
age_lower_bound <- age_Q1 - 1.5 * age_IQR
age_upper_bound <- age_Q3 + 1.5 * age_IQR
sample_dirty_dataset <- sample_dirty_dataset %>%
filter(age >= age_lower_bound & age <= age_upper_bound)
# Z-score方法检测capital_gain的异常值Outliers
capital_gain_z <- scale(sample_dirty_dataset$capital_gain)
capital_gain_outliers <- abs(capital_gain_z) > 2
# 将capital_gain的异常值Outliers替换为中位数
median_capital_gain <- median(sample_dirty_dataset$capital_gain, na.rm = TRUE)
sample_dirty_dataset$capital_gain[capital_gain_outliers] <- median_capital_gain
########################
# 1. Boxplot method
boxplot(sample_dirty_dataset$age)$out## numeric(0)
## integer(0)zscore <- abs(scale(sample_dirty_dataset$education_num))
plot(zscore, type = "n")
abline(h=2, col="red")
text(1:length(sample_dirty_dataset$education_num), zscore)
Summarization and Reports
## 'data.frame': 471 obs. of 14 variables:
## $ age : num 31 60 31 86 74 26 24 49 26 44 ...
## $ workclass : chr "State-gov" "Never-worked" "Self-emp-not-inc" "Local-gov" ...
## $ education : chr "Bachelors" "Prof-school" "5th-6th" "Masters" ...
## $ education_num : int 2 11 13 12 12 16 15 11 5 5 ...
## $ marital_status: chr "Married-spouse-absent" "Married-spouse-absent" "Married-AF-spouse" "Widowed" ...
## $ occupation : chr "Protective-serv" "Adm-clerical" "Handlers-cleaners" "Prof-specialty" ...
## $ relationship : chr "Wife" "Other-relative" "Husband" "Not-in-family" ...
## $ race : chr "Black" "Other" "Asian-Pac-Islander" "Asian-Pac-Islander" ...
## $ sex : chr "Male" "Female" "Male" "Female" ...
## $ capital_gain : num 35297 19616 91705 21279 74141 ...
## $ capital_loss : num 15093 51961 46300 52751 75593 ...
## $ hours_per_week: int 98 15 96 11 35 81 88 70 50 61 ...
## $ native_country: chr "Hungary" "Cambodia" "Philippines" "Thailand" ...
## $ income : chr "<=50K" "<=50K" "<=50K" "<=50K" ...## age workclass education education_num
## Min. :18.00 Length:471 Length:471 Min. : 1.000
## 1st Qu.:37.50 Class :character Class :character 1st Qu.: 4.000
## Median :53.00 Mode :character Mode :character Median : 9.000
## Mean :54.04 Mean : 8.616
## 3rd Qu.:69.50 3rd Qu.:13.000
## Max. :90.00 Max. :16.000
## marital_status occupation relationship race
## Length:471 Length:471 Length:471 Length:471
## Class :character Class :character Class :character Class :character
## Mode :character Mode :character Mode :character Mode :character
##
##
##
## sex capital_gain capital_loss hours_per_week
## Length:471 Min. : 127 Min. : 304 Min. : 1.00
## Class :character 1st Qu.:29313 1st Qu.:23496 1st Qu.:24.00
## Mode :character Median :49651 Median :48708 Median :49.00
## Mean :50081 Mean :48435 Mean :49.97
## 3rd Qu.:72925 3rd Qu.:71919 3rd Qu.:74.00
## Max. :99839 Max. :99944 Max. :99.00
## native_country income
## Length:471 Length:471
## Class :character Class :character
## Mode :character Mode :character
##
##
## ## age workclass education education_num marital_status
## 1 31 State-gov Bachelors 2 Married-spouse-absent
## 2 60 Never-worked Prof-school 11 Married-spouse-absent
## 3 31 Self-emp-not-inc 5th-6th 13 Married-AF-spouse
## 4 86 Local-gov Masters 12 Widowed
## 5 74 State-gov Preschool 12 Married-AF-spouse
## 6 26 Local-gov Preschool 16 Divorced
## occupation relationship race sex capital_gain
## 1 Protective-serv Wife Black Male 35297
## 2 Adm-clerical Other-relative Other Female 19616
## 3 Handlers-cleaners Husband Asian-Pac-Islander Male 91705
## 4 Prof-specialty Not-in-family Asian-Pac-Islander Female 21279
## 5 Tech-support Other-relative White Female 74141
## 6 Priv-house-serv Not-in-family Amer-Indian-Eskimo Male 78936
## capital_loss hours_per_week native_country income
## 1 15093 98 Hungary <=50K
## 2 51961 15 Cambodia <=50K
## 3 46300 96 Philippines <=50K
## 4 52751 11 Thailand <=50K
## 5 75593 35 Holand-Netherlands <=50K
## 6 7469 81 United-States <=50K#the reports are in form of comments because they were messing with creating a report for some reason
#Diagnostic report
#sample_dirty_dataset%>%diagnose_web_report(title= "Dirty dataset",subtitle = "Diagnostic report", title_color="purple", theme = "blue" )
#EDA report
#sample_dirty_dataset%>%eda_web_report(title= "Dirty dataset",subtitle = "EDA report", title_color="darkred", theme = "orange")
#Transformation report
#sample_dirty_dataset%>%transformation_web_report(title= "Dirty dataset",subtitle = "Transformation report", title_color="darkgreen", theme = "blue")