Data Cleaning Process
Klaudia Aleksiejew, Tymon Bujny, Paweł Gallas, Mateusz Barszczewski
2024-04-03
Sample data
# Load necessary libraries
library(tidyverse)## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr 1.1.4 ✔ readr 2.1.5
## ✔ forcats 1.0.0 ✔ stringr 1.5.1
## ✔ ggplot2 3.5.0 ✔ tibble 3.2.1
## ✔ lubridate 1.9.3 ✔ tidyr 1.3.1
## ✔ purrr 1.0.2
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors# Set seed for reproducibility
set.seed(7)
# Generate sample data
n <- 1000 # Number of observations
# Age between 18 and 90
age <- sample(18:90, n, replace = TRUE)
# Introduce errors in age column
age[sample(1:n, 50)] <- "InvalidAge"
# Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
# Introduce missing values in workclass column
workclass[sample(1:n, 50)] <- NA
# Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
# Introduce inconsistencies in education column
education[sample(1:n, 50)] <- "InvalidEducation"
# Education Number: 1-16
education_num <- sample(1:16, n, replace = TRUE)
# Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
# Introduce missing values in marital_status column
marital_status[sample(1:n, 50)] <- NA
# Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
# Introduce errors in occupation column
occupation[sample(1:n, 50)] <- "InvalidOccupation"
# Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
# Introduce missing values in relationship column
relationship[sample(1:n, 50)] <- NA
# Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
# Introduce inconsistencies in race column
race[sample(1:n, 50)] <- "InvalidRace"
# Sex: Female, Male
sex <- sample(c("Female", "Male"), n, replace = TRUE)
# Introduce missing values in sex column
sex[sample(1:n, 50)] <- NA
# Capital Gain: 0-99999
capital_gain <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_gain column
capital_gain[sample(1:n, 50)] <- -999
# Capital Loss: 0-99999
capital_loss <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_loss column
capital_loss[sample(1:n, 50)] <- -999
# Hours per week: 1-99
hours_per_week <- sample(1:99, n, replace = TRUE)
# Introduce missing values in hours_per_week column
hours_per_week[sample(1:n, 50)] <- NA
# Native country: United-States, Cambodia, England, ...
native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
# Introduce missing values in native_country column
native_country[sample(1:n, 50)] <- NA
# Income: <=50K, >50K
income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
# Create dataframe
sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)
# Print first few rows of the dataset
head(sample_dirty_dataset)## age workclass education education_num marital_status
## 1 59 Without-pay 10th 15 Separated
## 2 48 State-gov Assoc-acdm 6 Married-civ-spouse
## 3 83 State-gov Bachelors 1 Married-civ-spouse
## 4 32 Never-worked Prof-school 16 Married-AF-spouse
## 5 25 Private Masters 9 Married-civ-spouse
## 6 84 Without-pay 1st-4th 9 Widowed
## occupation relationship race sex capital_gain
## 1 Priv-house-serv Wife Black Male 35050
## 2 Machine-op-inspct Other-relative Amer-Indian-Eskimo Female -999
## 3 Priv-house-serv <NA> Black Male 84587
## 4 Transport-moving <NA> Asian-Pac-Islander Female 24715
## 5 InvalidOccupation Other-relative Other Female 16599
## 6 Adm-clerical Unmarried White Male 1505
## capital_loss hours_per_week native_country income
## 1 96892 34 Poland <=50K
## 2 20459 40 Iran <=50K
## 3 76544 52 Poland >50K
## 4 92227 8 Vietnam <=50K
## 5 -999 85 Hong <=50K
## 6 33344 43 Ireland <=50KSecond Task
inconsistencies_and_errors <- c()
if (any(sample_dirty_dataset$capital_gain < 0) | any(sample_dirty_dataset$capital_gain > 99999)) {
inconsistencies_and_errors <- c(inconsistencies_and_errors, "capital_gain")
}
if (any(sample_dirty_dataset$capital_loss < 0) | any(sample_dirty_dataset$capital_loss > 99999)) {
inconsistencies_and_errors <- c(inconsistencies_and_errors, "capital_loss")
}
for (col in colnames(sample_dirty_dataset)) {
if (any(grepl("Invalid", sample_dirty_dataset[[col]]))) {
inconsistencies_and_errors <- c(inconsistencies_and_errors, col)
}
}
print(inconsistencies_and_errors)## [1] "capital_gain" "capital_loss" "age" "education" "occupation"
## [6] "race"na_check <- is.na(sample_dirty_dataset)
na_counts <- colSums(na_check)
na_columns <- na_counts > 0
variables_with_na <- colnames(sample_dirty_dataset)[na_columns]
variables_with_na## [1] "workclass" "marital_status" "relationship" "sex"
## [5] "hours_per_week" "native_country"variables with errors: * age * occupation * capital_gain * capital_loss
variables with inconsistencies: * education * race
variables with missing values: * workclass * marital_status * relationship * sex * hours_per_week * native_country
Third Task
Erroneous values in the dataset replaced with NA
sample_dirty_dataset$capital_gain[sample_dirty_dataset$capital_gain < 0 | sample_dirty_dataset$capital_gain > 99999] <- NA
sample_dirty_dataset$capital_loss[sample_dirty_dataset$capital_loss < 0 | sample_dirty_dataset$capital_loss > 99999] <- NA
for (col in colnames(sample_dirty_dataset)) {
sample_dirty_dataset[[col]][grepl("Invalid", sample_dirty_dataset[[col]])] <- NA
}
head(sample_dirty_dataset)## age workclass education education_num marital_status
## 1 59 Without-pay 10th 15 Separated
## 2 48 State-gov Assoc-acdm 6 Married-civ-spouse
## 3 83 State-gov Bachelors 1 Married-civ-spouse
## 4 32 Never-worked Prof-school 16 Married-AF-spouse
## 5 25 Private Masters 9 Married-civ-spouse
## 6 84 Without-pay 1st-4th 9 Widowed
## occupation relationship race sex capital_gain
## 1 Priv-house-serv Wife Black Male 35050
## 2 Machine-op-inspct Other-relative Amer-Indian-Eskimo Female NA
## 3 Priv-house-serv <NA> Black Male 84587
## 4 Transport-moving <NA> Asian-Pac-Islander Female 24715
## 5 <NA> Other-relative Other Female 16599
## 6 Adm-clerical Unmarried White Male 1505
## capital_loss hours_per_week native_country income
## 1 96892 34 Poland <=50K
## 2 20459 40 Iran <=50K
## 3 76544 52 Poland >50K
## 4 92227 8 Vietnam <=50K
## 5 NA 85 Hong <=50K
## 6 33344 43 Ireland <=50KFourth Task
library(dlookr)## Registered S3 methods overwritten by 'dlookr':
## method from
## plot.transform scales
## print.transform scales##
## Attaching package: 'dlookr'## The following object is masked from 'package:tidyr':
##
## extract## The following object is masked from 'package:base':
##
## transformlibrary(tidyverse)
categorical_vars <- c("education", "race")
sample_dirty_dataset <- sample_dirty_dataset %>%
mutate(
education = ifelse(education %in% c('Bachelors', 'Some-college', '11th', 'HS-grad', 'Prof-school', 'Assoc-acdm', 'Assoc-voc', '9th', '7th-8th', '12th', 'Masters', '1st-4th', '10th', 'Doctorate', '5th-6th', 'Preschool'), education, NA),
race = ifelse(race %in% c('White', 'Asian-Pac-Islander', 'Amer-Indian-Eskimo', 'Other', 'Black'), race, NA)
)
head(sample_dirty_dataset) %>% select(education, race)## education race
## 1 10th Black
## 2 Assoc-acdm Amer-Indian-Eskimo
## 3 Bachelors Black
## 4 Prof-school Asian-Pac-Islander
## 5 Masters Other
## 6 1st-4th WhiteFifth Task
library(tidyverse)
#Imputing missing values using mean (for numerical variables)
sample_dirty_dataset_imputed <- sample_dirty_dataset %>%
mutate_at(vars( education_num, capital_gain, capital_loss, hours_per_week), ~ifelse(is.na(.), mean(., na.rm = TRUE), .))
# Removing observations with missing values if they cannot be imputed
sample_dirty_dataset_imputed <- sample_dirty_dataset_imputed %>%
drop_na()
head(sample_dirty_dataset_imputed)## age workclass education education_num marital_status
## 1 59 Without-pay 10th 15 Separated
## 2 48 State-gov Assoc-acdm 6 Married-civ-spouse
## 3 84 Without-pay 1st-4th 9 Widowed
## 4 25 Never-worked Masters 12 Never-married
## 5 76 Without-pay 11th 10 Married-spouse-absent
## 6 29 Federal-gov Preschool 5 Widowed
## occupation relationship race sex capital_gain
## 1 Priv-house-serv Wife Black Male 35050.00
## 2 Machine-op-inspct Other-relative Amer-Indian-Eskimo Female 49411.71
## 3 Adm-clerical Unmarried White Male 1505.00
## 4 Protective-serv Not-in-family Amer-Indian-Eskimo Male 60313.00
## 5 Farming-fishing Wife Asian-Pac-Islander Male 10088.00
## 6 Handlers-cleaners Not-in-family Other Female 48526.00
## capital_loss hours_per_week native_country income
## 1 96892 34 Poland <=50K
## 2 20459 40 Iran <=50K
## 3 33344 43 Ireland <=50K
## 4 73126 44 Cuba <=50K
## 5 7797 31 Germany <=50K
## 6 73226 31 Trinadad&Tobago <=50KSixth Task
###Method 1
age <- as.numeric(age)## Warning: NAs introduced by coercionzscore<-abs(scale(age))
plot(zscore, type = "n")
#Setting the threshold to 2 (2, 3 and above are considered outliers)
abline(h = 2, col = "red")
#Coloring the numbers above the threshold differently
text(1:length(age), zscore, col = ifelse(zscore > 2, "red", "black"))
###Method 2
# Extract outliers using boxplot
outliers_education <- boxplot(education_num)$out
cat("Outliers for Education Number:\n")## Outliers for Education Number:print(outliers_education)## numeric(0)outlier_indices <- which(education_num %in% outliers_education)
cat("Indices of outliers for Education Number:\n")## Indices of outliers for Education Number:print(outlier_indices)## integer(0)###Method 3
library(rrcov)## Loading required package: robustbase##
## Attaching package: 'robustbase'## The following object is masked _by_ '.GlobalEnv':
##
## education## Scalable Robust Estimators with High Breakdown Point (version 1.7-5)par(mfrow = c(2, 2))
plot(covMcd(age))
Seventh Task
library(AID)## Registered S3 method overwritten by 'quantmod':
## method from
## as.zoo.data.frame zoolibrary(MASS)##
## Attaching package: 'MASS'## The following object is masked from 'package:dplyr':
##
## selectlibrary(car)## Loading required package: carData##
## Attaching package: 'car'## The following object is masked from 'package:dplyr':
##
## recode## The following object is masked from 'package:purrr':
##
## somedata(sample_dirty_dataset_imputed)## Warning in data(sample_dirty_dataset_imputed): data set
## 'sample_dirty_dataset_imputed' not founddata <- sample_dirty_dataset_imputed[, 1]
library(AID)
out <- boxcoxnc(data, method = "mle", lambda = seq(-2, 2, 0.0001), verbose = FALSE, plot = FALSE)
out$lambda.hat ## [1] 0.6579library(MASS)
out <- boxcox(data ~ 1, lambda = seq(-2, 2, 0.0001), plotit = FALSE)
out$x[which.max(out$y)] ## [1] 0.6579library(AID)
out <- boxcoxnc(data, method = "sw", lambda = seq(-2,2,0.0001), verbose = F, plot = F)
out$lambda.hat## [1] 0.7141out <- boxcoxnc(data, method = "ad", lambda = seq(-2,2,0.0001), verbose = F, plot = F)
out$lambda.hat## [1] 0.7282#out <- boxcoxnc(data, method = "cvm", lambda = seq(-2,2,0.0001), verbose = F, plot = F)
#out$lambda.hat
out <- boxcoxnc(data, method = "sf", lambda = seq(-2,2,0.0001), verbose = F, plot = F)
out$lambda.hat## [1] 0.7164out <- boxcoxnc(data, method = "lt", lambda = seq(-2,2,0.0001), verbose = F, plot = F)
out$lambda.hat## [1] 0.8809out <- boxcoxnc(data, method = "jb", lambda = seq(-2,2,0.0001), verbose = F, plot = F)
out$lambda.hat## [1] 0.6385out <- boxcoxnc(data, method = "pt", lambda = seq(-2,2,0.0001), verbose = F, plot = F)
out$lambda.hat## [1] 0.5041library(AID)
out <- boxcoxnc(data, method = "sw", lambda = seq(-2,2,0.0001))
##
## Box-Cox power transformation
## -------------------------------------------------------------------
##
## lambda.hat : 0.7141
##
##
## Shapiro-Wilk normality test for transformed data (alpha = 0.05)
## -------------------------------------------------------------------
##
## statistic : 0.9585667
## p.value : 2.574894e-12
##
## Result : Transformed data are not normal.
## -------------------------------------------------------------------## Box-Cox power transformation
## -------------------------------------------------------------------
## data : data
##
## lambda.hat : -0.0605
##
##
## Shapiro-Wilk normality test for transformed data (alpha = 0.05)
## -------------------------------------------------------------------
##
## statistic : 0.9877619
## p.value : 0.9821313
##
## Result : Transformed data are normal.
## -------------------------------------------------------------------
out$lambda.hat## [1] 0.7141out$tf.data## [1] 24.351365 20.823283 31.740880 12.547571 29.454949 14.107050 27.099027
## [8] 17.053776 20.823283 11.741314 15.606012 29.454949 10.914731 13.723282
## [15] 19.144383 10.065292 23.092184 33.137889 27.695113 24.038921 25.279762
## [22] 24.038921 20.823283 21.152930 14.107050 17.408581 33.137889 29.164482
## [29] 11.330709 21.152930 23.092184 30.319877 28.286348 33.137889 19.822306
## [36] 16.335811 23.092184 30.891277 23.409366 28.580201 17.760726 24.971753
## [43] 32.022143 24.662298 16.335811 15.972446 21.480659 26.195398 29.164482
## [50] 12.547571 12.943740 29.454949 19.822306 27.991326 16.335811 16.335811
## [57] 20.823283 17.408581 21.806520 12.547571 22.773348 17.408581 33.414567
## [64] 24.662298 33.137889 25.279762 21.806520 27.397687 13.335575 14.487051
## [71] 23.409366 32.022143 13.335575 24.971753 30.032628 29.164482 33.137889
## [78] 24.662298 22.773348 18.110298 31.175462 13.723282 10.065292 17.408581
## [85] 9.631047 32.581849 21.152930 10.065292 19.822306 10.914731 18.802051
## [92] 22.130558 21.806520 26.799108 19.822306 25.279762 27.695113 21.152930
## [99] 25.891556 13.723282 33.414567 33.414567 23.409366 32.022143 10.493049
## [106] 28.872906 14.863448 30.032628 32.302461 16.696219 17.408581 30.606087
## [113] 20.491667 20.158026 20.823283 20.823283 24.662298 15.606012 32.581849
## [120] 10.065292 30.032628 23.409366 19.144383 19.484446 18.110298 26.799108
## [127] 11.330709 20.491667 23.724933 27.397687 30.891277 19.144383 21.806520
## [134] 10.493049 31.740880 31.458658 10.914731 12.547571 25.586353 21.152930
## [141] 29.744325 30.032628 10.493049 17.760726 32.302461 11.741314 14.863448
## [148] 14.863448 18.457380 17.408581 27.397687 30.319877 28.580201 22.130558
## [155] 33.414567 9.631047 22.773348 17.760726 22.130558 17.760726 30.032628
## [162] 12.146844 13.335575 27.099027 12.943740 30.032628 19.484446 23.409366
## [169] 22.452820 11.741314 30.319877 20.491667 10.065292 21.152930 29.744325
## [176] 29.164482 23.409366 28.872906 21.480659 13.335575 26.497907 12.943740
## [183] 22.452820 21.152930 16.696219 21.806520 29.164482 14.107050 28.286348
## [190] 15.236389 10.493049 20.158026 10.914731 21.806520 31.175462 18.802051
## [197] 30.032628 24.662298 24.971753 12.146844 30.032628 27.397687 24.662298
## [204] 10.914731 12.943740 20.823283 17.408581 18.110298 16.696219 28.872906
## [211] 22.130558 28.580201 32.022143 12.547571 28.286348 15.606012 32.860321
## [218] 12.943740 26.799108 20.158026 32.860321 28.872906 22.452820 24.971753
## [225] 13.335575 17.408581 9.631047 22.773348 32.581849 24.351365 19.144383
## [232] 21.806520 13.723282 32.302461 30.319877 26.799108 26.497907 30.032628
## [239] 15.972446 30.319877 30.606087 12.547571 21.152930 11.330709 15.972446
## [246] 14.487051 32.581849 31.175462 32.860321 21.152930 30.032628 21.480659
## [253] 32.022143 24.971753 14.107050 14.863448 10.914731 26.195398 30.891277
## [260] 16.335811 17.408581 15.972446 24.038921 23.409366 21.806520 31.740880
## [267] 10.914731 27.695113 10.065292 25.279762 25.586353 29.164482 22.773348
## [274] 27.397687 22.773348 13.723282 25.586353 9.631047 24.038921 14.487051
## [281] 9.631047 17.408581 25.279762 28.872906 19.822306 31.175462 32.302461
## [288] 30.606087 27.695113 27.695113 31.175462 25.279762 21.806520 17.408581
## [295] 12.943740 25.891556 32.022143 21.806520 18.802051 29.164482 25.891556
## [302] 22.452820 32.581849 27.099027 17.053776 31.740880 20.823283 11.741314
## [309] 33.414567 14.487051 17.053776 32.581849 32.022143 27.099027 10.493049
## [316] 19.822306 16.335811 17.760726 15.972446 20.158026 10.493049 27.099027
## [323] 21.806520 25.891556 27.991326 11.330709 18.802051 29.164482 27.397687
## [330] 18.457380 26.195398 32.022143 23.724933 12.146844 23.092184 14.863448
## [337] 32.581849 16.335811 18.802051 14.107050 12.943740 10.914731 25.891556
## [344] 32.022143 24.662298 21.152930 11.330709 27.099027 30.891277 26.799108
## [351] 30.891277 20.158026 32.860321 25.891556 30.606087 18.457380 17.408581
## [358] 13.335575 15.236389 21.806520 27.991326 31.458658 33.414567 31.175462
## [365] 21.480659 26.799108 33.137889 15.972446 29.744325 15.606012 16.335811
## [372] 12.146844 15.972446 32.581849 15.236389 14.863448 23.409366 32.302461
## [379] 27.397687 17.408581 33.137889 24.662298 13.335575 23.092184 12.146844
## [386] 14.863448 30.319877 28.872906 29.164482 30.032628 32.860321 27.991326
## [393] 19.484446 21.806520 16.335811 14.863448 24.351365 9.631047 9.631047
## [400] 14.107050 24.662298 12.943740 10.065292 32.022143 27.397687 11.741314
## [407] 12.943740 18.802051 27.397687 10.493049 23.724933 31.458658 16.335811
## [414] 27.397687 27.099027 24.971753 11.330709 12.547571 22.773348 28.286348
## [421] 15.972446 29.164482 18.802051 15.606012 24.351365 30.891277 19.144383
## [428] 20.158026 32.302461 28.580201 32.860321 15.972446 30.891277 14.487051
## [435] 22.130558 23.409366 30.606087 12.943740 27.397687 15.606012 22.452820
## [442] 23.409366 26.195398 14.487051 13.723282 27.397687 21.806520 24.351365
## [449] 15.606012 25.586353 27.397687 14.107050 20.158026 26.799108 25.586353
## [456] 18.457380 20.823283 20.158026 10.493049 29.164482 31.740880 17.760726
## [463] 18.110298 24.662298 20.491667 28.872906 25.279762 30.032628 16.696219
## [470] 24.351365 25.279762 18.110298 22.773348 13.723282 17.408581 11.741314
## [477] 23.092184 31.740880 22.452820 32.302461 20.491667 23.092184 18.457380
## [484] 12.547571 14.863448 21.152930 15.606012 24.662298 27.397687 10.914731
## [491] 21.806520 26.799108 15.236389 32.581849 21.152930 26.195398 24.971753
## [498] 28.580201 21.152930 18.110298 17.053776 17.053776 18.802051 26.497907
## [505] 17.408581 27.991326 15.606012 9.631047 23.409366 24.971753 32.581849
## [512] 21.806520 16.696219 21.480659 28.580201 28.286348 17.053776 26.497907
## [519] 22.773348 28.286348 22.773348 28.872906 9.631047 11.330709 18.110298
## [526] 13.335575 11.741314 14.107050 24.971753 15.236389 20.823283 20.491667
## [533] 14.487051 16.696219 27.099027 23.409366 20.158026 33.414567 23.092184
## [540] 10.914731 31.458658 19.144383 31.740880 16.335811 15.606012 20.491667
## [547] 24.351365 23.724933 15.236389 26.195398 30.319877 27.397687 19.822306
## [554] 21.806520 17.760726 14.487051 16.696219 28.286348 10.493049 11.741314
## [561] 21.152930 21.806520 21.152930 28.286348 17.760726 27.397687 21.806520
## [568] 28.580201 32.022143 33.414567 20.158026 19.822306 16.696219 14.487051
## [575] 28.286348 17.760726 33.137889 28.286348 19.822306 13.723282 24.971753
## [582] 22.130558 33.137889 24.662298 12.943740 14.107050 19.822306 26.195398
## [589] 15.972446 31.175462 32.860321 12.547571 16.335811 21.480659 11.330709
## [596] 21.152930 27.397687 30.319877 25.279762 28.286348 12.547571 17.408581
## [603] 26.497907 25.891556 31.458658 14.863448 31.458658 17.760726 29.164482
## [610] 27.397687 20.823283 11.330709 27.397687 28.286348 23.724933 17.408581
## [617] 21.806520 15.606012 21.806520 25.279762 29.164482 21.152930 27.991326
## [624] 33.414567 32.302461 12.547571 27.991326 31.175462 15.606012out <- boxcoxnc(data, method = "sw", lambda = seq(-2,2,0.0001), verbose = F, plot = F)Nineth Task - Conclusion
This dataset at the beginning was ‘dirty’ and we were supposed to clean it up. In step number two we wrote down information about errors, missing values and inconsistencies. In order to make this data more clean we either subsituded NA with values counted by mean (when it came to numerical values) or used methods given in videos we were provided. When it comes to outliers, we used 3 different methods, but results aren’t surprising. Our data is created randomly in const interwals, so it’s hard to meet weird values. We cleaned this dataset as possibly as we could.
Tenth Task
diagnose_web_report(sample_dirty_dataset_imputed)##
##
## processing file: diagnosis_temp.Rmd## | | | 0% | |. | 1% | |. | 3% [setup] | |.. | 4% | |... | 6% [load_packages] | |... | 7% | |.... | 9% [get-parameters] | |..... | 10% | |..... | 12% [unnamed-chunk-11] | |...... | 13% | |....... | 15% [diagose] | |....... | 16% | |........ | 18% [create-overview] | |......... | 19% | |......... | 21% [overview] | |.......... | 22% | |........... | 24% [overview-datastructure] | |........... | 25% | |............ | 27% [overview-pre] | |............. | 28% | |............. | 30% [overview-warnings] | |.............. | 31% | |............... | 33% [warnings_summary] | |............... | 34% | |................ | 36% [warnings] | |................. | 37% | |................. | 39% [overview-variables] | |.................. | 40% | |................... | 42% [variables] | |................... | 43% | |.................... | 45% [missing] | |..................... | 46% | |..................... | 48% [missing-list] | |...................... | 49% | |....................... | 51% [missing-data] | |........................ | 52% | |........................ | 54% [missing-visualization] | |......................... | 55% | |.......................... | 57% [missing-viz2] | |.......................... | 58% | |........................... | 60% [unique] | |............................ | 61% | |............................ | 63% [unique-categorical] | |............................. | 64% | |.............................. | 66% [unique-date-category] | |.............................. | 67% | |............................... | 69% [unique-numerical] | |................................ | 70% | |................................ | 72% [unique-data-numeric] | |................................. | 73% | |.................................. | 75% [outliers] | |.................................. | 76% | |................................... | 78% [outliers-list] | |.................................... | 79% | |.................................... | 81% [samples] | |..................................... | 82% | |...................................... | 84% [duplicated] | |...................................... | 85% | |....................................... | 87% [duplicated-list] | |........................................ | 88% | |........................................ | 90% [heades] | |......................................... | 91% | |.......................................... | 93% [sample-head] | |.......................................... | 94% | |........................................... | 96% [tails] | |............................................ | 97% | |............................................ | 99% [sample-tail] | |.............................................| 100% ## output file: diagnosis_temp.knit.md## "C:/Program Files/RStudio/resources/app/bin/quarto/bin/tools/pandoc" +RTS -K512m -RTS diagnosis_temp.knit.md --to html4 --from markdown+autolink_bare_uris+tex_math_single_backslash --output pandoc47c75fa131d.html --lua-filter "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\pagebreak.lua" --lua-filter "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\latex-div.lua" --embed-resources --standalone --variable bs3=TRUE --section-divs --template "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmd\h\default.html" --no-highlight --variable highlightjs=1 --variable theme=bootstrap --css "C:/Users/klaud/AppData/Local/R/win-library/4.3/dlookr/resources/dlookr-bootstrap.css" --mathjax --variable "mathjax-url=https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" --include-in-header "C:\Users\klaud\AppData\Local\Temp\RtmpgnI57H\rmarkdown-str47c1cb5629a.html" --variable code_folding=show --variable code_menu=1 --include-in-header header_temp.html --include-after-body "C:\Users\klaud\AppData\Local\R\win-library\4.3\dlookr\resources\footer.html"##
## Output created: C:\Users\klaud\AppData\Local\Temp\RtmpgnI57H/Diagnosis_Report.htmleda_web_report(sample_dirty_dataset_imputed)##
##
## processing file: eda_temp.Rmd## | | | 0% | |. | 2% | |.. | 3% [setup] | |.. | 5% | |... | 6% [load_packages] | |.... | 8% | |..... | 10% [unnamed-chunk-1] | |...... | 11% | |...... | 13% [udf] | |....... | 14% | |........ | 16% [check_variables] | |......... | 17% | |.......... | 19% [create-overview] | |.......... | 21% | |........... | 22% [overview] | |............ | 24% | |............. | 25% [overview-pre] | |............. | 27% | |.............. | 29% [unnamed-chunk-2] | |............... | 30% | |................ | 32% [unnamed-chunk-3] | |................. | 33% | |................. | 35% [variables] | |.................. | 37% | |................... | 38% [normality] | |.................... | 40% | |..................... | 41% [normality-list] | |..................... | 43% | |...................... | 44% [unnamed-chunk-4] | |....................... | 46% | |........................ | 48% [unnamed-chunk-5] | |......................... | 49% | |......................... | 51% [compare_numerical] | |.......................... | 52% | |........................... | 54% [unnamed-chunk-6] | |............................ | 56% | |............................. | 57% [compare-category] | |............................. | 59% | |.............................. | 60% [unnamed-chunk-7] | |............................... | 62% | |................................ | 63% [unnamed-chunk-8] | |................................. | 65% | |................................. | 67% [unnamed-chunk-9] | |.................................. | 68% | |................................... | 70% [correlation] | |.................................... | 71% | |..................................... | 73% [unnamed-chunk-10] | |..................................... | 75% | |...................................... | 76% [plot-correlation] | |....................................... | 78% | |........................................ | 79% [unnamed-chunk-11] | |........................................ | 81% | |......................................... | 83% [unnamed-chunk-12] | |.......................................... | 84% | |........................................... | 86% [group-numerical] | |............................................ | 87% | |............................................ | 89% [unnamed-chunk-13] | |............................................. | 90% | |.............................................. | 92% [group-categorical] | |............................................... | 94% | |................................................ | 95% [unnamed-chunk-14] | |................................................ | 97% | |................................................. | 98% [group-correlation] | |..................................................| 100% ## output file: eda_temp.knit.md## "C:/Program Files/RStudio/resources/app/bin/quarto/bin/tools/pandoc" +RTS -K512m -RTS eda_temp.knit.md --to html4 --from markdown+autolink_bare_uris+tex_math_single_backslash --output pandoc47c1aca3729.html --lua-filter "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\pagebreak.lua" --lua-filter "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\latex-div.lua" --embed-resources --standalone --variable bs3=TRUE --section-divs --template "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmd\h\default.html" --no-highlight --variable highlightjs=1 --variable theme=bootstrap --css "C:/Users/klaud/AppData/Local/R/win-library/4.3/dlookr/resources/dlookr-bootstrap.css" --mathjax --variable "mathjax-url=https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" --include-in-header "C:\Users\klaud\AppData\Local\Temp\RtmpgnI57H\rmarkdown-str47c4f875d70.html" --variable code_folding=show --variable code_menu=1 --include-in-header header_temp.html --include-after-body "C:\Users\klaud\AppData\Local\R\win-library\4.3\dlookr\resources\footer.html"##
## Output created: C:\Users\klaud\AppData\Local\Temp\RtmpgnI57H/EDA_Report.htmltransformation_web_report(sample_dirty_dataset_imputed)##
##
## processing file: transformation_temp.Rmd## | | | 0% | |. | 3% | |... | 5% [setup] | |.... | 8% | |..... | 10% [load_packages] | |....... | 13% | |........ | 15% [unnamed-chunk-1] | |......... | 18% | |........... | 21% [udf] | |............ | 23% | |............. | 26% [create-overview] | |............... | 28% | |................ | 31% [overview] | |................. | 33% | |................... | 36% [overview-pre] | |.................... | 38% | |..................... | 41% [unnamed-chunk-2] | |....................... | 44% | |........................ | 46% [unnamed-chunk-3] | |......................... | 49% | |........................... | 51% [unnamed-chunk-4] | |............................ | 54% | |............................. | 56% [nalist] | |............................... | 59% | |................................ | 62% [unnamed-chunk-5] | |................................. | 64% | |................................... | 67% [outlist] | |.................................... | 69% | |..................................... | 72% [unnamed-chunk-6] | |....................................... | 74% | |........................................ | 77% [skweness] | |......................................... | 79% | |........................................... | 82% [unnamed-chunk-7] | |............................................ | 85% | |............................................. | 87% [binning] | |............................................... | 90% | |................................................ | 92% [unnamed-chunk-8] | |................................................. | 95% | |................................................... | 97% [optimal-binning] | |....................................................| 100% ## output file: transformation_temp.knit.md## "C:/Program Files/RStudio/resources/app/bin/quarto/bin/tools/pandoc" +RTS -K512m -RTS transformation_temp.knit.md --to html4 --from markdown+autolink_bare_uris+tex_math_single_backslash --output pandoc47c7bd12acb.html --lua-filter "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\pagebreak.lua" --lua-filter "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\latex-div.lua" --embed-resources --standalone --variable bs3=TRUE --section-divs --template "C:\Users\klaud\AppData\Local\R\win-library\4.3\rmarkdown\rmd\h\default.html" --no-highlight --variable highlightjs=1 --variable theme=bootstrap --css "C:/Users/klaud/AppData/Local/R/win-library/4.3/dlookr/resources/dlookr-bootstrap.css" --mathjax --variable "mathjax-url=https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" --include-in-header "C:\Users\klaud\AppData\Local\Temp\RtmpgnI57H\rmarkdown-str47c4be764bb.html" --variable code_folding=show --variable code_menu=1 --include-in-header header_temp.html --include-after-body "C:\Users\klaud\AppData\Local\R\win-library\4.3\dlookr\resources\footer.html"##
## Output created: C:\Users\klaud\AppData\Local\Temp\RtmpgnI57H/Transformation_Report.html