#BOX PLOT#TASK 1 JAKUB KWIATKOWSKI
# Load necessary libraries
library(tidyverse)
#;-) ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
#;-) ✔ dplyr 1.1.4 ✔ readr 2.1.5
#;-) ✔ forcats 1.0.0 ✔ stringr 1.5.1
#;-) ✔ ggplot2 3.4.4 ✔ tibble 3.2.1
#;-) ✔ lubridate 1.9.3 ✔ tidyr 1.3.1
#;-) ✔ purrr 1.0.2
#;-) ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
#;-) ✖ dplyr::filter() masks stats::filter()
#;-) ✖ dplyr::lag() masks stats::lag()
#;-) ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
# Set seed for reproducibility
set.seed(2137)
# Generate sample data
n <- 1000 # Number of observations
# Age between 18 and 90
age <- sample(18:90, n, replace = TRUE)
# Introduce errors in age column
age[sample(1:n, 50)] <- "InvalidAge"
# Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
# Introduce missing values in workclass column
workclass[sample(1:n, 50)] <- NA
# Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
# Introduce inconsistencies in education column
education[sample(1:n, 50)] <- "InvalidEducation"
# Education Number: 1-16
education_num <- sample(1:16, n, replace = TRUE)
# Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
# Introduce missing values in marital_status column
marital_status[sample(1:n, 50)] <- NA
# Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
# Introduce errors in occupation column
occupation[sample(1:n, 50)] <- "InvalidOccupation"
# Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
# Introduce missing values in relationship column
relationship[sample(1:n, 50)] <- NA
# Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
# Introduce inconsistencies in race column
race[sample(1:n, 50)] <- "InvalidRace"
# Sex: Female, Male
sex <- sample(c("Female", "Male"), n, replace = TRUE)
# Introduce missing values in sex column
sex[sample(1:n, 50)] <- NA
# Capital Gain: 0-99999
capital_gain <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_gain column
capital_gain[sample(1:n, 50)] <- -999
# Capital Loss: 0-99999
capital_loss <- sample(0:99999, n, replace = TRUE)
# Introduce errors in capital_loss column
capital_loss[sample(1:n, 50)] <- -999
# Hours per week: 1-99
hours_per_week <- sample(1:99, n, replace = TRUE)
# Introduce missing values in hours_per_week column
hours_per_week[sample(1:n, 50)] <- NA
# Native country: United-States, Cambodia, England, ...
native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
# Introduce missing values in native_country column
native_country[sample(1:n, 50)] <- NA
# Income: <=50K, >50K
income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
# Create dataframe
sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)
# Print first few rows of the dataset
head(sample_dirty_dataset)
#;-) age workclass education education_num marital_status
#;-) 1 43 Federal-gov Masters 2 Never-married
#;-) 2 75 Local-gov Assoc-voc 11 Divorced
#;-) 3 80 Without-pay 1st-4th 16 Married-spouse-absent
#;-) 4 68 Self-emp-inc Prof-school 8 Never-married
#;-) 5 31 Without-pay Preschool 9 Widowed
#;-) 6 34 Self-emp-inc 11th 13 Married-AF-spouse
#;-) occupation relationship race sex capital_gain
#;-) 1 Prof-specialty Husband White Female 63185
#;-) 2 Prof-specialty Other-relative Asian-Pac-Islander Male 55268
#;-) 3 Tech-support Wife Other Male 57785
#;-) 4 Sales Husband White Female -999
#;-) 5 Other-service Husband Amer-Indian-Eskimo Male 17987
#;-) 6 Priv-house-serv Husband White Female 34166
#;-) capital_loss hours_per_week native_country income
#;-) 1 10878 54 Columbia <=50K
#;-) 2 24351 61 Cuba <=50K
#;-) 3 54083 58 <NA> <=50K
#;-) 4 16499 30 India <=50K
#;-) 5 54445 35 Jamaica <=50K
#;-) 6 50965 24 Outlying-US(Guam-USVI-etc) >50K#TASK 2 JAKUB KWIATKOWSKI
# Variables and Their Data Types:
#
# age: Intended to be numeric, but contains some "InvalidAge" entries, which are actually character strings.
# workclass: Categorical with a few NA (missing) entries.
# education: Categorical but contains some "InvalidEducation" entries
# education_num: Numeric, ranging from 1 to 16.
# marital_status: Categorical with some NA (missing) entries.
# occupation: Categorical but contains some "InvalidOccupation" entries.
# relationship: Categorical with some NA (missing) entries.
# race: Categorical but contains some "InvalidRace" entries.
# sex: Categorical with some NA (missing) entries.
# capital_gain and capital_loss: Numeric but contain -999 for some entries.
# hours_per_week: Numeric but includes some NA (missing) entries.
# native_country: Categorical with some NA (missing) entries.
# income: Categorical, representing two income groups.
# Identified Issues:
# Errors:
# age: Contains "InvalidAge" strings instead of numeric values.
# education: Contains "InvalidEducation" strings.
# occupation: Contains "InvalidOccupation" strings.
# race: Contains "InvalidRace" strings.
# capital_gain and capital_loss: Contain -999, which is not a valid value for these variables.
# Inconsistencies: The introduction of strings like "InvalidAge", "InvalidEducation", "InvalidOccupation", and "InvalidRace" in otherwise categorical or numeric fields creates inconsistencies.
# Missing Values:
# Several variables (workclass, marital_status, relationship, sex, hours_per_week, native_country) have missing values (NA).#TASK 3,4 JAKUB KWIATKOWSKI
# Load additional libraries
library(dlookr)
#;-) Warning: pakiet 'dlookr' został zbudowany w wersji R 4.3.3
#;-) Registered S3 methods overwritten by 'dlookr':
#;-) method from
#;-) plot.transform scales
#;-) print.transform scales
#;-)
#;-) Dołączanie pakietu: 'dlookr'
#;-) Następujący obiekt został zakryty z 'package:tidyr':
#;-)
#;-) extract
#;-) Następujący obiekt został zakryty z 'package:base':
#;-)
#;-) transform
library(editrules)
#;-) Warning: pakiet 'editrules' został zbudowany w wersji R 4.3.3
#;-) Ładowanie wymaganego pakietu: igraph
#;-) Warning: pakiet 'igraph' został zbudowany w wersji R 4.3.3
#;-)
#;-) Dołączanie pakietu: 'igraph'
#;-) Następujące obiekty zostały zakryte z 'package:lubridate':
#;-)
#;-) %--%, union
#;-) Następujące obiekty zostały zakryte z 'package:dplyr':
#;-)
#;-) as_data_frame, groups, union
#;-) Następujące obiekty zostały zakryte z 'package:purrr':
#;-)
#;-) compose, simplify
#;-) Następujący obiekt został zakryty z 'package:tidyr':
#;-)
#;-) crossing
#;-) Następujący obiekt został zakryty z 'package:tibble':
#;-)
#;-) as_data_frame
#;-) Następujące obiekty zostały zakryte z 'package:stats':
#;-)
#;-) decompose, spectrum
#;-) Następujący obiekt został zakryty z 'package:base':
#;-)
#;-) union
#;-)
#;-) Dołączanie pakietu: 'editrules'
#;-) Następujące obiekty zostały zakryte z 'package:igraph':
#;-)
#;-) blocks, normalize
#;-) Następujący obiekt został zakryty z 'package:dplyr':
#;-)
#;-) contains
#;-) Następujący obiekt został zakryty z 'package:purrr':
#;-)
#;-) reduce
#;-) Następujące obiekty zostały zakryte z 'package:tidyr':
#;-)
#;-) contains, separate
library(validate)
#;-) Warning: pakiet 'validate' został zbudowany w wersji R 4.3.3
#;-)
#;-) Dołączanie pakietu: 'validate'
#;-) Następujące obiekty zostały zakryte z 'package:igraph':
#;-)
#;-) compare, hierarchy
#;-) Następujący obiekt został zakryty z 'package:dplyr':
#;-)
#;-) expr
#;-) Następujący obiekt został zakryty z 'package:ggplot2':
#;-)
#;-) expr
# TASK 1: correct age
sample_dirty_dataset$age <- as.numeric(as.character(sample_dirty_dataset$age))
#;-) Warning: pojawiły się wartości NA na skutek przekształcenia
# Replace errors with NA
sample_dirty_dataset$age[sample_dirty_dataset$age < 18 | sample_dirty_dataset$age > 90 | is.na(sample_dirty_dataset$age)] <- NA
# TASK 2: inconsistent values
# Replace "InvalidEducation" and "InvalidRace" with NA
sample_dirty_dataset$education[sample_dirty_dataset$education == "InvalidEducation"] <- NA
sample_dirty_dataset$race[sample_dirty_dataset$race == "InvalidRace"] <- NA
# TASK 3: correct capital_gain and capital_loss
sample_dirty_dataset$capital_gain[sample_dirty_dataset$capital_gain == -999] <- NA
sample_dirty_dataset$capital_loss[sample_dirty_dataset$capital_loss == -999] <- NA
rules <- validator(
age >= 18 & age <= 90,
education_num >= 1 & education_num <= 16
)
errors <- confront(sample_dirty_dataset, rules)
summary(errors)
#;-) name items passes fails nNA error warning
#;-) 1 V1 1000 950 0 50 FALSE FALSE
#;-) 2 V2 1000 1000 0 0 FALSE FALSE
#;-) expression
#;-) 1 age - 18 >= -1e-08 & age - 90 <= 1e-08
#;-) 2 education_num - 1 >= -1e-08 & education_num - 16 <= 1e-08#TASK 5 Anna Radosz
library(dlookr)
library(mice)
#;-) Warning: pakiet 'mice' został zbudowany w wersji R 4.3.3
#;-) Warning in check_dep_version(): ABI version mismatch:
#;-) lme4 was built with Matrix ABI version 1
#;-) Current Matrix ABI version is 0
#;-) Please re-install lme4 from source or restore original 'Matrix' package
#;-)
#;-) Dołączanie pakietu: 'mice'
#;-) Następujący obiekt został zakryty z 'package:stats':
#;-)
#;-) filter
#;-) Następujące obiekty zostały zakryte z 'package:base':
#;-)
#;-) cbind, rbind
#changing NA values for capital gain, capital loss, age, hours per week
#Multiple inputation for capital gain
data_multiple <- complete(mice(sample_dirty_dataset, m = 5), action = "broad", include = TRUE)
#;-)
#;-) iter imp variable
#;-) 1 1 age capital_gain capital_loss hours_per_week
#;-) 1 2 age capital_gain capital_loss hours_per_week
#;-) 1 3 age capital_gain capital_loss hours_per_week
#;-) 1 4 age capital_gain capital_loss hours_per_week
#;-) 1 5 age capital_gain capital_loss hours_per_week
#;-) 2 1 age capital_gain capital_loss hours_per_week
#;-) 2 2 age capital_gain capital_loss hours_per_week
#;-) 2 3 age capital_gain capital_loss hours_per_week
#;-) 2 4 age capital_gain capital_loss hours_per_week
#;-) 2 5 age capital_gain capital_loss hours_per_week
#;-) 3 1 age capital_gain capital_loss hours_per_week
#;-) 3 2 age capital_gain capital_loss hours_per_week
#;-) 3 3 age capital_gain capital_loss hours_per_week
#;-) 3 4 age capital_gain capital_loss hours_per_week
#;-) 3 5 age capital_gain capital_loss hours_per_week
#;-) 4 1 age capital_gain capital_loss hours_per_week
#;-) 4 2 age capital_gain capital_loss hours_per_week
#;-) 4 3 age capital_gain capital_loss hours_per_week
#;-) 4 4 age capital_gain capital_loss hours_per_week
#;-) 4 5 age capital_gain capital_loss hours_per_week
#;-) 5 1 age capital_gain capital_loss hours_per_week
#;-) 5 2 age capital_gain capital_loss hours_per_week
#;-) 5 3 age capital_gain capital_loss hours_per_week
#;-) 5 4 age capital_gain capital_loss hours_per_week
#;-) 5 5 age capital_gain capital_loss hours_per_week
#;-) Warning: Number of logged events: 9
#;-) New names:
#;-) • `age` -> `age...1`
#;-) • `workclass` -> `workclass...2`
#;-) • `education` -> `education...3`
#;-) • `education_num` -> `education_num...4`
#;-) • `marital_status` -> `marital_status...5`
#;-) • `occupation` -> `occupation...6`
#;-) • `relationship` -> `relationship...7`
#;-) • `race` -> `race...8`
#;-) • `sex` -> `sex...9`
#;-) • `capital_gain` -> `capital_gain...10`
#;-) • `capital_loss` -> `capital_loss...11`
#;-) • `hours_per_week` -> `hours_per_week...12`
#;-) • `native_country` -> `native_country...13`
#;-) • `income` -> `income...14`
#;-) • `age` -> `age...15`
#;-) • `workclass` -> `workclass...16`
#;-) • `education` -> `education...17`
#;-) • `education_num` -> `education_num...18`
#;-) • `marital_status` -> `marital_status...19`
#;-) • `occupation` -> `occupation...20`
#;-) • `relationship` -> `relationship...21`
#;-) • `race` -> `race...22`
#;-) • `sex` -> `sex...23`
#;-) • `capital_gain` -> `capital_gain...24`
#;-) • `capital_loss` -> `capital_loss...25`
#;-) • `hours_per_week` -> `hours_per_week...26`
#;-) • `native_country` -> `native_country...27`
#;-) • `income` -> `income...28`
#;-) • `age` -> `age...29`
#;-) • `workclass` -> `workclass...30`
#;-) • `education` -> `education...31`
#;-) • `education_num` -> `education_num...32`
#;-) • `marital_status` -> `marital_status...33`
#;-) • `occupation` -> `occupation...34`
#;-) • `relationship` -> `relationship...35`
#;-) • `race` -> `race...36`
#;-) • `sex` -> `sex...37`
#;-) • `capital_gain` -> `capital_gain...38`
#;-) • `capital_loss` -> `capital_loss...39`
#;-) • `hours_per_week` -> `hours_per_week...40`
#;-) • `native_country` -> `native_country...41`
#;-) • `income` -> `income...42`
#;-) • `age` -> `age...43`
#;-) • `workclass` -> `workclass...44`
#;-) • `education` -> `education...45`
#;-) • `education_num` -> `education_num...46`
#;-) • `marital_status` -> `marital_status...47`
#;-) • `occupation` -> `occupation...48`
#;-) • `relationship` -> `relationship...49`
#;-) • `race` -> `race...50`
#;-) • `sex` -> `sex...51`
#;-) • `capital_gain` -> `capital_gain...52`
#;-) • `capital_loss` -> `capital_loss...53`
#;-) • `hours_per_week` -> `hours_per_week...54`
#;-) • `native_country` -> `native_country...55`
#;-) • `income` -> `income...56`
#;-) • `age` -> `age...57`
#;-) • `workclass` -> `workclass...58`
#;-) • `education` -> `education...59`
#;-) • `education_num` -> `education_num...60`
#;-) • `marital_status` -> `marital_status...61`
#;-) • `occupation` -> `occupation...62`
#;-) • `relationship` -> `relationship...63`
#;-) • `race` -> `race...64`
#;-) • `sex` -> `sex...65`
#;-) • `capital_gain` -> `capital_gain...66`
#;-) • `capital_loss` -> `capital_loss...67`
#;-) • `hours_per_week` -> `hours_per_week...68`
#;-) • `native_country` -> `native_country...69`
#;-) • `income` -> `income...70`
#;-) • `age` -> `age...71`
#;-) • `workclass` -> `workclass...72`
#;-) • `education` -> `education...73`
#;-) • `education_num` -> `education_num...74`
#;-) • `marital_status` -> `marital_status...75`
#;-) • `occupation` -> `occupation...76`
#;-) • `relationship` -> `relationship...77`
#;-) • `race` -> `race...78`
#;-) • `sex` -> `sex...79`
#;-) • `capital_gain` -> `capital_gain...80`
#;-) • `capital_loss` -> `capital_loss...81`
#;-) • `hours_per_week` -> `hours_per_week...82`
#;-) • `native_country` -> `native_country...83`
#;-) • `income` -> `income...84`
mean_multiple <- c(mean(data_multiple$capital_gain.1),
mean(data_multiple$capital_gain.2),
mean(data_multiple$capital_gain.3),
mean(data_multiple$capital_gain.4),
mean(data_multiple$capital_gain.5))
sample_dirty_dataset$capital_gain[is.na(sample_dirty_dataset$capital_gain)]<-mean(mean_multiple)
#Multiple inputation for capital loss
data_multiple <- complete(mice(sample_dirty_dataset, m = 5), action = "broad", include = TRUE)
#;-)
#;-) iter imp variable
#;-) 1 1 age capital_loss hours_per_week
#;-) 1 2 age capital_loss hours_per_week
#;-) 1 3 age capital_loss hours_per_week
#;-) 1 4 age capital_loss hours_per_week
#;-) 1 5 age capital_loss hours_per_week
#;-) 2 1 age capital_loss hours_per_week
#;-) 2 2 age capital_loss hours_per_week
#;-) 2 3 age capital_loss hours_per_week
#;-) 2 4 age capital_loss hours_per_week
#;-) 2 5 age capital_loss hours_per_week
#;-) 3 1 age capital_loss hours_per_week
#;-) 3 2 age capital_loss hours_per_week
#;-) 3 3 age capital_loss hours_per_week
#;-) 3 4 age capital_loss hours_per_week
#;-) 3 5 age capital_loss hours_per_week
#;-) 4 1 age capital_loss hours_per_week
#;-) 4 2 age capital_loss hours_per_week
#;-) 4 3 age capital_loss hours_per_week
#;-) 4 4 age capital_loss hours_per_week
#;-) 4 5 age capital_loss hours_per_week
#;-) 5 1 age capital_loss hours_per_week
#;-) 5 2 age capital_loss hours_per_week
#;-) 5 3 age capital_loss hours_per_week
#;-) 5 4 age capital_loss hours_per_week
#;-) 5 5 age capital_loss hours_per_week
#;-) Warning: Number of logged events: 9
#;-) New names:
#;-) • `age` -> `age...1`
#;-) • `workclass` -> `workclass...2`
#;-) • `education` -> `education...3`
#;-) • `education_num` -> `education_num...4`
#;-) • `marital_status` -> `marital_status...5`
#;-) • `occupation` -> `occupation...6`
#;-) • `relationship` -> `relationship...7`
#;-) • `race` -> `race...8`
#;-) • `sex` -> `sex...9`
#;-) • `capital_gain` -> `capital_gain...10`
#;-) • `capital_loss` -> `capital_loss...11`
#;-) • `hours_per_week` -> `hours_per_week...12`
#;-) • `native_country` -> `native_country...13`
#;-) • `income` -> `income...14`
#;-) • `age` -> `age...15`
#;-) • `workclass` -> `workclass...16`
#;-) • `education` -> `education...17`
#;-) • `education_num` -> `education_num...18`
#;-) • `marital_status` -> `marital_status...19`
#;-) • `occupation` -> `occupation...20`
#;-) • `relationship` -> `relationship...21`
#;-) • `race` -> `race...22`
#;-) • `sex` -> `sex...23`
#;-) • `capital_gain` -> `capital_gain...24`
#;-) • `capital_loss` -> `capital_loss...25`
#;-) • `hours_per_week` -> `hours_per_week...26`
#;-) • `native_country` -> `native_country...27`
#;-) • `income` -> `income...28`
#;-) • `age` -> `age...29`
#;-) • `workclass` -> `workclass...30`
#;-) • `education` -> `education...31`
#;-) • `education_num` -> `education_num...32`
#;-) • `marital_status` -> `marital_status...33`
#;-) • `occupation` -> `occupation...34`
#;-) • `relationship` -> `relationship...35`
#;-) • `race` -> `race...36`
#;-) • `sex` -> `sex...37`
#;-) • `capital_gain` -> `capital_gain...38`
#;-) • `capital_loss` -> `capital_loss...39`
#;-) • `hours_per_week` -> `hours_per_week...40`
#;-) • `native_country` -> `native_country...41`
#;-) • `income` -> `income...42`
#;-) • `age` -> `age...43`
#;-) • `workclass` -> `workclass...44`
#;-) • `education` -> `education...45`
#;-) • `education_num` -> `education_num...46`
#;-) • `marital_status` -> `marital_status...47`
#;-) • `occupation` -> `occupation...48`
#;-) • `relationship` -> `relationship...49`
#;-) • `race` -> `race...50`
#;-) • `sex` -> `sex...51`
#;-) • `capital_gain` -> `capital_gain...52`
#;-) • `capital_loss` -> `capital_loss...53`
#;-) • `hours_per_week` -> `hours_per_week...54`
#;-) • `native_country` -> `native_country...55`
#;-) • `income` -> `income...56`
#;-) • `age` -> `age...57`
#;-) • `workclass` -> `workclass...58`
#;-) • `education` -> `education...59`
#;-) • `education_num` -> `education_num...60`
#;-) • `marital_status` -> `marital_status...61`
#;-) • `occupation` -> `occupation...62`
#;-) • `relationship` -> `relationship...63`
#;-) • `race` -> `race...64`
#;-) • `sex` -> `sex...65`
#;-) • `capital_gain` -> `capital_gain...66`
#;-) • `capital_loss` -> `capital_loss...67`
#;-) • `hours_per_week` -> `hours_per_week...68`
#;-) • `native_country` -> `native_country...69`
#;-) • `income` -> `income...70`
#;-) • `age` -> `age...71`
#;-) • `workclass` -> `workclass...72`
#;-) • `education` -> `education...73`
#;-) • `education_num` -> `education_num...74`
#;-) • `marital_status` -> `marital_status...75`
#;-) • `occupation` -> `occupation...76`
#;-) • `relationship` -> `relationship...77`
#;-) • `race` -> `race...78`
#;-) • `sex` -> `sex...79`
#;-) • `capital_gain` -> `capital_gain...80`
#;-) • `capital_loss` -> `capital_loss...81`
#;-) • `hours_per_week` -> `hours_per_week...82`
#;-) • `native_country` -> `native_country...83`
#;-) • `income` -> `income...84`
mean_multiple <- c(mean(data_multiple$capital_loss.1),
mean(data_multiple$capital_loss.2),
mean(data_multiple$capital_loss.3),
mean(data_multiple$capital_loss.4),
mean(data_multiple$capital_loss.5))
sample_dirty_dataset$capital_loss[is.na(sample_dirty_dataset$capital_loss)]<-mean(mean_multiple)
#Multiple inputation for age
data_multiple <- complete(mice(sample_dirty_dataset, m = 5), action = "broad", include = TRUE)
#;-)
#;-) iter imp variable
#;-) 1 1 age hours_per_week
#;-) 1 2 age hours_per_week
#;-) 1 3 age hours_per_week
#;-) 1 4 age hours_per_week
#;-) 1 5 age hours_per_week
#;-) 2 1 age hours_per_week
#;-) 2 2 age hours_per_week
#;-) 2 3 age hours_per_week
#;-) 2 4 age hours_per_week
#;-) 2 5 age hours_per_week
#;-) 3 1 age hours_per_week
#;-) 3 2 age hours_per_week
#;-) 3 3 age hours_per_week
#;-) 3 4 age hours_per_week
#;-) 3 5 age hours_per_week
#;-) 4 1 age hours_per_week
#;-) 4 2 age hours_per_week
#;-) 4 3 age hours_per_week
#;-) 4 4 age hours_per_week
#;-) 4 5 age hours_per_week
#;-) 5 1 age hours_per_week
#;-) 5 2 age hours_per_week
#;-) 5 3 age hours_per_week
#;-) 5 4 age hours_per_week
#;-) 5 5 age hours_per_week
#;-) Warning: Number of logged events: 9
#;-) New names:
#;-) • `age` -> `age...1`
#;-) • `workclass` -> `workclass...2`
#;-) • `education` -> `education...3`
#;-) • `education_num` -> `education_num...4`
#;-) • `marital_status` -> `marital_status...5`
#;-) • `occupation` -> `occupation...6`
#;-) • `relationship` -> `relationship...7`
#;-) • `race` -> `race...8`
#;-) • `sex` -> `sex...9`
#;-) • `capital_gain` -> `capital_gain...10`
#;-) • `capital_loss` -> `capital_loss...11`
#;-) • `hours_per_week` -> `hours_per_week...12`
#;-) • `native_country` -> `native_country...13`
#;-) • `income` -> `income...14`
#;-) • `age` -> `age...15`
#;-) • `workclass` -> `workclass...16`
#;-) • `education` -> `education...17`
#;-) • `education_num` -> `education_num...18`
#;-) • `marital_status` -> `marital_status...19`
#;-) • `occupation` -> `occupation...20`
#;-) • `relationship` -> `relationship...21`
#;-) • `race` -> `race...22`
#;-) • `sex` -> `sex...23`
#;-) • `capital_gain` -> `capital_gain...24`
#;-) • `capital_loss` -> `capital_loss...25`
#;-) • `hours_per_week` -> `hours_per_week...26`
#;-) • `native_country` -> `native_country...27`
#;-) • `income` -> `income...28`
#;-) • `age` -> `age...29`
#;-) • `workclass` -> `workclass...30`
#;-) • `education` -> `education...31`
#;-) • `education_num` -> `education_num...32`
#;-) • `marital_status` -> `marital_status...33`
#;-) • `occupation` -> `occupation...34`
#;-) • `relationship` -> `relationship...35`
#;-) • `race` -> `race...36`
#;-) • `sex` -> `sex...37`
#;-) • `capital_gain` -> `capital_gain...38`
#;-) • `capital_loss` -> `capital_loss...39`
#;-) • `hours_per_week` -> `hours_per_week...40`
#;-) • `native_country` -> `native_country...41`
#;-) • `income` -> `income...42`
#;-) • `age` -> `age...43`
#;-) • `workclass` -> `workclass...44`
#;-) • `education` -> `education...45`
#;-) • `education_num` -> `education_num...46`
#;-) • `marital_status` -> `marital_status...47`
#;-) • `occupation` -> `occupation...48`
#;-) • `relationship` -> `relationship...49`
#;-) • `race` -> `race...50`
#;-) • `sex` -> `sex...51`
#;-) • `capital_gain` -> `capital_gain...52`
#;-) • `capital_loss` -> `capital_loss...53`
#;-) • `hours_per_week` -> `hours_per_week...54`
#;-) • `native_country` -> `native_country...55`
#;-) • `income` -> `income...56`
#;-) • `age` -> `age...57`
#;-) • `workclass` -> `workclass...58`
#;-) • `education` -> `education...59`
#;-) • `education_num` -> `education_num...60`
#;-) • `marital_status` -> `marital_status...61`
#;-) • `occupation` -> `occupation...62`
#;-) • `relationship` -> `relationship...63`
#;-) • `race` -> `race...64`
#;-) • `sex` -> `sex...65`
#;-) • `capital_gain` -> `capital_gain...66`
#;-) • `capital_loss` -> `capital_loss...67`
#;-) • `hours_per_week` -> `hours_per_week...68`
#;-) • `native_country` -> `native_country...69`
#;-) • `income` -> `income...70`
#;-) • `age` -> `age...71`
#;-) • `workclass` -> `workclass...72`
#;-) • `education` -> `education...73`
#;-) • `education_num` -> `education_num...74`
#;-) • `marital_status` -> `marital_status...75`
#;-) • `occupation` -> `occupation...76`
#;-) • `relationship` -> `relationship...77`
#;-) • `race` -> `race...78`
#;-) • `sex` -> `sex...79`
#;-) • `capital_gain` -> `capital_gain...80`
#;-) • `capital_loss` -> `capital_loss...81`
#;-) • `hours_per_week` -> `hours_per_week...82`
#;-) • `native_country` -> `native_country...83`
#;-) • `income` -> `income...84`
mean_multiple <- c(mean(data_multiple$age.1),
mean(data_multiple$age.2),
mean(data_multiple$age.3),
mean(data_multiple$age.4),
mean(data_multiple$age.5))
sample_dirty_dataset$age[is.na(sample_dirty_dataset$age)]<-mean(mean_multiple)
#Multiple inputation for hours per week
data_multiple <- complete(mice(sample_dirty_dataset, m = 5), action = "broad", include = TRUE)
#;-)
#;-) iter imp variable
#;-) 1 1 hours_per_week
#;-) 1 2 hours_per_week
#;-) 1 3 hours_per_week
#;-) 1 4 hours_per_week
#;-) 1 5 hours_per_week
#;-) 2 1 hours_per_week
#;-) 2 2 hours_per_week
#;-) 2 3 hours_per_week
#;-) 2 4 hours_per_week
#;-) 2 5 hours_per_week
#;-) 3 1 hours_per_week
#;-) 3 2 hours_per_week
#;-) 3 3 hours_per_week
#;-) 3 4 hours_per_week
#;-) 3 5 hours_per_week
#;-) 4 1 hours_per_week
#;-) 4 2 hours_per_week
#;-) 4 3 hours_per_week
#;-) 4 4 hours_per_week
#;-) 4 5 hours_per_week
#;-) 5 1 hours_per_week
#;-) 5 2 hours_per_week
#;-) 5 3 hours_per_week
#;-) 5 4 hours_per_week
#;-) 5 5 hours_per_week
#;-) Warning: Number of logged events: 9
#;-) New names:
#;-) • `age` -> `age...1`
#;-) • `workclass` -> `workclass...2`
#;-) • `education` -> `education...3`
#;-) • `education_num` -> `education_num...4`
#;-) • `marital_status` -> `marital_status...5`
#;-) • `occupation` -> `occupation...6`
#;-) • `relationship` -> `relationship...7`
#;-) • `race` -> `race...8`
#;-) • `sex` -> `sex...9`
#;-) • `capital_gain` -> `capital_gain...10`
#;-) • `capital_loss` -> `capital_loss...11`
#;-) • `hours_per_week` -> `hours_per_week...12`
#;-) • `native_country` -> `native_country...13`
#;-) • `income` -> `income...14`
#;-) • `age` -> `age...15`
#;-) • `workclass` -> `workclass...16`
#;-) • `education` -> `education...17`
#;-) • `education_num` -> `education_num...18`
#;-) • `marital_status` -> `marital_status...19`
#;-) • `occupation` -> `occupation...20`
#;-) • `relationship` -> `relationship...21`
#;-) • `race` -> `race...22`
#;-) • `sex` -> `sex...23`
#;-) • `capital_gain` -> `capital_gain...24`
#;-) • `capital_loss` -> `capital_loss...25`
#;-) • `hours_per_week` -> `hours_per_week...26`
#;-) • `native_country` -> `native_country...27`
#;-) • `income` -> `income...28`
#;-) • `age` -> `age...29`
#;-) • `workclass` -> `workclass...30`
#;-) • `education` -> `education...31`
#;-) • `education_num` -> `education_num...32`
#;-) • `marital_status` -> `marital_status...33`
#;-) • `occupation` -> `occupation...34`
#;-) • `relationship` -> `relationship...35`
#;-) • `race` -> `race...36`
#;-) • `sex` -> `sex...37`
#;-) • `capital_gain` -> `capital_gain...38`
#;-) • `capital_loss` -> `capital_loss...39`
#;-) • `hours_per_week` -> `hours_per_week...40`
#;-) • `native_country` -> `native_country...41`
#;-) • `income` -> `income...42`
#;-) • `age` -> `age...43`
#;-) • `workclass` -> `workclass...44`
#;-) • `education` -> `education...45`
#;-) • `education_num` -> `education_num...46`
#;-) • `marital_status` -> `marital_status...47`
#;-) • `occupation` -> `occupation...48`
#;-) • `relationship` -> `relationship...49`
#;-) • `race` -> `race...50`
#;-) • `sex` -> `sex...51`
#;-) • `capital_gain` -> `capital_gain...52`
#;-) • `capital_loss` -> `capital_loss...53`
#;-) • `hours_per_week` -> `hours_per_week...54`
#;-) • `native_country` -> `native_country...55`
#;-) • `income` -> `income...56`
#;-) • `age` -> `age...57`
#;-) • `workclass` -> `workclass...58`
#;-) • `education` -> `education...59`
#;-) • `education_num` -> `education_num...60`
#;-) • `marital_status` -> `marital_status...61`
#;-) • `occupation` -> `occupation...62`
#;-) • `relationship` -> `relationship...63`
#;-) • `race` -> `race...64`
#;-) • `sex` -> `sex...65`
#;-) • `capital_gain` -> `capital_gain...66`
#;-) • `capital_loss` -> `capital_loss...67`
#;-) • `hours_per_week` -> `hours_per_week...68`
#;-) • `native_country` -> `native_country...69`
#;-) • `income` -> `income...70`
#;-) • `age` -> `age...71`
#;-) • `workclass` -> `workclass...72`
#;-) • `education` -> `education...73`
#;-) • `education_num` -> `education_num...74`
#;-) • `marital_status` -> `marital_status...75`
#;-) • `occupation` -> `occupation...76`
#;-) • `relationship` -> `relationship...77`
#;-) • `race` -> `race...78`
#;-) • `sex` -> `sex...79`
#;-) • `capital_gain` -> `capital_gain...80`
#;-) • `capital_loss` -> `capital_loss...81`
#;-) • `hours_per_week` -> `hours_per_week...82`
#;-) • `native_country` -> `native_country...83`
#;-) • `income` -> `income...84`
mean_multiple <- c(mean(data_multiple$hours_per_week.1),
mean(data_multiple$hours_per_week.2),
mean(data_multiple$hours_per_week.3),
mean(data_multiple$hours_per_week.4),
mean(data_multiple$hours_per_week.5))
sample_dirty_dataset$hours_per_week[is.na(sample_dirty_dataset$hours_per_week)]<-mean(mean_multiple)
# Use summary() to see which variables have missing (NA) values
summary(sample_dirty_dataset)
#;-) age workclass education education_num
#;-) Min. :18.00 Length:1000 Length:1000 Min. : 1.000
#;-) 1st Qu.:38.00 Class :character Class :character 1st Qu.: 4.750
#;-) Median :54.41 Mode :character Mode :character Median : 8.000
#;-) Mean :54.45 Mean : 8.435
#;-) 3rd Qu.:71.00 3rd Qu.:12.000
#;-) Max. :90.00 Max. :16.000
#;-) marital_status occupation relationship race
#;-) Length:1000 Length:1000 Length:1000 Length:1000
#;-) Class :character Class :character Class :character Class :character
#;-) Mode :character Mode :character Mode :character Mode :character
#;-)
#;-)
#;-)
#;-) sex capital_gain capital_loss hours_per_week
#;-) Length:1000 Min. : 29 Min. : 35 Min. : 1.00
#;-) Class :character 1st Qu.:26000 1st Qu.:28245 1st Qu.:25.00
#;-) Mode :character Median :47945 Median :49733 Median :49.51
#;-) Mean :47922 Mean :49846 Mean :49.60
#;-) 3rd Qu.:68919 3rd Qu.:71771 3rd Qu.:74.00
#;-) Max. :99841 Max. :99820 Max. :99.00
#;-) native_country income
#;-) Length:1000 Length:1000
#;-) Class :character Class :character
#;-) Mode :character Mode :character
#;-)
#;-)
#;-)
# Number of rows in full dataset
nrow(sample_dirty_dataset)
#;-) [1] 1000
sample_clean_dataset <- na.omit(sample_dirty_dataset)
summary(sample_clean_dataset)
#;-) age workclass education education_num
#;-) Min. :18.00 Length:698 Length:698 Min. : 1.000
#;-) 1st Qu.:38.00 Class :character Class :character 1st Qu.: 5.000
#;-) Median :54.41 Mode :character Mode :character Median : 8.500
#;-) Mean :54.32 Mean : 8.477
#;-) 3rd Qu.:71.00 3rd Qu.:12.000
#;-) Max. :90.00 Max. :16.000
#;-) marital_status occupation relationship race
#;-) Length:698 Length:698 Length:698 Length:698
#;-) Class :character Class :character Class :character Class :character
#;-) Mode :character Mode :character Mode :character Mode :character
#;-)
#;-)
#;-)
#;-) sex capital_gain capital_loss hours_per_week
#;-) Length:698 Min. : 305 Min. : 35 Min. : 1.00
#;-) Class :character 1st Qu.:27444 1st Qu.:29489 1st Qu.:25.00
#;-) Mode :character Median :47945 Median :49733 Median :49.51
#;-) Mean :49194 Mean :50195 Mean :50.48
#;-) 3rd Qu.:70226 3rd Qu.:71862 3rd Qu.:75.00
#;-) Max. :99841 Max. :99820 Max. :99.00
#;-) native_country income
#;-) Length:698 Length:698
#;-) Class :character Class :character
#;-) Mode :character Mode :character
#;-)
#;-)
#;-)
nrow(sample_clean_dataset)
#;-) [1] 698#TASK 6 Anna Radosz ZSCORE OUTLIERS
library(dlookr)
library(mice)
### ZSCORE OUTLIERS CAPITAL LOSS
zscore <- abs(scale(sample_clean_dataset$capital_loss))
plot(zscore, type = "n", ylab = "Z-score")
abline(h = 2, col = "blue")
text(1:length(sample_clean_dataset$capital_loss), zscore, labels = round(zscore, 2))
title(main = "Capital loss") #BOX PLOT
library(dlookr)
library(mice)
### BOX PLOT CAPITAL GAIN
boxplot(sample_clean_dataset$capital_gain)
boxplot(sample_clean_dataset$capital_gain)$out
#;-) numeric(0)#OUTLIERS
library(dlookr)
library(mice)
library(robustbase)
#;-) Warning: pakiet 'robustbase' został zbudowany w wersji R 4.3.3
#;-)
#;-) Dołączanie pakietu: 'robustbase'
#;-) Następujący obiekt został zakryty _przez_ '.GlobalEnv':
#;-)
#;-) education
### OUTLIERS HOURS PER WEEK
par(mfrow=c(2,2))
plot(covMcd(sample_clean_dataset$hours_per_week))
#TASK 7 Anna
Radosz
library(dlookr)
# Standardization
summary(sample_clean_dataset)
#;-) age workclass education education_num
#;-) Min. :18.00 Length:698 Length:698 Min. : 1.000
#;-) 1st Qu.:38.00 Class :character Class :character 1st Qu.: 5.000
#;-) Median :54.41 Mode :character Mode :character Median : 8.500
#;-) Mean :54.32 Mean : 8.477
#;-) 3rd Qu.:71.00 3rd Qu.:12.000
#;-) Max. :90.00 Max. :16.000
#;-) marital_status occupation relationship race
#;-) Length:698 Length:698 Length:698 Length:698
#;-) Class :character Class :character Class :character Class :character
#;-) Mode :character Mode :character Mode :character Mode :character
#;-)
#;-)
#;-)
#;-) sex capital_gain capital_loss hours_per_week
#;-) Length:698 Min. : 305 Min. : 35 Min. : 1.00
#;-) Class :character 1st Qu.:27444 1st Qu.:29489 1st Qu.:25.00
#;-) Mode :character Median :47945 Median :49733 Median :49.51
#;-) Mean :49194 Mean :50195 Mean :50.48
#;-) 3rd Qu.:70226 3rd Qu.:71862 3rd Qu.:75.00
#;-) Max. :99841 Max. :99820 Max. :99.00
#;-) native_country income
#;-) Length:698 Length:698
#;-) Class :character Class :character
#;-) Mode :character Mode :character
#;-)
#;-)
#;-)
#We think that these variables doesn't need transformations#TASK 8 Szymon Wiśniewski
# The dlookr package provides various functions for data validation, exploration, and transformation.
# Includes functions:
# identify_outliers() - Identifies outliers in numeric variables
# impute_missing() - Provides tools for imputing missing values in datasets
# handle_duplicates() - Detects and handles duplicate rows in datasets.
# Mice is a package for multiple imputation of missing data in multivariate data sets.
# Includes functions:
# mice() - Performs multiple imputation by chained equations to impute missing values in datasets.
# complete() - Completes the imputation process and returns the imputed data set.
# pool() - Combines the results of multiple imputations into a single data set for further analysis.
# Tidyverse is a collection of R packages designed for data science, including data cleaning, visualization, and analysis.
# Includes functions:
# mutate() - Adds new variables or modifies existing variables in a data set based on specified transformations.
# summarize() - Computes summary statistics for variables in a data set, such as mean, median, or count.
# rename() - Renames variables in a data set to improve clarity or consistency.#TASK 9 Szymon Wiśniewski
# Data cleansing process consists of:
# Loading the Data set - loading the data set into R or other environment
# Exploring the Data set - examining the variables in the data set, their data types, and identifying any issues
# Data Cleaning - identifying and correcting any erroneous values in the data set.
# Data inconsistency correction - standardizing inconsistent values within categorical variables to ensure consistency across the data set.
# Missing data treatment - addressing missing values by imputing or removing them.
# Outlier handling - visualizing outliers and considering potential treatments, such as removal or imputation.
# Variable Transformations - transforming variables to achive standardization and normalization
# Data cleaning is a crucial step in the data analysis process as it ensures the reliability and accuracy of the results. It is essential for preparing datasets for analysis and decision-making. It requires careful examination, understanding of data characteristics, and application of appropriate techniques to ensure data quality.#TASK 10 Szymon Wiśniewski
library(titanic)
#;-) Warning: pakiet 'titanic' został zbudowany w wersji R 4.3.3
data("titanic_train")
library(dlookr)
#diagnostic report
diagnose_web_report(titanic_train)
#;-) processing file: diagnosis_temp.Rmd
#;-) output file: diagnosis_temp.knit.md
#;-) "C:/Program Files/RStudio/resources/app/bin/quarto/bin/tools/pandoc" +RTS -K512m -RTS diagnosis_temp.knit.md --to html4 --from markdown+autolink_bare_uris+tex_math_single_backslash --output pandoc10f8118575c3.html --lua-filter "C:\Users\N26\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\pagebreak.lua" --lua-filter "C:\Users\N26\AppData\Local\R\win-library\4.3\rmarkdown\rmarkdown\lua\latex-div.lua" --embed-resources --standalone --variable bs3=TRUE --section-divs --template "C:\Users\N26\AppData\Local\R\win-library\4.3\rmarkdown\rmd\h\default.html" --no-highlight --variable highlightjs=1 --variable theme=bootstrap --css "C:/Users/N26/AppData/Local/R/win-library/4.3/dlookr/resources/dlookr-bootstrap.css" --mathjax --variable "mathjax-url=https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" --include-in-header "C:\Users\N26\AppData\Local\Temp\Rtmp0ku5fK\rmarkdown-str10f84e6647a3.html" --variable code_folding=show --variable code_menu=1 --include-in-header header_temp.html --include-after-body "C:\Users\N26\AppData\Local\R\win-library\4.3\dlookr\resources\footer.html"
#;-)
#;-) Output created: C:\Users\N26\AppData\Local\Temp\Rtmp0ku5fK/Diagnosis_Report.html#Exploratory Data Analysis
library(titanic)
data("titanic_train")
library(dlookr)
#EDA report
eda_web_report(titanic_train, target="Age")
#;-) processing file: eda_temp.Rmd
#;-) Error in parse_block(g[-1], g[1], params.src, markdown_mode): Duplicate chunk label 'unnamed-chunk-2', which has been used for the chunk:
#;-) # Load necessary libraries
#;-) library(tidyverse)
#;-)
#;-) # Set seed for reproducibility
#;-) set.seed(2137)
#;-)
#;-) # Generate sample data
#;-) n <- 1000 # Number of observations
#;-)
#;-) # Age between 18 and 90
#;-) age <- sample(18:90, n, replace = TRUE)
#;-)
#;-) # Introduce errors in age column
#;-) age[sample(1:n, 50)] <- "InvalidAge"
#;-)
#;-) # Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
#;-) workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in workclass column
#;-) workclass[sample(1:n, 50)] <- NA
#;-)
#;-) # Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
#;-) education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
#;-)
#;-) # Introduce inconsistencies in education column
#;-) education[sample(1:n, 50)] <- "InvalidEducation"
#;-)
#;-) # Education Number: 1-16
#;-) education_num <- sample(1:16, n, replace = TRUE)
#;-)
#;-) # Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
#;-) marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in marital_status column
#;-) marital_status[sample(1:n, 50)] <- NA
#;-)
#;-) # Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
#;-) occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
#;-)
#;-) # Introduce errors in occupation column
#;-) occupation[sample(1:n, 50)] <- "InvalidOccupation"
#;-)
#;-) # Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
#;-) relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in relationship column
#;-) relationship[sample(1:n, 50)] <- NA
#;-)
#;-) # Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
#;-) race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
#;-)
#;-) # Introduce inconsistencies in race column
#;-) race[sample(1:n, 50)] <- "InvalidRace"
#;-)
#;-) # Sex: Female, Male
#;-) sex <- sample(c("Female", "Male"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in sex column
#;-) sex[sample(1:n, 50)] <- NA
#;-)
#;-) # Capital Gain: 0-99999
#;-) capital_gain <- sample(0:99999, n, replace = TRUE)
#;-)
#;-) # Introduce errors in capital_gain column
#;-) capital_gain[sample(1:n, 50)] <- -999
#;-)
#;-) # Capital Loss: 0-99999
#;-) capital_loss <- sample(0:99999, n, replace = TRUE)
#;-)
#;-) # Introduce errors in capital_loss column
#;-) capital_loss[sample(1:n, 50)] <- -999
#;-)
#;-) # Hours per week: 1-99
#;-) hours_per_week <- sample(1:99, n, replace = TRUE)
#;-)
#;-) # Introduce missing values in hours_per_week column
#;-) hours_per_week[sample(1:n, 50)] <- NA
#;-)
#;-) # Native country: United-States, Cambodia, England, ...
#;-) native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in native_country column
#;-) native_country[sample(1:n, 50)] <- NA
#;-)
#;-) # Income: <=50K, >50K
#;-) income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
#;-)
#;-) # Create dataframe
#;-) sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)
#;-)
#;-) # Print first few rows of the dataset
#;-) head(sample_dirty_dataset)#Transformation report
library(titanic)
data("titanic_train")
library(dlookr)
#diagnostic report
transformation_paged_report(titanic_train)
#;-) processing file: transformation_paged_temp.Rmd
#;-) Error in parse_block(g[-1], g[1], params.src, markdown_mode): Duplicate chunk label 'unnamed-chunk-2', which has been used for the chunk:
#;-) # Load necessary libraries
#;-) library(tidyverse)
#;-)
#;-) # Set seed for reproducibility
#;-) set.seed(2137)
#;-)
#;-) # Generate sample data
#;-) n <- 1000 # Number of observations
#;-)
#;-) # Age between 18 and 90
#;-) age <- sample(18:90, n, replace = TRUE)
#;-)
#;-) # Introduce errors in age column
#;-) age[sample(1:n, 50)] <- "InvalidAge"
#;-)
#;-) # Workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked
#;-) workclass <- sample(c("Private", "Self-emp-not-inc", "Self-emp-inc", "Federal-gov", "Local-gov", "State-gov", "Without-pay", "Never-worked"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in workclass column
#;-) workclass[sample(1:n, 50)] <- NA
#;-)
#;-) # Education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool
#;-) education <- sample(c("Bachelors", "Some-college", "11th", "HS-grad", "Prof-school", "Assoc-acdm", "Assoc-voc", "9th", "7th-8th", "12th", "Masters", "1st-4th", "10th", "Doctorate", "5th-6th", "Preschool"), n, replace = TRUE)
#;-)
#;-) # Introduce inconsistencies in education column
#;-) education[sample(1:n, 50)] <- "InvalidEducation"
#;-)
#;-) # Education Number: 1-16
#;-) education_num <- sample(1:16, n, replace = TRUE)
#;-)
#;-) # Marital Status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse
#;-) marital_status <- sample(c("Married-civ-spouse", "Divorced", "Never-married", "Separated", "Widowed", "Married-spouse-absent", "Married-AF-spouse"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in marital_status column
#;-) marital_status[sample(1:n, 50)] <- NA
#;-)
#;-) # Occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces
#;-) occupation <- sample(c("Tech-support", "Craft-repair", "Other-service", "Sales", "Exec-managerial", "Prof-specialty", "Handlers-cleaners", "Machine-op-inspct", "Adm-clerical", "Farming-fishing", "Transport-moving", "Priv-house-serv", "Protective-serv", "Armed-Forces"), n, replace = TRUE)
#;-)
#;-) # Introduce errors in occupation column
#;-) occupation[sample(1:n, 50)] <- "InvalidOccupation"
#;-)
#;-) # Relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried
#;-) relationship <- sample(c("Wife", "Own-child", "Husband", "Not-in-family", "Other-relative", "Unmarried"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in relationship column
#;-) relationship[sample(1:n, 50)] <- NA
#;-)
#;-) # Race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black
#;-) race <- sample(c("White", "Asian-Pac-Islander", "Amer-Indian-Eskimo", "Other", "Black"), n, replace = TRUE)
#;-)
#;-) # Introduce inconsistencies in race column
#;-) race[sample(1:n, 50)] <- "InvalidRace"
#;-)
#;-) # Sex: Female, Male
#;-) sex <- sample(c("Female", "Male"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in sex column
#;-) sex[sample(1:n, 50)] <- NA
#;-)
#;-) # Capital Gain: 0-99999
#;-) capital_gain <- sample(0:99999, n, replace = TRUE)
#;-)
#;-) # Introduce errors in capital_gain column
#;-) capital_gain[sample(1:n, 50)] <- -999
#;-)
#;-) # Capital Loss: 0-99999
#;-) capital_loss <- sample(0:99999, n, replace = TRUE)
#;-)
#;-) # Introduce errors in capital_loss column
#;-) capital_loss[sample(1:n, 50)] <- -999
#;-)
#;-) # Hours per week: 1-99
#;-) hours_per_week <- sample(1:99, n, replace = TRUE)
#;-)
#;-) # Introduce missing values in hours_per_week column
#;-) hours_per_week[sample(1:n, 50)] <- NA
#;-)
#;-) # Native country: United-States, Cambodia, England, ...
#;-) native_country <- sample(c("United-States", "Cambodia", "England", "Puerto-Rico", "Canada", "Germany", "Outlying-US(Guam-USVI-etc)", "India", "Japan", "Greece", "South", "China", "Cuba", "Iran", "Honduras", "Philippines", "Italy", "Poland", "Jamaica", "Vietnam", "Mexico", "Portugal", "Ireland", "France", "Dominican-Republic", "Laos", "Ecuador", "Taiwan", "Haiti", "Columbia", "Hungary", "Guatemala", "Nicaragua", "Scotland", "Thailand", "Yugoslavia", "El-Salvador", "Trinadad&Tobago", "Peru", "Hong", "Holand-Netherlands"), n, replace = TRUE)
#;-)
#;-) # Introduce missing values in native_country column
#;-) native_country[sample(1:n, 50)] <- NA
#;-)
#;-) # Income: <=50K, >50K
#;-) income <- sample(c("<=50K", ">50K"), n, replace = TRUE, prob = c(0.75, 0.25))
#;-)
#;-) # Create dataframe
#;-) sample_dirty_dataset <- data.frame(age, workclass, education, education_num, marital_status, occupation, relationship, race, sex, capital_gain, capital_loss, hours_per_week, native_country, income)
#;-)
#;-) # Print first few rows of the dataset
#;-) head(sample_dirty_dataset)✖ Install the styler package in order to use `style = TRUE`.
Komunikaty ostrzegawcze:
1: The closing backticks on line 369 ("````") in sample_data_reprex.Rmd do not match the opening backticks "```" on line 359. You are recommended to fix either the opening or closing delimiter of the code chunk to use exactly the same numbers of backticks and same level of indentation (or blockquote).
2: The closing backticks on line 383 ("````") in sample_data_reprex.Rmd do not match the opening backticks "```" on line 372. You are recommended to fix either the opening or closing delimiter of the code chunk to use exactly the same numbers of backticks and same level of indentation (or blockquote).
3: The closing backticks on line 397 ("````") in sample_data_reprex.Rmd do not match the opening backticks "```" on line 386. You are recommended to fix either the opening or closing delimiter of the code chunk to use exactly the same numbers of backticks and same level of indentation (or blockquote). sessionInfo()
#;-) R version 4.3.2 (2023-10-31 ucrt)
#;-) Platform: x86_64-w64-mingw32/x64 (64-bit)
#;-) Running under: Windows 11 x64 (build 22631)
#;-)
#;-) Matrix products: default
#;-)
#;-)
#;-) locale:
#;-) [1] LC_COLLATE=Polish_Poland.utf8 LC_CTYPE=Polish_Poland.utf8
#;-) [3] LC_MONETARY=Polish_Poland.utf8 LC_NUMERIC=C
#;-) [5] LC_TIME=Polish_Poland.utf8
#;-)
#;-) time zone: Europe/Warsaw
#;-) tzcode source: internal
#;-)
#;-) attached base packages:
#;-) [1] stats graphics grDevices utils datasets methods base
#;-)
#;-) other attached packages:
#;-) [1] htmltools_0.5.7 reactable_0.4.4 kableExtra_1.4.0 titanic_0.1.0
#;-) [5] robustbase_0.99-2 mice_3.16.0 validate_1.1.5 editrules_2.9.3
#;-) [9] igraph_2.0.3 dlookr_0.6.3 lubridate_1.9.3 forcats_1.0.0
#;-) [13] stringr_1.5.1 dplyr_1.1.4 purrr_1.0.2 readr_2.1.5
#;-) [17] tidyr_1.3.1 tibble_3.2.1 ggplot2_3.4.4 tidyverse_2.0.0
#;-)
#;-) loaded via a namespace (and not attached):
#;-) [1] gridExtra_2.3 lpSolveAPI_5.5.2.0-17.11 rlang_1.1.3
#;-) [4] magrittr_2.0.3 hrbrthemes_0.8.7 compiler_4.3.2
#;-) [7] systemfonts_1.0.5 vctrs_0.6.5 sysfonts_0.8.9
#;-) [10] httpcode_0.3.0 pkgconfig_2.0.3 shape_1.4.6.1
#;-) [13] crayon_1.5.2 fastmap_1.1.1 ellipsis_0.3.2
#;-) [16] backports_1.4.1 fontawesome_0.5.2 labeling_0.4.3
#;-) [19] utf8_1.2.4 promises_1.2.1 rmarkdown_2.25
#;-) [22] tzdb_0.4.0 nloptr_2.0.3 xfun_0.42
#;-) [25] glmnet_4.1-8 jomo_2.7-6 reprex_2.1.0
#;-) [28] cachem_1.0.8 showtext_0.9-7 jsonlite_1.8.8
#;-) [31] highr_0.10 later_1.3.2 pan_1.9
#;-) [34] broom_1.0.5 R6_2.5.1 bslib_0.6.1
#;-) [37] stringi_1.8.3 pagedown_0.20 boot_1.3-28.1
#;-) [40] extrafontdb_1.0 rpart_4.1.21 jquerylib_0.1.4
#;-) [43] bookdown_0.38 Rcpp_1.0.12 iterators_1.0.14
#;-) [46] knitr_1.45 base64enc_0.1-3 extrafont_0.19
#;-) [49] httpuv_1.6.15 Matrix_1.6-1.1 splines_4.3.2
#;-) [52] nnet_7.3-19 timechange_0.3.0 tidyselect_1.2.0
#;-) [55] rstudioapi_0.15.0 yaml_2.3.8 codetools_0.2-19
#;-) [58] curl_5.2.0 lattice_0.21-9 shiny_1.8.1.1
#;-) [61] withr_3.0.0 evaluate_0.23 survival_3.5-7
#;-) [64] xml2_1.3.6 pillar_1.9.0 settings_0.2.7
#;-) [67] foreach_1.5.2 generics_0.1.3 hms_1.1.3
#;-) [70] munsell_0.5.0 scales_1.3.0 minqa_1.2.6
#;-) [73] xtable_1.8-4 glue_1.7.0 gdtools_0.3.7
#;-) [76] tools_4.3.2 gfonts_0.2.0 lme4_1.1-35.2
#;-) [79] fs_1.6.3 grid_4.3.2 Rttf2pt1_1.3.12
#;-) [82] colorspace_2.1-0 nlme_3.1-163 showtextdb_3.0
#;-) [85] cli_3.6.2 fontBitstreamVera_0.1.1 fansi_1.0.6
#;-) [88] viridisLite_0.4.2 svglite_2.1.3 DEoptimR_1.1-3
#;-) [91] gtable_0.3.4 reactR_0.5.0 sass_0.4.8
#;-) [94] digest_0.6.34 fontquiver_0.2.1 crul_1.4.0
#;-) [97] farver_2.1.1 htmlwidgets_1.6.4 lifecycle_1.0.4
#;-) [100] mitml_0.4-5 mime_0.12 fontLiberation_0.1.0
#;-) [103] MASS_7.3-60