lab4

lab4

diagnose(sample_dirty_dataset) %>% flextable()

#setting age as integer and it will automatically change invalidAge as N/A
sample_dirty_dataset$age <- as.integer(sample_dirty_dataset$age)

# columns that contains N/A: workclass, martial_status, reationship, sex, hours_per_week, native_country, 
# age -> invalidAge
# education -> InvalidEducation
# occupation -> InvalidOccupation 
# race -> invalidRace
# capital gain/loss -> negative numbers

#Changing all invaild data to NA
sample_dirty_dataset$age[sample_dirty_dataset$age == "invalidAge"] <- NA
sample_dirty_dataset$education[sample_dirty_dataset$education == "InvalidEducation"] <- NA
sample_dirty_dataset$occupation[sample_dirty_dataset$occupation == "InvalidOccupation"] <- NA
sample_dirty_dataset$race[sample_dirty_dataset$race == "InvalidRace"] <- NA
sample_dirty_dataset$capital_gain[sample_dirty_dataset$capital_gain < 0] <- NA
sample_dirty_dataset$capital_loss[sample_dirty_dataset$capital_loss < 0] <- NA

diagnose(sample_dirty_dataset) %>% flextable()

plot_na_hclust(sample_dirty_dataset)

plot_na_intersect(sample_dirty_dataset)

# changing character data to factors
sample_dirty_dataset <- sample_dirty_dataset %>%
    mutate_if(is.character, as.factor)

new_data <- complete(mice(sample_dirty_dataset, m = 1, method ="cart"))

## 
##  iter imp variable
##   1   1  age  workclass  education  marital_status  occupation  relationship  race  sex  capital_gain  capital_loss  hours_per_week  native_country
##   2   1  age  workclass  education  marital_status  occupation  relationship  race  sex  capital_gain  capital_loss  hours_per_week  native_country
##   3   1  age  workclass  education  marital_status  occupation  relationship  race  sex  capital_gain  capital_loss  hours_per_week  native_country
##   4   1  age  workclass  education  marital_status  occupation  relationship  race  sex  capital_gain  capital_loss  hours_per_week  native_country
##   5   1  age  workclass  education  marital_status  occupation  relationship  race  sex  capital_gain  capital_loss  hours_per_week  native_country

diagnose(sample_dirty_dataset) %>% flextable()

diagnose(new_data) %>% flextable()

#all numeric,integer and factor missing values are gone!

#Detecting outliers
#1 Method
boxplot(new_data$age)

boxplot(new_data$capital_gain)

boxplot(new_data$capital_loss)

boxplot(new_data$hours_per_week)

#2 Method
diagnose_outlier(new_data)

##        variables outliers_cnt outliers_ratio outliers_mean with_mean
## 1            age            0              0           NaN    53.886
## 2  education_num            0              0           NaN     8.345
## 3   capital_gain            0              0           NaN 49733.000
## 4   capital_loss            0              0           NaN 48718.277
## 5 hours_per_week            0              0           NaN    48.087
##   without_mean
## 1       53.886
## 2        8.345
## 3    49733.000
## 4    48718.277
## 5       48.087

#3 Method
lower_bound <- quantile(new_data$capital_gain, 0.025)
upper_bound <- quantile(new_data$capital_gain, 0.975)
outlier_ind <- which(new_data$capital_gain < lower_bound | new_data$capital_gain > upper_bound)
new_data[outlier_ind, "capital_gain"]

##  [1]  2553 97981   990 99180 97595  1097 98599 98953  2076 98168 98978  2603
## [13]  2557   550  1536 99134   581 98754 99960  1256 97315   872  1277 98687
## [25] 98302  2271 99342   127 97555  1524   419 99257 99411   232 98660 98314
## [37]  2136 97283 97939 97315  2207 97555 99839   403   530  1569  1524  1536
## [49] 98540    20

#Transformation may not be needed it all depends what we need our data for.
#Lets assume we need to transform age. Dont know why we would like to do it but whatever

head(new_data)%>% flextable() #dataframe before transformatiom

df <- new_data

bc_out <- boxcox(age ~ 1, data = df, lambda = seq(0.01, 2, by = 0.1))

opt_lambda <- bc_out$x[which.max(bc_out$y)]  #Choosing optimal lambda

df$age_transformed <- (df$age^opt_lambda - 1) / opt_lambda #Transformation

head(df) %>% flextable() #Dataframe after transformation of age table

Functions used: diagnose() - Diagnoses data quality with a focus on missing values, outliers, and invalid values.

flextable() - creation of tables.

as.integer() - Converts data to integer format; non-integer -> NA.

mutate_if() - Modifies columns that meet a condition.

complete() - imputation on missing data.

plot_na_hclust() - Visualizes the patterns of missing data.

plot_na_intersect() - Visualizes the intersections of missing data.

boxplot() - Creates a box plot

diagnose_outlier() (dlookr package) - Diagnoses outliers

quantile() - Computes the quantiles of data.

which() - Used to select only data which matches condition.

boxcox() (MASS package) - Performs Box-cox, plots a chart so we can choose optimal lamda for futher transformation.

Steps:

Issue Identification - Use diagnose() to highlight data quality issues.

Handling Invalid Data - Replace error entries with NA.

Data Type Corrections - Cast age to integers, throw out non-integer values.

Missing Values Analysis - Visualize missing data patterns and correlations.

Data Imputation - Use mice for missing data imputation.

Data Transformation - Use Box-Cox transformation for normality.

Outlier Detection - Identify extremes using box plots and quantiles.

df %>% diagnose_web_report()
df %>% eda_paged_report(output_format = "html")
df %>% transformation_report(output_format = "html")