Assignment4

Q3

# Load dataset
dirty_iris <- read.csv("https://raw.githubusercontent.com/edwindj/datacleaning/master/data/dirty_iris.csv")

# View column names
names(dirty_iris)

## [1] "Sepal.Length" "Sepal.Width"  "Petal.Length" "Petal.Width"  "Species"

# Count missing values in Petal.Length
sum(is.na(dirty_iris$Petal.Length))

## [1] 19

Q4

# Number of complete observations
sum(complete.cases(dirty_iris))

## [1] 96

# Percentage of complete observations
sum(complete.cases(dirty_iris)) / nrow(dirty_iris) * 100

## [1] 64

Q5

# Check for special numeric values besides NA
sapply(dirty_iris, function(x) sum(is.nan(x)))     # Check for NaN

## Sepal.Length  Sepal.Width Petal.Length  Petal.Width      Species 
##            0            0            0            0            0

sapply(dirty_iris, function(x) sum(is.infinite(x))) # Check for Inf or -Inf

## Sepal.Length  Sepal.Width Petal.Length  Petal.Width      Species 
##            0            0            0            1            0

Q6

# Locate and replace Inf values with NA
dirty_iris[is.infinite(dirty_iris$Petal.Width), "Petal.Width"] <- NA

# Verify replacement
sum(is.infinite(dirty_iris$Petal.Width))

## [1] 0

sum(is.na(dirty_iris$Petal.Width))

## [1] 13

Q7

# Find observations that violate the rules
violations <- subset(dirty_iris, Sepal.Width <= 0 | Sepal.Length > 30)

# Count how many rows violate the rules
nrow(violations)

## [1] 4

# (Optional) View the violating records
violations

##     Sepal.Length Sepal.Width Petal.Length Petal.Width    Species
## 16           5.0          -3          3.5         1.0 versicolor
## 28          73.0          29         63.0          NA  virginica
## 125         49.0          30         14.0         2.0     setosa
## 130          5.7           0          1.7         0.3     setosa

Q8

# Locate and correct Sepal.Width values that violate the rule (> 0)

# Convert negative Sepal.Width values to their absolute value, ignoring NAs
dirty_iris$Sepal.Width[!is.na(dirty_iris$Sepal.Width) & dirty_iris$Sepal.Width < 0] <- 
  abs(dirty_iris$Sepal.Width[!is.na(dirty_iris$Sepal.Width) & dirty_iris$Sepal.Width < 0])

# Replace Sepal.Width values equal to 0 with NA
dirty_iris$Sepal.Width[!is.na(dirty_iris$Sepal.Width) & dirty_iris$Sepal.Width == 0] <- NA

# Verify corrections
summary(dirty_iris$Sepal.Width)

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max.    NA's 
##   2.200   2.800   3.000   3.462   3.300  30.000      18

Q9

# Install VIM
if(!require(VIM)) install.packages("VIM", dependencies = TRUE)

## Loading required package: VIM

## Loading required package: colorspace

## Loading required package: grid

## VIM is ready to use.

## Suggestions and bug-reports can be submitted at: https://github.com/statistikat/VIM/issues

## 
## Attaching package: 'VIM'

## The following object is masked from 'package:datasets':
## 
##     sleep

library(VIM)

# A) Sepal.Width: mean imputation
dirty_iris$Sepal.Width[is.na(dirty_iris$Sepal.Width)] <- 
  mean(dirty_iris$Sepal.Width, na.rm = TRUE)

# B) Petal.Length: median imputation
dirty_iris$Petal.Length[is.na(dirty_iris$Petal.Length)] <- 
  median(dirty_iris$Petal.Length, na.rm = TRUE)

# C) Sepal.Length: linear regression imputation
lm_model <- lm(Sepal.Length ~ Sepal.Width + Petal.Length + Petal.Width + Species, 
               data = dirty_iris)
dirty_iris$Sepal.Length[is.na(dirty_iris$Sepal.Length)] <- 
  predict(lm_model, newdata = dirty_iris[is.na(dirty_iris$Sepal.Length), ])

# D) Petal.Width: kNN imputation
dirty_iris <- VIM::kNN(dirty_iris, variable = "Petal.Width", k = 5, imp_var = FALSE)

## Sepal.Length  Sepal.Width Petal.Length Sepal.Length  Sepal.Width Petal.Length 
##          0.0          2.2          0.0         73.0         30.0         63.0

# Verify no missing values remain
sapply(dirty_iris, function(x) sum(is.na(x)))

## Sepal.Length  Sepal.Width Petal.Length  Petal.Width      Species 
##            0            0            0            0            0