Assignment 4

dirty_iris <- read.csv("https://raw.githubusercontent.com/edwindj/datacleaning/master/data/dirty_iris.csv")

sum(is.na(dirty_iris$Petal.Length))

## [1] 19

num_complete <- sum(complete.cases(dirty_iris))

#Total Observations
num_total <- nrow(dirty_iris)

# % of Complete Observations
perc_complete <- (num_complete / num_total) * 100
num_complete

## [1] 96

perc_complete

## [1] 64

# Check for special values in numeric columns
sapply(dirty_iris[, 1:4], function(x) {
  c(
    NA_count   = sum(is.na(x)),
    NaN_count  = sum(is.nan(x)),
    Inf_count  = sum(is.infinite(x))
  )
})

##           Sepal.Length Sepal.Width Petal.Length Petal.Width
## NA_count            10          17           19          12
## NaN_count            0           0            0           0
## Inf_count            0           0            0           1

num_cols <- sapply(dirty_iris, is.numeric)
# Replace NaN with NA
dirty_iris[, num_cols][is.nan(as.matrix(dirty_iris[, num_cols]))] <- NA

# Replace Inf and -Inf with NA
dirty_iris[, num_cols][is.infinite(as.matrix(dirty_iris[, num_cols]))] <- NA

# Check if replacement worked
sapply(dirty_iris[, num_cols], function(x) {
  c(
    NA_count = sum(is.na(x)),
    NaN_count = sum(is.nan(x)),
    Inf_count = sum(is.infinite(x))
  )
})

##           Sepal.Length Sepal.Width Petal.Length Petal.Width
## NA_count            10          17           19          13
## NaN_count            0           0            0           0
## Inf_count            0           0            0           0

# Identify violations
violations <- subset(dirty_iris, Sepal.Width <= 0 | Sepal.Length > 30)

# View violating rows
violations

##     Sepal.Length Sepal.Width Petal.Length Petal.Width    Species
## 16           5.0          -3          3.5         1.0 versicolor
## 28          73.0          29         63.0          NA  virginica
## 125         49.0          30         14.0         2.0     setosa
## 130          5.7           0          1.7         0.3     setosa

# Count how many violations
nrow(violations)

## [1] 4

# Correct negative values: replace with Absolute Value
dirty_iris$Sepal.Width[which(dirty_iris$Sepal.Width < 0)] <- 
  abs(dirty_iris$Sepal.Width[which(dirty_iris$Sepal.Width < 0)])

# Correct Zero Values: replace with NA
dirty_iris$Sepal.Width[which(dirty_iris$Sepal.Width == 0)] <- NA

# Check the Corrected rows
dirty_iris[which(dirty_iris$Sepal.Width <= 0 | is.na(dirty_iris$Sepal.Width)), "Sepal.With"]

## NULL

# --- 1. Sepal.Width: mean imputation ---
dirty_iris$Sepal.Width[is.na(dirty_iris$Sepal.Width)] <- 
  mean(dirty_iris$Sepal.Width, na.rm = TRUE)

# --- 2. Petal.Length: median imputation ---
dirty_iris$Petal.Length[is.na(dirty_iris$Petal.Length)] <- 
  median(dirty_iris$Petal.Length, na.rm = TRUE)

# --- 3. Sepal.Length: linear regression imputation ---
lm_model <- lm(Sepal.Length ~ Sepal.Width + Petal.Length + Petal.Width + Species,
               data = dirty_iris, na.action = na.exclude)
predicted <- predict(lm_model, newdata = dirty_iris)
dirty_iris$Sepal.Length[is.na(dirty_iris$Sepal.Length)] <- 
  predicted[is.na(dirty_iris$Sepal.Length)]

# --- 4. Petal.Width: kNN imputation ---




# Check if missing values remain
colSums(is.na(dirty_iris))

## Sepal.Length  Sepal.Width Petal.Length  Petal.Width      Species 
##            0            0            0           13            0