Assignment 5 ~ Katie Quinn

library(Hmisc)

Registered S3 method overwritten by 'htmlwidgets':
  method           from         
  print.htmlwidget tools:rstudio
Registered S3 method overwritten by 'data.table':
  method           from
  print.data.table     

Attaching package: ‘Hmisc’

The following objects are masked from ‘package:base’:

    format.pval, units

library(dplyr)

Attaching package: ‘dplyr’

The following objects are masked from ‘package:Hmisc’:

    src, summarize

The following objects are masked from ‘package:stats’:

    filter, lag

The following objects are masked from ‘package:base’:

    intersect, setdiff, setequal, union

pacman::p_load(VIM)

dirty_iris <- read.csv("https://raw.githubusercontent.com/edwindj/datacleaning/master/data/dirty_iris.csv")

Question 1.

for (i in 1:ncol(dirty_iris)){
  iris[sample(1:nrow(dirty_iris), 10, replace = FALSE), i] <- NA
}

summary(dirty_iris)

  Sepal.Length     Sepal.Width      Petal.Length    Petal.Width    Species         
 Min.   : 0.000   Min.   :-3.000   Min.   : 0.00   Min.   :0.1   Length:150        
 1st Qu.: 5.100   1st Qu.: 2.800   1st Qu.: 1.60   1st Qu.:0.3   Class :character  
 Median : 5.750   Median : 3.000   Median : 4.50   Median :1.3   Mode  :character  
 Mean   : 6.559   Mean   : 3.391   Mean   : 4.45   Mean   :Inf                     
 3rd Qu.: 6.400   3rd Qu.: 3.300   3rd Qu.: 5.10   3rd Qu.:1.8                     
 Max.   :73.000   Max.   :30.000   Max.   :63.00   Max.   :Inf                     
 NA's   :10       NA's   :17       NA's   :19      NA's   :12                      

sum(is.na(dirty_iris$Petal.Length))

[1] 19

Question 2.

# Count total rows
total_rows <- nrow(dirty_iris)

# Count complete cases (rows with no missing values)
complete_rows <- sum(complete.cases(dirty_iris))

# Calculate percentage of complete observations
percentage_complete <- (complete_rows / total_rows) * 100

# Print results
cat("Number of complete cases:", complete_rows, "\n")

Number of complete cases: 96 

cat("Percentage of complete cases:", percentage_complete, "%\n")

Percentage of complete cases: 64 %

Question 3.

sapply(dirty_iris, function(x) {
  if (is.numeric(x)) {
    c(NA_values = sum(is.na(x)),
      NaN_values = sum(is.nan(x)),
      Inf_values = sum(x == Inf, na.rm = TRUE),
      Neg_Inf_values = sum(x == -Inf, na.rm = TRUE))
  }
})

$Sepal.Length
     NA_values     NaN_values     Inf_values Neg_Inf_values 
            10              0              0              0 

$Sepal.Width
     NA_values     NaN_values     Inf_values Neg_Inf_values 
            17              0              0              0 

$Petal.Length
     NA_values     NaN_values     Inf_values Neg_Inf_values 
            19              0              0              0 

$Petal.Width
     NA_values     NaN_values     Inf_values Neg_Inf_values 
            12              0              1              0 

$Species
NULL

Question 4.

dirty_iris[dirty_iris == Inf] <- NA

summary(dirty_iris)

  Sepal.Length     Sepal.Width      Petal.Length    Petal.Width      Species         
 Min.   : 0.000   Min.   :-3.000   Min.   : 0.00   Min.   :0.100   Length:150        
 1st Qu.: 5.100   1st Qu.: 2.800   1st Qu.: 1.60   1st Qu.:0.300   Class :character  
 Median : 5.750   Median : 3.000   Median : 4.50   Median :1.300   Mode  :character  
 Mean   : 6.559   Mean   : 3.391   Mean   : 4.45   Mean   :1.207                     
 3rd Qu.: 6.400   3rd Qu.: 3.300   3rd Qu.: 5.10   3rd Qu.:1.800                     
 Max.   :73.000   Max.   :30.000   Max.   :63.00   Max.   :2.500                     
 NA's   :10       NA's   :17       NA's   :19      NA's   :13                        

sapply(dirty_iris, function(x) {
  if (is.numeric(x)) {
    c(NA_values = sum(is.na(x)),
      NaN_values = sum(is.nan(x)),
      Inf_values = sum(x == Inf, na.rm = TRUE),
      Neg_Inf_values = sum(x == -Inf, na.rm = TRUE))
  }
})

$Sepal.Length
     NA_values     NaN_values     Inf_values Neg_Inf_values 
            10              0              0              0 

$Sepal.Width
     NA_values     NaN_values     Inf_values Neg_Inf_values 
            17              0              0              0 

$Petal.Length
     NA_values     NaN_values     Inf_values Neg_Inf_values 
            19              0              0              0 

$Petal.Width
     NA_values     NaN_values     Inf_values Neg_Inf_values 
            13              0              0              0 

$Species
NULL

Question 5.

num_negative_sepal_width <- sum(dirty_iris$Sepal.Width <= 0, na.rm = TRUE)
cat("Number of observations with negative Sepal.Width:", num_negative_sepal_width, "\n")

Number of observations with negative Sepal.Width: 2 

num_large_sepal_length <- sum(dirty_iris$Sepal.Length > 30, na.rm = TRUE)
cat("Number of observations with Sepal.Length > 30:", num_large_sepal_length, "\n")

Number of observations with Sepal.Length > 30: 2 

Question 6.

# Ensure no Inf or NaN values before imputation
dirty_iris[!is.finite(as.matrix(dirty_iris))] <- NA  

### 1. Mean Imputation for Sepal.Width
dirty_iris$Sepal.Width[is.na(dirty_iris$Sepal.Width)] <- mean(dirty_iris$Sepal.Width, na.rm = TRUE)

### 2. Median Imputation for Petal.Length
dirty_iris$Petal.Length[is.na(dirty_iris$Petal.Length)] <- median(dirty_iris$Petal.Length, na.rm = TRUE)

### 3. Linear Regression Imputation for Sepal.Length
# Check if there are enough complete cases for regression
complete_data <- dirty_iris %>% filter(!is.na(Sepal.Width) & !is.na(Petal.Length) & !is.na(Petal.Width))

if (nrow(complete_data) > 1) {  # Ensure enough data points for regression
    lm_model <- lm(Sepal.Length ~ Sepal.Width + Petal.Length + Petal.Width, data = complete_data)
    missing_sepal_length_rows <- which(is.na(dirty_iris$Sepal.Length))
    
    if (length(missing_sepal_length_rows) > 0) {
        dirty_iris$Sepal.Length[missing_sepal_length_rows] <- predict(lm_model, newdata = dirty_iris[missing_sepal_length_rows, ])
    }
} else {
    cat("Not enough complete cases for regression. Sepal.Length not imputed.\n")
}

Not enough complete cases for regression. Sepal.Length not imputed.

### 4. kNN Imputation for Petal.Width
# Select only numeric columns for kNN
numeric_cols <- dirty_iris %>% select(where(is.numeric))

# Perform kNN imputation on numeric columns
imputed_numeric <- kNN(numeric_cols, variable = "Petal.Width", k = 5)

Warning: All observations of Petal.Width are missing, therefore the variable will not be imputed!
Warning: Nothing is imputed, because all variables to be imputed only contains missings.

# Replace the original column with the imputed version
dirty_iris$Petal.Width <- imputed_numeric$Petal.Width

# Remove extra "_imp" columns added by kNN
dirty_iris <- dirty_iris %>% select(-contains("_imp"))

# Final Check: Count remaining missing values
cat("Remaining missing values after imputation:\n")

Remaining missing values after imputation:

print(colSums(is.na(dirty_iris)))

Sepal.Length  Sepal.Width Petal.Length  Petal.Width      Species 
         150          150          150          150          150 

abs(dirty_iris$Sepal.Width[dirty_iris$Sepal.Width < 0])

  [1] NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA
 [38] NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA
 [75] NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA
[112] NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA
[149] NA NA

Question 7.

dirty_iris[16,2] <- abs(dirty_iris[16, 2])
dirty_iris$Sepal.Width[dirty_iris$Sepal.Width == 0] <- NA


mean(dirty_iris$Sepal.Width, na.rm = TRUE) 

median(dirty_iris$Petal.Length, na.rm = TRUE)
median(dirty_iris$Petal.Length, na.rm = TRUE)

kNN(dirty_iris, variable = "Petal.Width")