Question 3

Read data and find number of missing values in Petal.Length

dirty_iris <- read.csv("https://raw.githubusercontent.com/edwindj/datacleaning/master/data/dirty_iris.csv")

sum(is.na(dirty_iris$Petal.Length))
## [1] 19

Question 4

Number and percentage of complete observations

num_complete <- sum(complete.cases(dirty_iris))
print(num_complete)
## [1] 96
perc_complete <- (num_complete/150) * 100
print(perc_complete)
## [1] 64

Question 5

Check for special values

table(is.na(dirty_iris))
## 
## FALSE  TRUE 
##   692    58
str(dirty_iris)
## 'data.frame':    150 obs. of  5 variables:
##  $ Sepal.Length: num  6.4 6.3 6.2 5 5.7 5.3 6.4 5.9 5.8 4.8 ...
##  $ Sepal.Width : num  3.2 3.3 NA 3.4 2.6 NA 2.7 3 2.7 3.1 ...
##  $ Petal.Length: num  4.5 6 5.4 1.6 3.5 NA 5.3 5.1 4.1 1.6 ...
##  $ Petal.Width : num  1.5 2.5 2.3 0.4 1 0.2 NA 1.8 1 0.2 ...
##  $ Species     : chr  "versicolor" "virginica" "virginica" "setosa" ...
summary(dirty_iris)
##   Sepal.Length     Sepal.Width      Petal.Length    Petal.Width 
##  Min.   : 0.000   Min.   :-3.000   Min.   : 0.00   Min.   :0.1  
##  1st Qu.: 5.100   1st Qu.: 2.800   1st Qu.: 1.60   1st Qu.:0.3  
##  Median : 5.750   Median : 3.000   Median : 4.50   Median :1.3  
##  Mean   : 6.559   Mean   : 3.391   Mean   : 4.45   Mean   :Inf  
##  3rd Qu.: 6.400   3rd Qu.: 3.300   3rd Qu.: 5.10   3rd Qu.:1.8  
##  Max.   :73.000   Max.   :30.000   Max.   :63.00   Max.   :Inf  
##  NA's   :10       NA's   :17       NA's   :19      NA's   :12   
##    Species         
##  Length:150        
##  Class :character  
##  Mode  :character  
##                    
##                    
##                    
## 
# Check for Inf (Infinite) values
any(is.infinite(dirty_iris$Sepal.Length))
## [1] FALSE
any(is.infinite(dirty_iris$Sepal.Width))
## [1] FALSE
any(is.infinite(dirty_iris$Petal.Length))
## [1] FALSE
any(is.infinite(dirty_iris$Petal.Width))
## [1] TRUE
# Or check the whole numeric part of the dataframe at once
sapply(dirty_iris[1:4], function(x) sum(is.infinite(x)))
## Sepal.Length  Sepal.Width Petal.Length  Petal.Width 
##            0            0            0            1

Question 6

Replace Inf with NA

which(dirty_iris$Petal.Width=="Inf")
## [1] 86
dirty_iris$Petal.Width[which(dirty_iris$Petal.Width =="Inf")] <- NA
sum(which(dirty_iris$Petal.Width=="Inf"))
## [1] 0

Question 7

Fixing erroneous values

rule1_violation <- dirty_iris$Sepal.Width <= 0
rule2_violation <- dirty_iris$Sepal.Length > 30

# 2. Combine the rules using the OR operator (|) 
# because a row is an error if it fails EITHER rule.
# We use na.rm=TRUE or handle NAs because logical comparisons with NA result in NA
violations_mask <- (rule1_violation | rule2_violation)

# 3. Use which() to get the row indices, ignoring NAs
error_indices <- which(violations_mask)

# 4. Count the number of violating observations
length(error_indices)
## [1] 4
# 5. View the actual rows that are incorrect
dirty_iris[error_indices, ]
##     Sepal.Length Sepal.Width Petal.Length Petal.Width    Species
## 16           5.0          -3          3.5         1.0 versicolor
## 28          73.0          29         63.0          NA  virginica
## 125         49.0          30         14.0         2.0     setosa
## 130          5.7           0          1.7         0.3     setosa

Question 8

Find Sepal width <= 0 and make reasonable corrections.

zero_mask <- dirty_iris$Sepal.Width == 0 & !is.na(dirty_iris$Sepal.Width)
dirty_iris$Sepal.Width[zero_mask] <- NA

neg_indices <- which(dirty_iris$Sepal.Width < 0)
dirty_iris$Sepal.Width[neg_indices] <- abs(dirty_iris$Sepal.Width[neg_indices])

Question 9

Imputations

## Impute median method
dirty_iris$Sepal.Width[is.na(dirty_iris$Sepal.Width)] <- median(dirty_iris$Sepal.Width, na.rm=TRUE)

## Impute mean method
dirty_iris$Petal.Length[is.na(dirty_iris$Petal.Length)] <- mean(dirty_iris$Petal.Length , na.rm = TRUE)

# linear Regression
# inf values in petal.width create error
# Fill NAs in the predictors so the model has "X" values to work with
# install.packages("VIM")
library(VIM)
## Loading required package: colorspace
## Loading required package: grid
## VIM is ready to use.
## Suggestions and bug-reports can be submitted at: https://github.com/statistikat/VIM/issues
## 
## Attaching package: 'VIM'
## The following object is masked from 'package:datasets':
## 
##     sleep
# Check for NAs
sum(is.na(dirty_iris$Sepal.Width))
## [1] 0
sum(is.na(dirty_iris$Petal.Width))
## [1] 13
# Check for Infinite values
sum(is.infinite(dirty_iris$Sepal.Width))
## [1] 0
sum(is.infinite(dirty_iris$Petal.Width))
## [1] 0
dirty_iris$Sepal.Length[1:10] <- NA

model <- lm(Sepal.Length ~ Sepal.Width + Petal.Width, data = dirty_iris)
I <- is.na(dirty_iris$Sepal.Length)
dirty_iris$Sepal.Length[I] <- predict(model, newdata = dirty_iris[I, ])
#verify
sum(is.na(dirty_iris$Sepal.Length))
## [1] 1
## Impute kNN meathod

dirty_iris <- kNN(dirty_iris, variable = "Petal.Width", k = 5)
#Remove column Petal.Width_imp
dirty_iris$Petal.Width_imp <- NULL
#verify
sum(is.na(dirty_iris$Petal.Width))
## [1] 0