R Notebook

marks <- c(89, 95, 85, NA, 76, 69, 78, NA, 73, NA, 90, 69)

mean and median imputation

mean(marks, na.rm=TRUE)

[1] 80.44444

median(marks, na.rm = TRUE)

[1] 78

mean(marks[!is.na(marks)])

[1] 80.44444

pacman::p_load(Hmisc)

pacman::p_load(Hmisc)

Ratio

old_marks <- c(90, 100, 75, 89, 80, 77, 88, 69, 75, 88, 95, 70)

missing_location <- is.na(marks)
missing_location

 [1] FALSE FALSE FALSE  TRUE FALSE FALSE FALSE  TRUE FALSE  TRUE FALSE FALSE

sum(old_marks)

[1] 996

sum(marks, na.rm = TRUE)

[1] 724

pacman::p_load(Hmisc,VIM)

R <- sum(old_marks[!missing_location])/sum(marks, na.rm = TRUE)
R

[1] 1.035912

data("iris") # load the data
head(iris)

iris$Sepal.Length[1:10] <- NA
iris$Sepal.Length[1:10] <- NA
head(iris)

model <- lm(Sepal.Length ~ Sepal.Width + Petal.Length, data = iris)
model

Call:
lm(formula = Sepal.Length ~ Sepal.Width + Petal.Length, data = iris)

Coefficients:
 (Intercept)   Sepal.Width  Petal.Length  
      2.2716        0.5931        0.4689  

I <- is.na(iris$Sepal.Length)

iris$Sepal.Length[I] <- predict(model,newdata = iris[I,])
head(iris,10)

data(iris)

for (i in 1:ncol(iris)) {
 iris[sample(nrow(iris), 10, replace = FALSE),i] <- NA
}

summary(iris)

  Sepal.Length    Sepal.Width     Petal.Length    Petal.Width          Species  
 Min.   :4.300   Min.   :2.000   Min.   :1.000   Min.   :0.100   setosa    :43  
 1st Qu.:5.100   1st Qu.:2.800   1st Qu.:1.500   1st Qu.:0.250   versicolor:42  
 Median :5.800   Median :3.000   Median :4.300   Median :1.300   virginica :45  
 Mean   :5.861   Mean   :3.061   Mean   :3.728   Mean   :1.181   NA's      :20  
 3rd Qu.:6.400   3rd Qu.:3.300   3rd Qu.:5.100   3rd Qu.:1.800                  
 Max.   :7.900   Max.   :4.200   Max.   :6.900   Max.   :2.500                  
 NA's   :18      NA's   :19      NA's   :19      NA's   :19                     

iris2 <- kNN(iris)
summary(iris2)

  Sepal.Length    Sepal.Width    Petal.Length    Petal.Width        Species  
 Min.   :4.300   Min.   :2.00   Min.   :1.000   Min.   :0.1   setosa    :50  
 1st Qu.:5.100   1st Qu.:2.80   1st Qu.:1.600   1st Qu.:0.3   versicolor:50  
 Median :5.800   Median :3.00   Median :4.250   Median :1.3   virginica :50  
 Mean   :5.841   Mean   :3.06   Mean   :3.775   Mean   :1.2                  
 3rd Qu.:6.475   3rd Qu.:3.30   3rd Qu.:5.100   3rd Qu.:1.8                  
 Max.   :7.900   Max.   :4.20   Max.   :6.900   Max.   :2.5                  
 Sepal.Length_imp Sepal.Width_imp Petal.Length_imp Petal.Width_imp Species_imp    
 Mode :logical    Mode :logical   Mode :logical    Mode :logical   Mode :logical  
 FALSE:132        FALSE:131       FALSE:131        FALSE:131       FALSE:130      
 TRUE :18         TRUE :19        TRUE :19         TRUE :19        TRUE :20       
                                                                                  
                                                                                  
                                                                                  

iris2 <- iris2[1:5]
iris2 <- subset(iris2, select=Sepal.Leng)

dirty_iris <- read.csv("https://raw.githubusercontent.com/edwindj/datacleaning/master/data/dirty_iris.csv")

sum(is.na(dirty_iris$Petal.Length))

[1] 19

num_complete_cases <- sum(complete.cases(dirty_iris))
percent_complete_cases <- (num_complete_cases / nrow(dirty_iris)) * 100
cat("Number of complete observations:", num_complete_cases, "\n")

Number of complete observations: 96 

cat("Percentage of complete observations:", percent_complete_cases, "%\n")

Percentage of complete observations: 64 %

special_values <- sapply(dirty_iris, function(x) {
  if (is.numeric(x)) {
    list(
      Inf_values = sum(is.infinite(x)),
      NaN_values = sum(is.nan(x))
    )
  } else {
    NULL
  }
})
print(special_values)

$Sepal.Length
$Sepal.Length$Inf_values
[1] 0

$Sepal.Length$NaN_values
[1] 0


$Sepal.Width
$Sepal.Width$Inf_values
[1] 0

$Sepal.Width$NaN_values
[1] 0


$Petal.Length
$Petal.Length$Inf_values
[1] 0

$Petal.Length$NaN_values
[1] 0


$Petal.Width
$Petal.Width$Inf_values
[1] 1

$Petal.Width$NaN_values
[1] 0


$Species
NULL

dirty_iris[] <- lapply(dirty_iris, function(x) {
  if (is.numeric(x)) {
    x[is.infinite(x)] <- NA  # Replace Inf and -Inf with NA
    x[is.nan(x)] <- NA        # Replace NaN with NA
  }
  return(x)
})
sapply(dirty_iris, function(x) sum(is.na(x)))

Sepal.Length  Sepal.Width Petal.Length  Petal.Width      Species 
          10           17           19           13            0 

invalid_sepal_width <- dirty_iris$Sepal.Width <= 0
invalid_sepal_length <- dirty_iris$Sepal.Length > 30
violating_rows <- which(invalid_sepal_width | invalid_sepal_length)
num_violations <- length(violating_rows)
cat("Number of observations violating the rules:", num_violations, "\n")

Number of observations violating the rules: 4 

dirty_iris[violating_rows, ]

NA

dirty_iris$Sepal.Width[is.na(dirty_iris$Sepal.Width)] <- mean(dirty_iris$Sepal.Width, na.rm = TRUE)

dirty_iris$Petal.Length[is.na(dirty_iris$Petal.Length)] <- median(dirty_iris$Petal.Length, na.rm = TRUE)

model_sepal_length <- lm(Sepal.Length ~ Sepal.Width + Petal.Length + Petal.Width, data = dirty_iris)

dirty_iris$Sepal.Length[is.na(dirty_iris$Sepal.Length)] <- predict(model_sepal_length, newdata = dirty_iris[is.na(dirty_iris$Sepal.Length), ])