options(repos = c(CRAN = "https://cran.rstudio.com/"))
install.packages("skimr")
## Installing package into 'C:/Users/HP/AppData/Local/R/win-library/4.6'
## (as 'lib' is unspecified)
## package 'skimr' successfully unpacked and MD5 sums checked
## 
## The downloaded binary packages are in
##  C:\Users\HP\AppData\Local\Temp\RtmpOsur75\downloaded_packages
install.packages("plotly")
## Installing package into 'C:/Users/HP/AppData/Local/R/win-library/4.6'
## (as 'lib' is unspecified)
## package 'plotly' successfully unpacked and MD5 sums checked
## 
## The downloaded binary packages are in
##  C:\Users\HP\AppData\Local\Temp\RtmpOsur75\downloaded_packages
install.packages("corrplot")
## Installing package into 'C:/Users/HP/AppData/Local/R/win-library/4.6'
## (as 'lib' is unspecified)
## package 'corrplot' successfully unpacked and MD5 sums checked
## 
## The downloaded binary packages are in
##  C:\Users\HP\AppData\Local\Temp\RtmpOsur75\downloaded_packages
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.2.1     ✔ readr     2.2.0
## ✔ forcats   1.0.1     ✔ stringr   1.6.0
## ✔ ggplot2   4.0.3     ✔ tibble    3.3.1
## ✔ lubridate 1.9.5     ✔ tidyr     1.3.2
## ✔ purrr     1.2.2     
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(skimr)
library(plotly)
## 
## Attaching package: 'plotly'
## 
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## 
## The following object is masked from 'package:stats':
## 
##     filter
## 
## The following object is masked from 'package:graphics':
## 
##     layout
library(corrplot)
## corrplot 0.95 loaded
#creating two small datasets to practice advance verbs
customers <- tibble(
  customer_id = 1:5,
full_name = c("John Doe","Jane Smith","Alice Johnson","Bob Brown","Charlse Davis"),
birth_year = c(1990,1985,1992,1980,1995),
birth_month = c("Jan","Mar","Jul","Dec","Oct")
)

sales <- tibble(
  transaction_id = 101:106,
  customer_id = c(1,2,2,3,4,5),  # Note: ID 99 doesn't exist in customers!
  amount = c(250.50,120.00,340.75,NA,50.00,900.00),
  store = c("North","South","South","East","West","North")
)
customers_clean <- customers %>%
  separate(col = full_name, 
           into = c("first_name", "last_name"), 
           sep = " ") # Separate wherever there is a space

head(customers_clean)
## # A tibble: 5 × 5
##   customer_id first_name last_name birth_year birth_month
##         <int> <chr>      <chr>          <dbl> <chr>      
## 1           1 John       Doe             1990 Jan        
## 2           2 Jane       Smith           1985 Mar        
## 3           3 Alice      Johnson         1992 Jul        
## 4           4 Bob        Brown           1980 Dec        
## 5           5 Charlse    Davis           1995 Oct
customers_clean <- customers_clean %>%
  unite(col = "birth_date", 
        birth_month, birth_year, 
        sep = "-")

head(customers_clean)
## # A tibble: 5 × 4
##   customer_id first_name last_name birth_date
##         <int> <chr>      <chr>     <chr>     
## 1           1 John       Doe       Jan-1990  
## 2           2 Jane       Smith     Mar-1985  
## 3           3 Alice      Johnson   Jul-1992  
## 4           4 Bob        Brown     Dec-1980  
## 5           5 Charlse    Davis     Oct-1995
sales
## # A tibble: 6 × 4
##   transaction_id customer_id amount store
##            <int>       <dbl>  <dbl> <chr>
## 1            101           1   250. North
## 2            102           2   120  South
## 3            103           2   341. South
## 4            104           3    NA  East 
## 5            105           4    50  West 
## 6            106           5   900  North
names(customers_clean)
## [1] "customer_id" "first_name"  "last_name"   "birth_date"
# Left Join
sales_left <- customers_clean %>%
  left_join(sales, by = "customer_id")

# Notice that transaction 106 (customer 99) has NAs for names because they aren't in the customer table!
print("Left Join Result:")
## [1] "Left Join Result:"
sales_left
## # A tibble: 6 × 7
##   customer_id first_name last_name birth_date transaction_id amount store
##         <dbl> <chr>      <chr>     <chr>               <int>  <dbl> <chr>
## 1           1 John       Doe       Jan-1990              101   250. North
## 2           2 Jane       Smith     Mar-1985              102   120  South
## 3           2 Jane       Smith     Mar-1985              103   341. South
## 4           3 Alice      Johnson   Jul-1992              104    NA  East 
## 5           4 Bob        Brown     Dec-1980              105    50  West 
## 6           5 Charlse    Davis     Oct-1995              106   900  North
# Full Join
sales_full <- sales %>%
  full_join(customers_clean, by = "customer_id")

# Notice that transaction 106 (customer 99) has NAs for names because they aren't in the customer table!
print("Full Join Result:")
## [1] "Full Join Result:"
sales_full
## # A tibble: 6 × 7
##   transaction_id customer_id amount store first_name last_name birth_date
##            <int>       <dbl>  <dbl> <chr> <chr>      <chr>     <chr>     
## 1            101           1   250. North John       Doe       Jan-1990  
## 2            102           2   120  South Jane       Smith     Mar-1985  
## 3            103           2   341. South Jane       Smith     Mar-1985  
## 4            104           3    NA  East  Alice      Johnson   Jul-1992  
## 5            105           4    50  West  Bob        Brown     Dec-1980  
## 6            106           5   900  North Charlse    Davis     Oct-1995
# Inner Join
sales_inner <- sales %>%
  inner_join(customers_clean, by = "customer_id")

# Notice transaction 106 is completely gone because customer 99 wasn't in both tables.
print("Inner Join Result:")
## [1] "Inner Join Result:"
sales_inner
## # A tibble: 6 × 7
##   transaction_id customer_id amount store first_name last_name birth_date
##            <int>       <dbl>  <dbl> <chr> <chr>      <chr>     <chr>     
## 1            101           1   250. North John       Doe       Jan-1990  
## 2            102           2   120  South Jane       Smith     Mar-1985  
## 3            103           2   341. South Jane       Smith     Mar-1985  
## 4            104           3    NA  East  Alice      Johnson   Jul-1992  
## 5            105           4    50  West  Bob        Brown     Dec-1980  
## 6            106           5   900  North Charlse    Davis     Oct-1995
# Let's create a quick summary table of total sales per store per customer
sales_summary <- sales_full %>%
  group_by(store, first_name) %>%
  summarize(total_spent = sum(amount, na.rm = TRUE)) %>%
  drop_na(first_name) # Remove the unknown customer
## `summarise()` has regrouped the output.
## ℹ Summaries were computed grouped by store and first_name.
## ℹ Output is grouped by store.
## ℹ Use `summarise(.groups = "drop_last")` to silence this message.
## ℹ Use `summarise(.by = c(store, first_name))` for per-operation grouping
##   (`?dplyr::dplyr_by`) instead.
sales_summary
## # A tibble: 5 × 3
## # Groups:   store [4]
##   store first_name total_spent
##   <chr> <chr>            <dbl>
## 1 East  Alice               0 
## 2 North Charlse           900 
## 3 North John              250.
## 4 South Jane              461.
## 5 West  Bob                50
# Make it WIDE (A matrix of Stores vs Customers)

wide_sales <- sales_summary %>%
  pivot_wider(names_from = store, values_from = total_spent)

print("Wide Format (Great for humans):")
## [1] "Wide Format (Great for humans):"
wide_sales
## # A tibble: 5 × 5
##   first_name  East North South  West
##   <chr>      <dbl> <dbl> <dbl> <dbl>
## 1 Alice          0   NA    NA     NA
## 2 Charlse       NA  900    NA     NA
## 3 John          NA  250.   NA     NA
## 4 Jane          NA   NA   461.    NA
## 5 Bob           NA   NA    NA     50
# Make it LONG again (Great for plotting)
long_sales <- wide_sales %>%
  pivot_longer(cols = -first_name, # Pivot all columns EXCEPT first_name
               names_to = "store", 
               values_to = "total_spent")

print("Long Format (Great for machines):")
## [1] "Long Format (Great for machines):"
long_sales
## # A tibble: 20 × 3
##    first_name store total_spent
##    <chr>      <chr>       <dbl>
##  1 Alice      East           0 
##  2 Alice      North         NA 
##  3 Alice      South         NA 
##  4 Alice      West          NA 
##  5 Charlse    East          NA 
##  6 Charlse    North        900 
##  7 Charlse    South         NA 
##  8 Charlse    West          NA 
##  9 John       East          NA 
## 10 John       North        250.
## 11 John       South         NA 
## 12 John       West          NA 
## 13 Jane       East          NA 
## 14 Jane       North         NA 
## 15 Jane       South        461.
## 16 Jane       West          NA 
## 17 Bob        East          NA 
## 18 Bob        North         NA 
## 19 Bob        South         NA 
## 20 Bob        West          50
# Look at our sales_full data. Bob Brown has an NA for amount.
# Let's replace his missing amount with 0.

sales_fixed <- sales_full %>%
  mutate(amount = replace_na(amount, 0))

sales_fixed
## # A tibble: 6 × 7
##   transaction_id customer_id amount store first_name last_name birth_date
##            <int>       <dbl>  <dbl> <chr> <chr>      <chr>     <chr>     
## 1            101           1   250. North John       Doe       Jan-1990  
## 2            102           2   120  South Jane       Smith     Mar-1985  
## 3            103           2   341. South Jane       Smith     Mar-1985  
## 4            104           3     0  East  Alice      Johnson   Jul-1992  
## 5            105           4    50  West  Bob        Brown     Dec-1980  
## 6            106           5   900  North Charlse    Davis     Oct-1995
## CLEANING MESSY DATA
messy_customers <- tibble(
  id = 1:4,
  name = c("   jOhn DOE  ", "alice  johnson", "BOB   BROWN", "  charlie davis "),
  phone = c("555-123-4567", "(555) 987 6543", "555.555.5555", "555 111-2222")
)

print("Look at this mess:")
## [1] "Look at this mess:"
messy_customers
## # A tibble: 4 × 3
##      id name               phone         
##   <int> <chr>              <chr>         
## 1     1 "   jOhn DOE  "    555-123-4567  
## 2     2 "alice  johnson"   (555) 987 6543
## 3     3 "BOB   BROWN"      555.555.5555  
## 4     4 "  charlie davis " 555 111-2222
clean_text_customers <- messy_customers %>%
  mutate(
    # 1. Fix the names: Squish the spaces, then make it Title Case
    name = str_squish(name),
    name = str_to_title(name),
    
    # 2. Fix the phones: Let's remove all dashes, dots, and parentheses
    
    # We replace them with an empty string "" (which deletes them)
    # The \\D is a special code (Regex) that means "Anything that is NOT a digit"
    
    phone = str_replace_all(phone, pattern = "\\D", replacement = "")
  )

print("Ah, much better:")
## [1] "Ah, much better:"
clean_text_customers
## # A tibble: 4 × 3
##      id name          phone     
##   <int> <chr>         <chr>     
## 1     1 John Doe      5551234567
## 2     2 Alice Johnson 5559876543
## 3     3 Bob Brown     5555555555
## 4     4 Charlie Davis 5551112222
### Exploratory Data Analysis (EDA)
my_eda_data <- diamonds 
# Calculating basic descriptive statistics for diamond prices
my_eda_data %>%
  summarize(
    Mean_Price = mean(price, na.rm = TRUE),
    Median_Price = median(price, na.rm = TRUE),
    Standard_Deviation = sd(price, na.rm = TRUE),
    Minimum = min(price, na.rm = TRUE),
    Maximum = max(price, na.rm = TRUE)
  )
## # A tibble: 1 × 5
##   Mean_Price Median_Price Standard_Deviation Minimum Maximum
##        <dbl>        <dbl>              <dbl>   <int>   <int>
## 1      3933.         2401              3989.     326   18823
# Calculating Quantiles and IQR for diamond prices
quantile(my_eda_data$price)
##       0%      25%      50%      75%     100% 
##   326.00   950.00  2401.00  5324.25 18823.00
# The IQR function
IQR(my_eda_data$price)
## [1] 4374.25
# Covariance & Correlation
# Calculating Correlation between Carat (weight) and Price
cor(my_eda_data$carat, my_eda_data$price)
## [1] 0.9215913
## Step 1: Data Overview
# 1. Glimpse shows dimensions, column names, data types, and a preview.
glimpse(my_eda_data)
## Rows: 53,940
## Columns: 10
## $ carat   <dbl> 0.23, 0.21, 0.23, 0.29, 0.31, 0.24, 0.24, 0.26, 0.22, 0.23, 0.…
## $ cut     <ord> Ideal, Premium, Good, Premium, Good, Very Good, Very Good, Ver…
## $ color   <ord> E, E, E, I, J, J, I, H, E, H, J, J, F, J, E, E, I, J, J, J, I,…
## $ clarity <ord> SI2, SI1, VS1, VS2, SI2, VVS2, VVS1, SI1, VS2, VS1, SI1, VS1, …
## $ depth   <dbl> 61.5, 59.8, 56.9, 62.4, 63.3, 62.8, 62.3, 61.9, 65.1, 59.4, 64…
## $ table   <dbl> 55, 61, 65, 58, 58, 57, 57, 55, 61, 61, 55, 56, 61, 54, 62, 58…
## $ price   <int> 326, 326, 327, 334, 335, 336, 336, 337, 337, 338, 339, 340, 34…
## $ x       <dbl> 3.95, 3.89, 4.05, 4.20, 4.34, 3.94, 3.95, 4.07, 3.87, 4.00, 4.…
## $ y       <dbl> 3.98, 3.84, 4.07, 4.23, 4.35, 3.96, 3.98, 4.11, 3.78, 4.05, 4.…
## $ z       <dbl> 2.43, 2.31, 2.31, 2.63, 2.75, 2.48, 2.47, 2.53, 2.49, 2.39, 2.…
# 2. Skim provides an incredibly detailed statistical summary and mini-histograms!
skim(my_eda_data)
Data summary
Name my_eda_data
Number of rows 53940
Number of columns 10
_______________________
Column type frequency:
factor 3
numeric 7
________________________
Group variables None

Variable type: factor

skim_variable n_missing complete_rate ordered n_unique top_counts
cut 0 1 TRUE 5 Ide: 21551, Pre: 13791, Ver: 12082, Goo: 4906
color 0 1 TRUE 7 G: 11292, E: 9797, F: 9542, H: 8304
clarity 0 1 TRUE 8 SI1: 13065, VS2: 12258, SI2: 9194, VS1: 8171

Variable type: numeric

skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist
carat 0 1 0.80 0.47 0.2 0.40 0.70 1.04 5.01 ▇▂▁▁▁
depth 0 1 61.75 1.43 43.0 61.00 61.80 62.50 79.00 ▁▁▇▁▁
table 0 1 57.46 2.23 43.0 56.00 57.00 59.00 95.00 ▁▇▁▁▁
price 0 1 3932.80 3989.44 326.0 950.00 2401.00 5324.25 18823.00 ▇▂▁▁▁
x 0 1 5.73 1.12 0.0 4.71 5.70 6.54 10.74 ▁▁▇▃▁
y 0 1 5.73 1.14 0.0 4.72 5.71 6.54 58.90 ▇▁▁▁▁
z 0 1 3.54 0.71 0.0 2.91 3.53 4.04 31.80 ▇▁▁▁▁ 
##Step 2: Univariate Analysis
###Analyze one variable at a time to understand its distribution (Centrality, Spread, Outliers).
# Statistical view: Frequency Table
my_eda_data %>% 
  count(cut, sort = TRUE) %>%
  mutate(percentage = (n / sum(n)) * 100)
## # A tibble: 5 × 3
##   cut           n percentage
##   <ord>     <int>      <dbl>
## 1 Ideal     21551      40.0 
## 2 Premium   13791      25.6 
## 3 Very Good 12082      22.4 
## 4 Good       4906       9.10
## 5 Fair       1610       2.98
# Visual view: Bar Chart
ggplot(my_eda_data, aes(x = cut)) +
  geom_bar(fill = "brown") +
  theme_minimal() +
  labs(title = "Distribution of Diamond Cuts", x = "Cut Quality", y = "Count")

# Histogram to see the curve (Is it a bell curve? Is it skewed?)
ggplot(my_eda_data, aes(x = price)) +
  geom_histogram(bins = 50, fill = "darkgreen", color = "darkblue") +
  theme_minimal() +
  labs(title = "Distribution of Diamond Prices (Highly Right-Skewed)")

# Boxplot to see outliers
ggplot(my_eda_data, aes(y = price)) +
  geom_boxplot(fill = "steelblue") +
  theme_minimal() +
  labs(title = "Boxplot showing Price Outliers")

###Step 3: Bivariate Analysis
##Analyze two variables together to find relationships 
##(Covariance and Correlation).
#3a. Numerical vs. Numerical
# Does an increase in X lead to an increase in Y? Use scatter plots.
ggplot(my_eda_data, aes(x = carat, y = price)) +
  geom_point(alpha = 0.1) + # alpha makes dots transparent to handle overplotting
  geom_smooth(method = "lm", color = "red") + # Adds a linear trend line
  theme_minimal() +
  labs(title = "Price vs. Carat: Strong Positive Correlation")
## `geom_smooth()` using formula = 'y ~ x'

###3b. Categorical vs. Numerical
##Compare the distribution of a continuous variable across different groups using side-by-side boxplots.
ggplot(my_eda_data, aes(x = cut, y = price, fill = cut)) +
  geom_boxplot() +
  theme_minimal() +
  labs(title = "Diamond Price Distribution by Cut Quality")

# Step 4: Multivariate & The Correlation Matrix
# 1. Select ONLY numerical columns
num_data <- my_eda_data %>% select(where(is.numeric))

# 2. Calculate the correlation matrix
cor_matrix <- cor(num_data)

# 3. Plot it using corrplot
corrplot(cor_matrix, method = "color", type = "upper", 
         addCoef.col = "black", # Adds the numerical coefficients
         tl.col = "black", tl.srt = 45, 
         diag = FALSE,
         title = "Correlation Matrix of Diamond Attributes",
         mar = c(0,0,2,0))