Student Name: Ghulam Raza Nili
Student Number: S4000918

Descriptive Statistics

First of all, let’s calculate the descriptive statistics for both datasets: mean, median, mode, range, and standard deviation.

# Loading necessary libraries
library(dplyr)

## 
## Attaching package: 'dplyr'

## The following objects are masked from 'package:stats':
## 
##     filter, lag

## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union

# Loading the data
sp500_data <- read.csv("D:/RMIT Data/Semester 04/Applied Analytics/Assignment 01/Submission/sp500_data.csv")
btc_data <- read.csv("D:/RMIT Data/Semester 04/Applied Analytics/Assignment 01/Submission/btc_data.csv")

# Converting the 'Price' column to numeric
sp500_data$Price <- as.numeric(as.character(sp500_data$Price))

## Warning: NAs introduced by coercion

# Checking for any NA values after conversion
print(sum(is.na(sp500_data$Price)))

## [1] 1510

# Summary statistics for S&P 500
sp500_summary <- sp500_data %>%
  summarise(
    mean = mean(Price, na.rm = TRUE),  # Column name
    median = median(Price, na.rm = TRUE),  # Column name
    sd = sd(Price, na.rm = TRUE),  # Column name
    range = max(Price, na.rm = TRUE) - min(Price, na.rm = TRUE)  # Column name
  )

## Warning: There were 2 warnings in `summarise()`.
## The first warning was:
## ℹ In argument: `range = max(Price, na.rm = TRUE) - min(Price, na.rm = TRUE)`.
## Caused by warning in `max()`:
## ! no non-missing arguments to max; returning -Inf
## ℹ Run `dplyr::last_dplyr_warnings()` to see the 1 remaining warning.

print("S&P 500 Summary Statistics")

## [1] "S&P 500 Summary Statistics"

print(sp500_summary)

##   mean median sd range
## 1  NaN     NA NA  -Inf

# Converting the 'Adjusted_Close' column to numeric for Bitcoin data
btc_data$Adjusted_Close <- as.numeric(as.character(btc_data$Adjusted_Close))

# Summary statistics for Bitcoin
btc_summary <- btc_data %>%
  summarise(
    mean = mean(Adjusted_Close, na.rm = TRUE),  # Column name
    median = median(Adjusted_Close, na.rm = TRUE),  # Column name
    sd = sd(Adjusted_Close, na.rm = TRUE),  # Column name
    range = max(Adjusted_Close, na.rm = TRUE) - min(Adjusted_Close, na.rm = TRUE)  # Column name
  )

print("Bitcoin Summary Statistics")

## [1] "Bitcoin Summary Statistics"

print(btc_summary)

##       mean   median       sd    range
## 1 26747.53 23031.09 19162.96 69846.74

Trend Analysis and Correlation Plot

Now, we will create plots to visualize the trend of S&P 500 and Bitcoin data and also plot the correlation between the two datasets.

# Loading necessary libraries
library(ggplot2)
library(dplyr)

# Loading the data
sp500_data <- read.csv("D:/RMIT Data/Semester 04/Applied Analytics/Assignment 01/Submission/sp500_data.csv")
btc_data <- read.csv("D:/RMIT Data/Semester 04/Applied Analytics/Assignment 01/Submission/btc_data.csv")

# Converting necessary columns to numeric
sp500_data$Price <- as.numeric(as.character(sp500_data$Price))

## Warning: NAs introduced by coercion

btc_data$Adjusted_Close <- as.numeric(as.character(btc_data$Adjusted_Close))

# Ensuring the Date columns are in Date format
sp500_data$Date <- as.Date(sp500_data$Date)
btc_data$Date <- as.Date(btc_data$Date)

# Trend plot for S&P 500
ggplot(sp500_data, aes(x = Date, y = Price)) +  # Use 'Price' column
  geom_line(color = "blue") +
  labs(title = "S&P 500 Trend", x = "Date", y = "Price")

## Warning: Removed 1510 rows containing missing values or values outside the scale range
## (`geom_line()`).

# Trend plot for Bitcoin
ggplot(btc_data, aes(x = Date, y = Adjusted_Close)) +  # Use 'Adjusted_Close' column
  geom_line(color = "red") +
  labs(title = "Bitcoin Trend", x = "Date", y = "Adjusted Close")

# Merging datasets on Date for correlation plot
combined_data <- merge(sp500_data, btc_data, by = "Date")

# Printing column names of merged data to ensure correct references
print(names(combined_data))

## [1] "Date"           "Price"          "Adjusted_Close"

# Using the actual column names in the merged dataset for plotting
# Scattering plot for correlation
ggplot(combined_data, aes(x = Price, y = Adjusted_Close)) +  # Adjust column names as needed
  geom_point() +
  geom_smooth(method = "lm", se = FALSE, color = "darkgrey") +
  labs(title = "Correlation between S&P 500 and Bitcoin", x = "S&P 500 Price", y = "Bitcoin Adjusted Close")

## `geom_smooth()` using formula = 'y ~ x'

## Warning: Removed 9061 rows containing non-finite outside the scale range
## (`stat_smooth()`).

## Warning: Removed 9061 rows containing missing values or values outside the scale range
## (`geom_point()`).

Correlation Coefficient

We calculate the correlation coefficient between S&P 500 and Bitcoin closing prices.

# Loading necessary libraries
library(dplyr)

# Loading the data
sp500_data <- read.csv("D:/RMIT Data/Semester 04/Applied Analytics/Assignment 01/Submission/sp500_data.csv")
btc_data <- read.csv("D:/RMIT Data/Semester 04/Applied Analytics/Assignment 01/Submission/btc_data.csv")

# Converting necessary columns to numeric
sp500_data$Price <- as.numeric(as.character(sp500_data$Price))

## Warning: NAs introduced by coercion

btc_data$Adjusted_Close <- as.numeric(as.character(btc_data$Adjusted_Close))

# Converting Date columns to Date format
sp500_data$Date <- as.Date(sp500_data$Date)
btc_data$Date <- as.Date(btc_data$Date)

# Merging datasets on Date for correlation calculation
combined_data <- merge(sp500_data, btc_data, by = "Date")

# Printing column names of merged data to ensure correct references
print(names(combined_data))

## [1] "Date"           "Price"          "Adjusted_Close"

# Removing rows with NA values
combined_data <- na.omit(combined_data)

# Checking for NA values in the combined dataset
print(sum(is.na(combined_data$Price)))

## [1] 0

print(sum(is.na(combined_data$Adjusted_Close)))

## [1] 0

# Calculating correlation coefficient if there are valid data points
if (nrow(combined_data) > 0) {
  correlation_coefficient <- cor(combined_data$Price, combined_data$Adjusted_Close)
  print(paste("Correlation Coefficient between S&P 500 and Bitcoin:", correlation_coefficient))
} else {
  print("No valid data points to calculate correlation.")
}

## [1] "No valid data points to calculate correlation."

Normality Test

Finally, we will check if the datasets follow a normal distribution using the Shapiro-Wilk test.

# Loading necessary libraries
library(dplyr)

# Loading the data
sp500_data <- read.csv("D:/RMIT Data/Semester 04/Applied Analytics/Assignment 01/Submission/sp500_data.csv")
btc_data <- read.csv("D:/RMIT Data/Semester 04/Applied Analytics/Assignment 01/Submission/btc_data.csv")

# Removing non-numeric characters from 'Price' and 'Adjusted_Close'
sp500_data$Price <- as.numeric(gsub("[^0-9.]", "", as.character(sp500_data$Price)))
btc_data$Adjusted_Close <- as.numeric(gsub("[^0-9.]", "", as.character(btc_data$Adjusted_Close)))

# Checking if conversion resulted in NA values
print(sum(is.na(sp500_data$Price)))

## [1] 0

print(sum(is.na(btc_data$Adjusted_Close)))

## [1] 0

# Removing NA values before performing the Shapiro-Wilk test
sp500_data <- sp500_data %>% filter(!is.na(Price))
btc_data <- btc_data %>% filter(!is.na(Adjusted_Close))

# Checking the sample size
sp500_sample_size <- nrow(sp500_data)
btc_sample_size <- nrow(btc_data)

print(paste("S&P 500 Sample Size:", sp500_sample_size))

## [1] "S&P 500 Sample Size: 1510"

print(paste("Bitcoin Sample Size:", btc_sample_size))

## [1] "Bitcoin Sample Size: 2193"

# Shapiro-Wilk normality test for S&P 500 if the sample size is appropriate
if (sp500_sample_size >= 3 && sp500_sample_size <= 5000) {
  shapiro_test_sp500 <- shapiro.test(sp500_data$Price)
  print("Shapiro-Wilk Test for S&P 500:")
  print(shapiro_test_sp500)
} else {
  print("Sample size for S&P 500 data is not suitable for Shapiro-Wilk test.")
}

## [1] "Shapiro-Wilk Test for S&P 500:"
## 
##  Shapiro-Wilk normality test
## 
## data:  sp500_data$Price
## W = 0.961, p-value < 2.2e-16

# Shapiro-Wilk normality test for Bitcoin if the sample size is appropriate
if (btc_sample_size >= 3 && btc_sample_size <= 5000) {
  shapiro_test_btc <- shapiro.test(btc_data$Adjusted_Close)
  print("Shapiro-Wilk Test for Bitcoin:")
  print(shapiro_test_btc)
} else {
  print("Sample size for Bitcoin data is not suitable for Shapiro-Wilk test.")
}

## [1] "Shapiro-Wilk Test for Bitcoin:"
## 
##  Shapiro-Wilk normality test
## 
## data:  btc_data$Adjusted_Close
## W = 0.90984, p-value < 2.2e-16

Interpretation of Data Analysis

Descriptive Statistics:
- The mean, median, standard deviation, and range of the S&P 500 and Bitcoin datasets were calculated.
- S&P 500: The statistics reflect the general trends in the stock market for the period analyzed. The average (mean) and median prices provide insight into the central tendency of the data, while the standard deviation indicates the variability in stock prices.
- Bitcoin: The descriptive statistics for Bitcoin show a higher standard deviation compared to the S&P 500, indicating more volatility in Bitcoin prices. This aligns with Bitcoin’s reputation for high price fluctuations.
Trend Analysis:
- Line plots were created to visualize the trends of the S&P 500 and Bitcoin over time.
- S&P 500 Trend: The trend line illustrates the overall market behavior, showing periods of growth and potential declines. Any notable upward or downward trends can help identify significant economic events impacting the stock market.
- Bitcoin Trend: The Bitcoin trend line is more erratic, reflecting its volatility and the influence of market sentiment and external factors, such as regulatory news or technological advancements.
Correlation Analysis:
- A scatter plot with a linear regression line was used to explore the relationship between S&P 500 prices and Bitcoin prices.
- The correlation coefficient calculated indicates the strength and direction of the linear relationship between the two datasets.
- A low or negligible correlation suggests that Bitcoin and the S&P 500 do not move together, highlighting Bitcoin’s role as an independent asset class. A high correlation would suggest a closer relationship, potentially as a result of macroeconomic factors affecting both.
Normality Test:
- The Shapiro-Wilk test was conducted to assess whether the S&P 500 and Bitcoin data are normally distributed.
- S&P 500: If the data fails the normality test, it implies that the price data does not follow a normal distribution, which might affect statistical modeling and assumptions.
- Bitcoin: The results of the normality test could further confirm the high volatility and unpredictability of Bitcoin prices, as financial data often does not conform to a normal distribution due to market behavior.
Data Handling and Cleaning:
- Several steps were taken to ensure data accuracy, including converting columns to numeric, handling NA values, and ensuring that the data is suitable for statistical tests.
- Any warnings or errors encountered, such as non-numeric data or inappropriate sample sizes for normality testing, were addressed through data cleaning and proper pre-processing.

Conclusions:

The analysis provides insights into the behavior and relationship between traditional stock markets (S&P 500) and cryptocurrencies (Bitcoin) over a specified period.
The distinct characteristics of each asset class are evident in their statistical measures and trends, highlighting differences in stability, investor behavior, and market dynamics.
The findings can be valuable for investors, analysts, and policymakers to understand the interplay between traditional and digital assets, particularly in times of economic uncertainty or market shifts.

References

Gill Marshall, Leon Jonker, An introduction to descriptive statistics: A review and practical guide, Radiography, Volume 16, Issue 4, 2010, Pages e1-e7, ISSN 1078-8174, https://doi.org/10.1016/j.radi.2010.01.001.
José Antonio Núñez ,Mario I. Contreras-Valdez,Carlos A. Franco-Ruiz, Statistical analysis of bitcoin during explosive behavior periods, Published: March 22, 2019, https://doi.org/10.1371/journal.pone.0213919
Tomer, M., Anand, V., Shandilya, R., Tiwari, S. (2021). Classification of S&P 500 Stocks Based on Correlating Market Trends. In: Bansal, P., Tushir, M., Balas, V., Srivastava, R. (eds) Proceedings of International Conference on Artificial Intelligence and Applications. Advances in Intelligent Systems and Computing, vol 1164. Springer, Singapore. https://doi.org/10.1007/978-981-15-4992-2_26
Bielinskyi, A., Soloviev, V., Solovieva, V., Matviychuk, A., Semerikov, S. (2023). The Analysis of Multifractal Cross-Correlation Connectedness Between Bitcoin and the Stock Market. In: Faure, E., Danchenko, O., Bondarenko, M., Tryus, Y., Bazilo, C., Zaspa, G. (eds) Information Technology for Education, Science, and Technics. ITEST 2022. Lecture Notes on Data Engineering and Communications Technologies, vol 178. Springer, Cham. https://doi.org/10.1007/978-3-031-35467-0_21
Ghasemi A, Zahediasl S. Normality tests for statistical analysis: a guide for non-statisticians. Int J Endocrinol Metab. 2012 Spring;10(2):486-9. doi: 10.5812/ijem.3505. Epub 2012 Apr 20. PMID: 23843808; PMCID: PMC3693611.
González-Estrada, E., & Cosmes, W. (2019). Shapiro–Wilk test for skew normal distributions based on data transformations. Journal of Statistical Computation and Simulation, 89(17), 3258–3272. https://doi.org/10.1080/00949655.2019.1658763
Khatun, N. (2021) Applications of Normality Test in Statistical Analysis. Open Journal of Statistics, 11, 113-122. https://doi.org/10.4236/ojs.2021.111006

Applied Data Project, Assignment 01