Loading libraries
library(dplyr)
##
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
##
## filter, lag
## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union
library(tidyr)
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ forcats 1.0.1 ✔ readr 2.2.0
## ✔ ggplot2 4.0.2 ✔ stringr 1.6.0
## ✔ lubridate 1.9.5 ✔ tibble 3.3.1
## ✔ purrr 1.2.1
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(ggplot2)
data <- read_csv("netflix_titles.csv")
## Rows: 8807 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (11): show_id, type, title, director, cast, country, date_added, rating,...
## dbl (1): release_year
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
data
## # A tibble: 8,807 × 12
## show_id type title director cast country date_added release_year rating
## <chr> <chr> <chr> <chr> <chr> <chr> <chr> <dbl> <chr>
## 1 s1 Movie Dick J… Kirsten… <NA> United… September… 2020 PG-13
## 2 s2 TV Show Blood … <NA> Ama … South … September… 2021 TV-MA
## 3 s3 TV Show Gangla… Julien … Sami… <NA> September… 2021 TV-MA
## 4 s4 TV Show Jailbi… <NA> <NA> <NA> September… 2021 TV-MA
## 5 s5 TV Show Kota F… <NA> Mayu… India September… 2021 TV-MA
## 6 s6 TV Show Midnig… Mike Fl… Kate… <NA> September… 2021 TV-MA
## 7 s7 Movie My Lit… Robert … Vane… <NA> September… 2021 PG
## 8 s8 Movie Sankofa Haile G… Kofi… United… September… 1993 TV-MA
## 9 s9 TV Show The Gr… Andy De… Mel … United… September… 2021 TV-14
## 10 s10 Movie The St… Theodor… Meli… United… September… 2021 PG-13
## # ℹ 8,797 more rows
## # ℹ 3 more variables: duration <chr>, listed_in <chr>, description <chr>
Understanding the data
head(data)
## # A tibble: 6 × 12
## show_id type title director cast country date_added release_year rating
## <chr> <chr> <chr> <chr> <chr> <chr> <chr> <dbl> <chr>
## 1 s1 Movie Dick Jo… Kirsten… <NA> United… September… 2020 PG-13
## 2 s2 TV Show Blood &… <NA> Ama … South … September… 2021 TV-MA
## 3 s3 TV Show Ganglan… Julien … Sami… <NA> September… 2021 TV-MA
## 4 s4 TV Show Jailbir… <NA> <NA> <NA> September… 2021 TV-MA
## 5 s5 TV Show Kota Fa… <NA> Mayu… India September… 2021 TV-MA
## 6 s6 TV Show Midnigh… Mike Fl… Kate… <NA> September… 2021 TV-MA
## # ℹ 3 more variables: duration <chr>, listed_in <chr>, description <chr>
str(data)
## spc_tbl_ [8,807 × 12] (S3: spec_tbl_df/tbl_df/tbl/data.frame)
## $ show_id : chr [1:8807] "s1" "s2" "s3" "s4" ...
## $ type : chr [1:8807] "Movie" "TV Show" "TV Show" "TV Show" ...
## $ title : chr [1:8807] "Dick Johnson Is Dead" "Blood & Water" "Ganglands" "Jailbirds New Orleans" ...
## $ director : chr [1:8807] "Kirsten Johnson" NA "Julien Leclercq" NA ...
## $ cast : chr [1:8807] NA "Ama Qamata, Khosi Ngema, Gail Mabalane, Thabang Molaba, Dillon Windvogel, Natasha Thahane, Arno Greeff, Xolile "| __truncated__ "Sami Bouajila, Tracy Gotoas, Samuel Jouy, Nabiha Akkari, Sofia Lesaffre, Salim Kechiouche, Noureddine Farihi, G"| __truncated__ NA ...
## $ country : chr [1:8807] "United States" "South Africa" NA NA ...
## $ date_added : chr [1:8807] "September 25, 2021" "September 24, 2021" "September 24, 2021" "September 24, 2021" ...
## $ release_year: num [1:8807] 2020 2021 2021 2021 2021 ...
## $ rating : chr [1:8807] "PG-13" "TV-MA" "TV-MA" "TV-MA" ...
## $ duration : chr [1:8807] "90 min" "2 Seasons" "1 Season" "1 Season" ...
## $ listed_in : chr [1:8807] "Documentaries" "International TV Shows, TV Dramas, TV Mysteries" "Crime TV Shows, International TV Shows, TV Action & Adventure" "Docuseries, Reality TV" ...
## $ description : chr [1:8807] "As her father nears the end of his life, filmmaker Kirsten Johnson stages his death in inventive and comical wa"| __truncated__ "After crossing paths at a party, a Cape Town teen sets out to prove whether a private-school swimming star is h"| __truncated__ "To protect his family from a powerful drug lord, skilled thief Mehdi and his expert team of robbers are pulled "| __truncated__ "Feuds, flirtations and toilet talk go down among the incarcerated women at the Orleans Justice Center in New Or"| __truncated__ ...
## - attr(*, "spec")=
## .. cols(
## .. show_id = col_character(),
## .. type = col_character(),
## .. title = col_character(),
## .. director = col_character(),
## .. cast = col_character(),
## .. country = col_character(),
## .. date_added = col_character(),
## .. release_year = col_double(),
## .. rating = col_character(),
## .. duration = col_character(),
## .. listed_in = col_character(),
## .. description = col_character()
## .. )
## - attr(*, "problems")=<externalptr>
Missing Values
colSums(is.na(data))
## show_id type title director cast country
## 0 0 0 2634 825 831
## date_added release_year rating duration listed_in description
## 10 0 4 3 0 0
Removing Duplicates
data <- data[!duplicated(data), ]
Data Type Conversion(to have categorical data)
data$type <- as.factor(data$type)
data$rating <- as.factor(data$rating)
New variables created for deeper analysis
data <- data %>%
mutate(
date_added = as.Date(date_added, format="%B %d, %Y"),
year_added = as.numeric(format(date_added, "%Y")),
duration_num = as.numeric(gsub("[^0-9]", "", duration))
)
What is the average release year for Movies vs TV Shows?
data %>%
group_by(type) %>%
summarise(avg_year = mean(release_year, na.rm = TRUE))
## # A tibble: 2 × 2
## type avg_year
## <fct> <dbl>
## 1 Movie 2013.
## 2 TV Show 2017.
Which countries produce the most content?
data$country[data$country == ""] <- "Unknown"
data %>%
count(country, sort = TRUE) %>%
head(5)
## # A tibble: 5 × 2
## country n
## <chr> <int>
## 1 United States 2818
## 2 India 972
## 3 <NA> 831
## 4 United Kingdom 419
## 5 Japan 245
What are the most popular genres?
data %>%
count(listed_in, sort = TRUE) %>%
head(5)
## # A tibble: 5 × 2
## listed_in n
## <chr> <int>
## 1 Dramas, International Movies 362
## 2 Documentaries 359
## 3 Stand-Up Comedy 334
## 4 Comedies, Dramas, International Movies 274
## 5 Dramas, Independent Movies, International Movies 252
Which year had the highest content addition?
data %>%
count(year_added, sort = TRUE) %>%
head(1)
## # A tibble: 1 × 2
## year_added n
## <dbl> <int>
## 1 2019 2016
Which country produces more Movies vs TV Shows?
data %>%
count(country, type) %>%
arrange(desc(n))
## # A tibble: 849 × 3
## country type n
## <chr> <fct> <int>
## 1 United States Movie 2058
## 2 India Movie 893
## 3 United States TV Show 760
## 4 <NA> Movie 440
## 5 <NA> TV Show 391
## 6 United Kingdom TV Show 213
## 7 United Kingdom Movie 206
## 8 Japan TV Show 169
## 9 South Korea TV Show 158
## 10 Canada Movie 122
## # ℹ 839 more rows
What is the average duration for Movies only?
data %>%
filter(type == "Movie") %>%
summarise(avg_duration = mean(duration_num, na.rm = TRUE))
## # A tibble: 1 × 1
## avg_duration
## <dbl>
## 1 99.6
What is the average duration for TV Shows only?
data %>%
filter(type == "TV Show") %>%
summarise(avg_duration = mean(duration_num, na.rm = TRUE))
## # A tibble: 1 × 1
## avg_duration
## <dbl>
## 1 1.76
What is the average duration of all content?
mean(data$duration_num, na.rm = TRUE)
## [1] 69.84689
Are there outliers in release year?
Q1 <- quantile(data$release_year, 0.25, na.rm = TRUE)
Q3 <- quantile(data$release_year, 0.75, na.rm = TRUE)
IQR_value <- Q3 - Q1
data.frame(Q1 = Q1, Q3 = Q3, IQR = IQR_value)
## Q1 Q3 IQR
## 25% 2013 2019 6
#Q1 (25%) = 2013 → 25% of content was released on or before 2013
#Q3 (75%) = 2019 → 75% of content was released on or before 2019
#IQR = 6 → The middle 50% of release years lies within a 6-year range (2013 to 2019)
Outliers in Release Year
ggplot(data, aes(y=release_year)) +
geom_boxplot(fill="lightgreen") +
ggtitle("Outliers in Release Year")
#The boxplot shows that most release years are clustered within a normal range, with a few extreme values appearing as outliers.
How has Netflix content grown over time?
ggplot(data, aes(x=year_added)) +
geom_histogram(binwidth=1, fill="green") +
ggtitle("Content Added Over Time") +
xlab("Year Added") + ylab("Content")
#The histogram shows that most content was added in recent years, indicating rapid growth of OTT platforms over time.
What is the distribution shape of release year?
ggplot(data, aes(x=release_year)) +
geom_histogram(binwidth=1, fill="blue")
#The histogram shows that most content is concentrated in recent years, indicating a higher production of modern releases.
What is correlation between release year and duration?
cor(data$release_year, data$duration_num, use="complete.obs")
## [1] -0.2491815
#The correlation result shows a weak relationship between release year and duration, indicating that content length does not significantly depend on the year of release.
Linear regression
model <- lm(duration_num ~ release_year, data=data)
summary(model)
##
## Call:
## lm(formula = duration_num ~ release_year, data = data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -196.87 -59.49 16.91 36.77 247.64
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 2961.22063 119.77939 24.72 <2e-16 ***
## release_year -1.43551 0.05947 -24.14 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 49.21 on 8802 degrees of freedom
## (3 observations deleted due to missingness)
## Multiple R-squared: 0.06209, Adjusted R-squared: 0.06198
## F-statistic: 582.7 on 1 and 8802 DF, p-value: < 2.2e-16
#The regression results show that release year has little to no significant effect on duration, indicating a weak relationship between the two variables.
Plotting the curve
ggplot(data, aes(x=release_year, y=duration_num)) +
geom_point() +
geom_smooth(method="lm")
## `geom_smooth()` using formula = 'y ~ x'
#The graph shows a slight downward trend, indicating that newer content tends to have shorter durations, but the relationship is weak due to high variability in data.
Model predict duration based on release year
new_data <- data.frame(release_year = 2022)
head(predict(model, new.data = new_data))
## 1 2 3 4 5 6
## 61.49260 60.05709 60.05709 60.05709 60.05709 60.05709
#The model predicts the expected duration for content released in 2022, which is around 60 mins
Split Data into training and testing
train_index <- sample(1:nrow(data), 0.7*nrow(data))
train_data <- data[train_index, ]
test_data <- data[-train_index, ]
#The dataset is divided into 70% training data and 30% testing data to build and evaluate the model’s performance effectively.
Predicting test data
predictions <- predict(model, new.data=test_data)
head(predictions)
## 1 2 3 4 5 6
## 61.49260 60.05709 60.05709 60.05709 60.05709 60.05709
#The model predicts duration values for the test dataset, which can be used to assess how well the linear regression model performs on unseen data.
Boxplot for Rating Distribution
library(dplyr)
top_ratings <- data %>%
count(rating, sort = TRUE) %>%
head(6) %>%
pull(rating)
ggplot(data %>% filter(rating %in% top_ratings),
aes(x = rating, y = duration_num)) +
geom_boxplot(fill = "lightblue") +
ggtitle("Duration across Top Rating Categories") +
xlab("Rating") +
ylab("Duration")
#The boxplot shows that movie ratings like PG-13 and R have longer durations, while TV ratings such as TV-Y7 have shorter durations,
Creating a binary variable
data$type_bin <- ifelse(data$type == "Movie", 1, 0)
#A new binary variable is created where Movies are coded as 1 and TV Shows as 0 to enable numerical analysis and modeling.
Logistic Model
model_log <- glm(type_bin ~ duration_num, data=data, family="binomial")
summary(model_log)
##
## Call:
## glm(formula = type_bin ~ duration_num, family = "binomial", data = data)
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -8.54719 0.76095 -11.232 <2e-16 ***
## duration_num 0.67923 0.07162 9.483 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 10814.417 on 8803 degrees of freedom
## Residual deviance: 89.142 on 8802 degrees of freedom
## (3 observations deleted due to missingness)
## AIC: 93.142
##
## Number of Fisher Scoring iterations: 14
#For the classification of movies and shows
Prediction of logistic model
predict=predict(model_log, type="response")
head(predict)
## 1 2 3 4 5 6
## 1.0000000000 0.0007544841 0.0003826694 0.0003826694 0.0007544841 0.0003826694
#The model outputs probability values between 0 and 1, indicating the likelihood of each content being a Movie (closer to 1) or a TV Show (closer to 0).
Fitting multiple regression model
model_multi <- lm(duration_num ~ release_year + year_added, data = train_data)
summary(model_multi)
##
## Call:
## lm(formula = duration_num ~ release_year + year_added, data = train_data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -201.13 -60.68 16.82 36.32 249.82
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) -950.29252 803.23957 -1.183 0.237
## release_year -1.50488 0.07198 -20.907 < 2e-16 ***
## year_added 2.00660 0.39860 5.034 4.94e-07 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 49.15 on 6152 degrees of freedom
## (9 observations deleted due to missingness)
## Multiple R-squared: 0.06757, Adjusted R-squared: 0.06727
## F-statistic: 222.9 on 2 and 6152 DF, p-value: < 2.2e-16
#You created multiple regression to improve prediction accuracy by using more than one independent variable.
Prediction on test data
predictions <- predict(model_multi, newdata = test_data)
results <- data.frame(
Actual = test_data$duration_num,
Predicted = predictions
)
head(results)
## Actual Predicted
## 1 2 63.68479
## 2 1 63.68479
## 3 125 105.82145
## 4 1 63.68479
## 5 2 65.18967
## 6 94 63.68479
#The model predictions are nearly constant and do not match actual values well, indicating poor model performance.