N_PROJECT2

Quarto

Quarto enables you to weave together content and executable code into a finished document. To learn more about Quarto see https://quarto.org.

Running Code

When you click the Render button a document will be generated that includes both content and the output of embedded code. You can embed code like this:

packages <- c(
  "readr", "dplyr", "tidyr",
  "janitor", "ggplot2"
)

new_packages <- packages[
  !packages %in% installed.packages()[, "Package"]
]

if (length(new_packages) > 0) {
  install.packages(new_packages)
}

library(readr)
library(dplyr)

Attaching package: 'dplyr'

The following objects are masked from 'package:stats':

    filter, lag

The following objects are masked from 'package:base':

    intersect, setdiff, setequal, union

library(tidyr)
library(janitor)

Attaching package: 'janitor'

The following objects are masked from 'package:stats':

    chisq.test, fisher.test

library(ggplot2)

college_majors <- read_csv(file.choose())

Rows: 11399000 Columns: 25

── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr (23): program_id, opeid6, institution_name, institution_control, institu...
dbl  (2): Observation_ID, Value

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

student_dropout <- read_csv(file.choose())

Rows: 3 Columns: 5
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr (1): Variable
dbl (4): N, Mean, Median, SD

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

ai_usage <- read_csv(file.choose())

Rows: 864 Columns: 19
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr (17): user_id, age, gender, country, user_type, education_level, profess...
dbl  (2): Observation_ID, Value

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

# Verify the datasets
print(dim(college_majors))

[1] 11399000       25

print(dim(student_dropout))

[1] 3 5

print(dim(ai_usage))

[1] 864  19

glimpse(college_majors)

Rows: 11,399,000
Columns: 25
$ program_id                         <chr> "opeid031505-51.07-1", "opeid031505…
$ opeid6                             <chr> "031505", "031505", "031505", "0315…
$ institution_name                   <chr> "A - Technical College", "A - Techn…
$ institution_control                <chr> "Private, for-profit", "Private, fo…
$ institution_city                   <chr> NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ institution_state                  <chr> NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ institution_region                 <chr> NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ cip_code_4digit                    <chr> "51.07", "51.07", "51.07", "51.07",…
$ cip_title                          <chr> "Health and Medical Administrative …
$ cip_family_code                    <chr> "51", "51", "51", "51", "51", "51",…
$ cip_family_title                   <chr> "Health Professions and Related Pro…
$ credential_name                    <chr> "Undergraduate Certificate or Diplo…
$ distance_education                 <chr> "Not reported to IPEDS", "Not repor…
$ earnings_trajectory_category       <chr> "steady", "steady", "steady", "stea…
$ largest_linked_occupation          <chr> "First-line supervisors of office a…
$ largest_linked_occupation_soc      <chr> "43-1011", "43-1011", "43-1011", "4…
$ occupation_typical_entry_education <chr> "Bachelor's degree", "Bachelor's de…
$ ai_tool_examples                   <chr> "OpenAI ChatGPT", "OpenAI ChatGPT",…
$ earnings_4yr_status                <chr> "reported", "reported", "reported",…
$ earnings_1yr_status                <chr> "reported", "reported", "reported",…
$ earnings_5yr_status                <chr> "reported", "reported", "reported",…
$ debt_status                        <chr> "reported", "reported", "reported",…
$ Observation_ID                     <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,…
$ Variable                           <chr> "unitid", "is_main_campus", "instit…
$ Value                              <dbl> NA, 1.0, NA, NA, NA, NA, NA, NA, NA…

glimpse(student_dropout)

Rows: 3
Columns: 5
$ Variable <chr> "accuracy_rating", "productivity_score", "satisfaction_score"
$ N        <dbl> 274, 274, 281
$ Mean     <dbl> 3.416058, 5.235036, 6.131673
$ Median   <dbl> 3.5, 5.0, 6.0
$ SD       <dbl> 0.8571384, 1.8217888, 1.5261157

glimpse(ai_usage)

Rows: 864
Columns: 19
$ user_id                     <chr> "U0269", "U0269", "U0269", "U0217", "U0217…
$ age                         <chr> "30", "30", "30", "30", "30", "30", "17", …
$ gender                      <chr> "Male", "Male", "Male", "Male", "Male", "M…
$ country                     <chr> "United Kingdom", "United Kingdom", "Unite…
$ user_type                   <chr> "Professional", "Professional", "Professio…
$ education_level             <chr> "Master's", "Master's", "Master's", "Bache…
$ profession                  <chr> "Business Analyst", "Business Analyst", "B…
$ monthly_income              <chr> "5421", "5421", "5421", "8903", "8903", "8…
$ ai_tool                     <chr> "ChatGPT", "ChatGPT", "ChatGPT", "ChatGPT"…
$ ai_purpose                  <chr> "Office Work", "Office Work", "Office Work…
$ ai_usage_hours_per_day      <chr> "2 hrs", "2 hrs", "2 hrs", "5.7", "5.7", "…
$ tasks_performed             <chr> "Data analysis using AI-powered analytics …
$ monthly_ai_cost             <chr> NA, NA, NA, "0", "0", "0", "20", "20", "20…
$ time_saved_hours_per_week   <chr> "3.9", "3.9", "3.9", "11.4", "11.4", "11.4…
$ work_or_study_hours_per_day <chr> "7.3", "7.3", "7.3", "8.4 hrs", "8.4 hrs",…
$ would_recommend             <chr> "Yes", "Yes", "Yes", "Yes", "Yes", "Yes", …
$ Observation_ID              <dbl> 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, …
$ Variable                    <chr> "productivity_score", "accuracy_rating", "…
$ Value                       <dbl> 5.3, 4.0, 7.0, 9.9, 3.0, 9.0, 3.2, 2.0, 4.…

print(head(college_majors, 20))

# A tibble: 20 × 25
   program_id       opeid6 institution_name institution_control institution_city
   <chr>            <chr>  <chr>            <chr>               <chr>           
 1 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 2 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 3 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 4 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 5 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 6 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 7 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 8 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 9 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
10 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
11 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
12 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
13 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
14 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
15 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
16 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
17 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
18 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
19 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
20 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
# ℹ 20 more variables: institution_state <chr>, institution_region <chr>,
#   cip_code_4digit <chr>, cip_title <chr>, cip_family_code <chr>,
#   cip_family_title <chr>, credential_name <chr>, distance_education <chr>,
#   earnings_trajectory_category <chr>, largest_linked_occupation <chr>,
#   largest_linked_occupation_soc <chr>,
#   occupation_typical_entry_education <chr>, ai_tool_examples <chr>,
#   earnings_4yr_status <chr>, earnings_1yr_status <chr>, …

print(head(student_dropout, 20))

# A tibble: 3 × 5
  Variable               N  Mean Median    SD
  <chr>              <dbl> <dbl>  <dbl> <dbl>
1 accuracy_rating      274  3.42    3.5 0.857
2 productivity_score   274  5.24    5   1.82 
3 satisfaction_score   281  6.13    6   1.53 

print(head(ai_usage, 20))

# A tibble: 20 × 19
   user_id age   gender country        user_type    education_level profession  
   <chr>   <chr> <chr>  <chr>          <chr>        <chr>           <chr>       
 1 U0269   30    Male   United Kingdom Professional Master's        Business An…
 2 U0269   30    Male   United Kingdom Professional Master's        Business An…
 3 U0269   30    Male   United Kingdom Professional Master's        Business An…
 4 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 5 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 6 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 7 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
 8 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
 9 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
10 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
11 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
12 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
13 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
14 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
15 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
16 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
17 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
18 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
19 U0246   32    Male   france         Professional Bachelor's      Développeur…
20 U0246   32    Male   france         Professional Bachelor's      Développeur…
# ℹ 12 more variables: monthly_income <chr>, ai_tool <chr>, ai_purpose <chr>,
#   ai_usage_hours_per_day <chr>, tasks_performed <chr>, monthly_ai_cost <chr>,
#   time_saved_hours_per_week <chr>, work_or_study_hours_per_day <chr>,
#   would_recommend <chr>, Observation_ID <dbl>, Variable <chr>, Value <dbl>

college_clean <- college_majors %>%
  clean_names()

dropout_clean <- student_dropout %>%
  clean_names()

ai_clean <- ai_usage %>%
  clean_names()

print(names(college_clean))

 [1] "program_id"                         "opeid6"                            
 [3] "institution_name"                   "institution_control"               
 [5] "institution_city"                   "institution_state"                 
 [7] "institution_region"                 "cip_code_4digit"                   
 [9] "cip_title"                          "cip_family_code"                   
[11] "cip_family_title"                   "credential_name"                   
[13] "distance_education"                 "earnings_trajectory_category"      
[15] "largest_linked_occupation"          "largest_linked_occupation_soc"     
[17] "occupation_typical_entry_education" "ai_tool_examples"                  
[19] "earnings_4yr_status"                "earnings_1yr_status"               
[21] "earnings_5yr_status"                "debt_status"                       
[23] "observation_id"                     "variable"                          
[25] "value"                             

print(names(dropout_clean))

[1] "variable" "n"        "mean"     "median"   "sd"      

print(names(ai_clean))

 [1] "user_id"                     "age"                        
 [3] "gender"                      "country"                    
 [5] "user_type"                   "education_level"            
 [7] "profession"                  "monthly_income"             
 [9] "ai_tool"                     "ai_purpose"                 
[11] "ai_usage_hours_per_day"      "tasks_performed"            
[13] "monthly_ai_cost"             "time_saved_hours_per_week"  
[15] "work_or_study_hours_per_day" "would_recommend"            
[17] "observation_id"              "variable"                   
[19] "value"                      

# Function to summarize missing values
missing_summary <- function(data) {
  data %>%
    summarise(
      across(
        everything(),
        ~sum(is.na(.x))
      )
    ) %>%
    pivot_longer(
      cols = everything(),
      names_to = "Variable",
      values_to = "Missing_Count"
    )
}

print(missing_summary(college_clean), n = Inf)

# A tibble: 25 × 2
   Variable                           Missing_Count
   <chr>                                      <int>
 1 program_id                                     0
 2 opeid6                                         0
 3 institution_name                               0
 4 institution_control                            0
 5 institution_city                          564800
 6 institution_state                         564800
 7 institution_region                        564800
 8 cip_code_4digit                                0
 9 cip_title                                      0
10 cip_family_code                                0
11 cip_family_title                               0
12 credential_name                                0
13 distance_education                             0
14 earnings_trajectory_category             9442950
15 largest_linked_occupation                 276400
16 largest_linked_occupation_soc             276400
17 occupation_typical_entry_education        276400
18 ai_tool_examples                         7677500
19 earnings_4yr_status                            0
20 earnings_1yr_status                            0
21 earnings_5yr_status                            0
22 debt_status                                    0
23 observation_id                                 0
24 variable                                       0
25 value                                    4168760

print(missing_summary(dropout_clean), n = Inf)

# A tibble: 5 × 2
  Variable Missing_Count
  <chr>            <int>
1 variable             0
2 n                    0
3 mean                 0
4 median               0
5 sd                   0

print(missing_summary(ai_clean), n = Inf)

# A tibble: 19 × 2
   Variable                    Missing_Count
   <chr>                               <int>
 1 user_id                                 0
 2 age                                    69
 3 gender                                 30
 4 country                                30
 5 user_type                               0
 6 education_level                        36
 7 profession                             42
 8 monthly_income                         30
 9 ai_tool                                30
10 ai_purpose                             39
11 ai_usage_hours_per_day                 21
12 tasks_performed                        51
13 monthly_ai_cost                        36
14 time_saved_hours_per_week              30
15 work_or_study_hours_per_day            27
16 would_recommend                        15
17 observation_id                          0
18 variable                                0
19 value                                  35

# Number of original observations
original_rows <- c(
  nrow(college_clean),
  nrow(dropout_clean),
  nrow(ai_clean)
)

# Remove duplicates
college_clean <- college_clean %>% distinct()
dropout_clean <- dropout_clean %>% distinct()
ai_clean <- ai_clean %>% distinct()

# Compare
duplicate_summary <- data.frame(
  Dataset = c(
    "College Majors",
    "Student Performance",
    "AI Usage"
  ),
  Original_Rows = original_rows,
  Cleaned_Rows = c(
    nrow(college_clean),
    nrow(dropout_clean),
    nrow(ai_clean)
  )
) %>%
  mutate(
    Duplicates_Removed =
      Original_Rows - Cleaned_Rows
  )

print(duplicate_summary)

              Dataset Original_Rows Cleaned_Rows Duplicates_Removed
1      College Majors      11399000     11399000                  0
2 Student Performance             3            3                  0
3            AI Usage           864          864                  0

# Function for wide-to-long transformation
make_long <- function(data) {
  
  data %>%
    mutate(Observation_ID = row_number()) %>%
    pivot_longer(
      cols = where(is.numeric) &
        !all_of("Observation_ID"),
      names_to = "Variable",
      values_to = "Value"
    )
}

college_long <- make_long(college_clean)
dropout_long <- make_long(dropout_clean)
ai_long <- make_long(ai_clean)

# Display transformed datasets
print(head(college_long, 20))

# A tibble: 20 × 26
   program_id       opeid6 institution_name institution_control institution_city
   <chr>            <chr>  <chr>            <chr>               <chr>           
 1 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 2 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 3 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 4 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 5 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 6 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 7 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 8 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 9 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
10 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
11 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
12 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
13 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
14 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
15 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
16 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
17 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
18 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
19 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
20 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
# ℹ 21 more variables: institution_state <chr>, institution_region <chr>,
#   cip_code_4digit <chr>, cip_title <chr>, cip_family_code <chr>,
#   cip_family_title <chr>, credential_name <chr>, distance_education <chr>,
#   earnings_trajectory_category <chr>, largest_linked_occupation <chr>,
#   largest_linked_occupation_soc <chr>,
#   occupation_typical_entry_education <chr>, ai_tool_examples <chr>,
#   earnings_4yr_status <chr>, earnings_1yr_status <chr>, …

print(head(dropout_long, 20))

# A tibble: 12 × 4
   variable           Observation_ID Variable   Value
   <chr>                       <int> <chr>      <dbl>
 1 accuracy_rating                 1 n        274    
 2 accuracy_rating                 1 mean       3.42 
 3 accuracy_rating                 1 median     3.5  
 4 accuracy_rating                 1 sd         0.857
 5 productivity_score              2 n        274    
 6 productivity_score              2 mean       5.24 
 7 productivity_score              2 median     5    
 8 productivity_score              2 sd         1.82 
 9 satisfaction_score              3 n        281    
10 satisfaction_score              3 mean       6.13 
11 satisfaction_score              3 median     6    
12 satisfaction_score              3 sd         1.53 

print(head(ai_long, 20))

# A tibble: 20 × 20
   user_id age   gender country        user_type    education_level profession  
   <chr>   <chr> <chr>  <chr>          <chr>        <chr>           <chr>       
 1 U0269   30    Male   United Kingdom Professional Master's        Business An…
 2 U0269   30    Male   United Kingdom Professional Master's        Business An…
 3 U0269   30    Male   United Kingdom Professional Master's        Business An…
 4 U0269   30    Male   United Kingdom Professional Master's        Business An…
 5 U0269   30    Male   United Kingdom Professional Master's        Business An…
 6 U0269   30    Male   United Kingdom Professional Master's        Business An…
 7 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 8 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 9 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
10 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
11 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
12 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
13 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
14 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
15 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
16 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
17 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
18 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
19 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
20 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
# ℹ 13 more variables: monthly_income <chr>, ai_tool <chr>, ai_purpose <chr>,
#   ai_usage_hours_per_day <chr>, tasks_performed <chr>, monthly_ai_cost <chr>,
#   time_saved_hours_per_week <chr>, work_or_study_hours_per_day <chr>,
#   would_recommend <chr>, variable <chr>, Observation_ID <int>,
#   Variable <chr>, Value <dbl>

# Window function
apply_windows <- function(data) {
  
  data %>%
    group_by(Variable) %>%
    arrange(Value, Observation_ID, .by_group = TRUE) %>%
    mutate(
      Row_Number = row_number(),
      Min_Rank = min_rank(Value),
      Dense_Rank = dense_rank(Value),
      Percent_Rank = percent_rank(Value),
      Previous_Value = lag(Value),
      Next_Value = lead(Value),
      Cumulative_Sum = cumsum(
        replace_na(Value, 0)
      )
    ) %>%
    ungroup()
}

# Apply to three datasets
college_window <- apply_windows(college_long)
dropout_window <- apply_windows(dropout_long)
ai_window <- apply_windows(ai_long)

# Print results
print(head(college_window, 20))

# A tibble: 20 × 33
   program_id       opeid6 institution_name institution_control institution_city
   <chr>            <chr>  <chr>            <chr>               <chr>           
 1 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 2 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 3 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 4 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 5 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 6 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 7 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 8 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 9 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
10 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
11 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
12 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
13 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
14 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
15 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
16 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
17 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
18 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
19 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
20 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
# ℹ 28 more variables: institution_state <chr>, institution_region <chr>,
#   cip_code_4digit <chr>, cip_title <chr>, cip_family_code <chr>,
#   cip_family_title <chr>, credential_name <chr>, distance_education <chr>,
#   earnings_trajectory_category <chr>, largest_linked_occupation <chr>,
#   largest_linked_occupation_soc <chr>,
#   occupation_typical_entry_education <chr>, ai_tool_examples <chr>,
#   earnings_4yr_status <chr>, earnings_1yr_status <chr>, …

print(head(dropout_window, 20))

# A tibble: 12 × 11
   variable       Observation_ID Variable   Value Row_Number Min_Rank Dense_Rank
   <chr>                   <int> <chr>      <dbl>      <int>    <int>      <int>
 1 accuracy_rati…              1 mean       3.42           1        1          1
 2 productivity_…              2 mean       5.24           2        2          2
 3 satisfaction_…              3 mean       6.13           3        3          3
 4 accuracy_rati…              1 median     3.5            1        1          1
 5 productivity_…              2 median     5              2        2          2
 6 satisfaction_…              3 median     6              3        3          3
 7 accuracy_rati…              1 n        274              1        1          1
 8 productivity_…              2 n        274              2        1          1
 9 satisfaction_…              3 n        281              3        3          2
10 accuracy_rati…              1 sd         0.857          1        1          1
11 satisfaction_…              3 sd         1.53           2        2          2
12 productivity_…              2 sd         1.82           3        3          3
# ℹ 4 more variables: Percent_Rank <dbl>, Previous_Value <dbl>,
#   Next_Value <dbl>, Cumulative_Sum <dbl>

print(head(ai_window, 20))

# A tibble: 20 × 27
   user_id age   gender country        user_type    education_level profession  
   <chr>   <chr> <chr>  <chr>          <chr>        <chr>           <chr>       
 1 U0269   30    Male   United Kingdom Professional Master's        Business An…
 2 U0269   30    Male   United Kingdom Professional Master's        Business An…
 3 U0269   30    Male   United Kingdom Professional Master's        Business An…
 4 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 5 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 6 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 7 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
 8 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
 9 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
10 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
11 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
12 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
13 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
14 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
15 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
16 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
17 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
18 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
19 U0246   32    Male   france         Professional Bachelor's      Développeur…
20 U0246   32    Male   france         Professional Bachelor's      Développeur…
# ℹ 20 more variables: monthly_income <chr>, ai_tool <chr>, ai_purpose <chr>,
#   ai_usage_hours_per_day <chr>, tasks_performed <chr>, monthly_ai_cost <chr>,
#   time_saved_hours_per_week <chr>, work_or_study_hours_per_day <chr>,
#   would_recommend <chr>, variable <chr>, Observation_ID <int>,
#   Variable <chr>, Value <dbl>, Row_Number <int>, Min_Rank <int>,
#   Dense_Rank <int>, Percent_Rank <dbl>, Previous_Value <dbl>,
#   Next_Value <dbl>, Cumulative_Sum <dbl>

# Feature engineering function
create_features <- function(data) {
  
  data %>%
    group_by(Variable) %>%
    mutate(
      Variable_Mean = mean(
        Value, na.rm = TRUE
      ),
      
      Variable_SD = sd(
        Value, na.rm = TRUE
      ),
      
      Centered_Value =
        Value - Variable_Mean,
      
      Z_Score = if_else(
        !is.na(Variable_SD) &
          Variable_SD > 0,
        Centered_Value / Variable_SD,
        NA_real_
      ),
      
      Value_Category = case_when(
        is.na(Z_Score) ~ NA_character_,
        Z_Score < -1 ~ "Below Average",
        Z_Score > 1 ~ "Above Average",
        TRUE ~ "Average"
      )
    ) %>%
    ungroup()
}

# Apply feature engineering
college_features <- create_features(college_window)

dropout_features <- create_features(dropout_window)

ai_features <- create_features(ai_window)

# Print transformed results
print(head(college_features, 20))

# A tibble: 20 × 38
   program_id       opeid6 institution_name institution_control institution_city
   <chr>            <chr>  <chr>            <chr>               <chr>           
 1 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 2 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 3 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 4 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 5 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 6 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 7 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 8 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
 9 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
10 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
11 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
12 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
13 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
14 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
15 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
16 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
17 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
18 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
19 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
20 opeid031505-51.… 031505 A - Technical C… Private, for-profit <NA>            
# ℹ 33 more variables: institution_state <chr>, institution_region <chr>,
#   cip_code_4digit <chr>, cip_title <chr>, cip_family_code <chr>,
#   cip_family_title <chr>, credential_name <chr>, distance_education <chr>,
#   earnings_trajectory_category <chr>, largest_linked_occupation <chr>,
#   largest_linked_occupation_soc <chr>,
#   occupation_typical_entry_education <chr>, ai_tool_examples <chr>,
#   earnings_4yr_status <chr>, earnings_1yr_status <chr>, …

print(head(dropout_features, 20))

# A tibble: 12 × 16
   variable       Observation_ID Variable   Value Row_Number Min_Rank Dense_Rank
   <chr>                   <int> <chr>      <dbl>      <int>    <int>      <int>
 1 accuracy_rati…              1 mean       3.42           1        1          1
 2 productivity_…              2 mean       5.24           2        2          2
 3 satisfaction_…              3 mean       6.13           3        3          3
 4 accuracy_rati…              1 median     3.5            1        1          1
 5 productivity_…              2 median     5              2        2          2
 6 satisfaction_…              3 median     6              3        3          3
 7 accuracy_rati…              1 n        274              1        1          1
 8 productivity_…              2 n        274              2        1          1
 9 satisfaction_…              3 n        281              3        3          2
10 accuracy_rati…              1 sd         0.857          1        1          1
11 satisfaction_…              3 sd         1.53           2        2          2
12 productivity_…              2 sd         1.82           3        3          3
# ℹ 9 more variables: Percent_Rank <dbl>, Previous_Value <dbl>,
#   Next_Value <dbl>, Cumulative_Sum <dbl>, Variable_Mean <dbl>,
#   Variable_SD <dbl>, Centered_Value <dbl>, Z_Score <dbl>,
#   Value_Category <chr>

print(head(ai_features, 20))

# A tibble: 20 × 32
   user_id age   gender country        user_type    education_level profession  
   <chr>   <chr> <chr>  <chr>          <chr>        <chr>           <chr>       
 1 U0269   30    Male   United Kingdom Professional Master's        Business An…
 2 U0269   30    Male   United Kingdom Professional Master's        Business An…
 3 U0269   30    Male   United Kingdom Professional Master's        Business An…
 4 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 5 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 6 U0217   30    Male   United States  Professional Bachelor's      Customer Su…
 7 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
 8 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
 9 U0152   17    Male   Canada         Student      <NA>            Computer Sc…
10 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
11 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
12 U0264   35    FEMALE Brazil         Professional Bachelor's      Desenvolved…
13 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
14 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
15 U0263   19    Female Canada         Student      Undergraduate   Biology Stu…
16 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
17 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
18 U0225   19    <NA>   United States  Student      Undergraduate   Business An…
19 U0246   32    Male   france         Professional Bachelor's      Développeur…
20 U0246   32    Male   france         Professional Bachelor's      Développeur…
# ℹ 25 more variables: monthly_income <chr>, ai_tool <chr>, ai_purpose <chr>,
#   ai_usage_hours_per_day <chr>, tasks_performed <chr>, monthly_ai_cost <chr>,
#   time_saved_hours_per_week <chr>, work_or_study_hours_per_day <chr>,
#   would_recommend <chr>, variable <chr>, Observation_ID <int>,
#   Variable <chr>, Value <dbl>, Row_Number <int>, Min_Rank <int>,
#   Dense_Rank <int>, Percent_Rank <dbl>, Previous_Value <dbl>,
#   Next_Value <dbl>, Cumulative_Sum <dbl>, Variable_Mean <dbl>, …

summarize_data <- function(data) {
  
  data %>%
    group_by(Variable) %>%
    summarise(
      N = sum(!is.na(Value)),
      Mean = mean(Value, na.rm = TRUE),
      Median = median(Value, na.rm = TRUE),
      SD = sd(Value, na.rm = TRUE),
      .groups = "drop"
    )
}

college_summary <- summarize_data(college_features)
dropout_summary <- summarize_data(dropout_features)
ai_summary <- summarize_data(ai_features)

print(college_summary, n = Inf)

# A tibble: 2 × 5
  Variable              N    Mean  Median     SD
  <chr>             <int>   <dbl>   <dbl>  <dbl>
1 observation_id 11399000 113990. 113990. 65812.
2 value           7230240  16251.     12  44327.

print(dropout_summary, n = Inf)

# A tibble: 4 × 5
  Variable     N   Mean Median    SD
  <chr>    <int>  <dbl>  <dbl> <dbl>
1 mean         3   4.93   5.24 1.38 
2 median       3   4.83   5    1.26 
3 n            3 276.   274    4.04 
4 sd           3   1.40   1.53 0.494

print(ai_summary, n = Inf)

# A tibble: 2 × 5
  Variable           N   Mean Median    SD
  <chr>          <int>  <dbl>  <dbl> <dbl>
1 observation_id   864 144.    144.  83.2 
2 value            829   4.94    4.9  1.84

# Graph function
plot_features <- function(data, title) {
  
  ggplot(
    data,
    aes(x = Value_Category, fill = Value_Category)
  ) +
    geom_bar(na.rm = TRUE) +
    facet_wrap(~Variable, scales = "free_y") +
    labs(
      title = title,
      x = "Category",
      y = "Frequency"
    ) +
    theme_minimal() +
    theme(
      axis.text.x = element_text(
        angle = 45,
        hjust = 1
      ),
      legend.position = "none"
    )
}

# College Majors
print(plot_features(
  college_features,
  "College Majors: Engineered Categories"
))

# Student Academic Performance
print(plot_features(
  dropout_features,
  "Student Performance: Engineered Categories"
))

# AI Usage
print(plot_features(
  ai_features,
  "AI Usage: Engineered Categories"
))

# Cleaned datasets
write_csv(college_clean, "college_clean.csv")
write_csv(dropout_clean, "dropout_clean.csv")
write_csv(ai_clean, "ai_clean.csv")

# Long-format datasets
write_csv(college_long, "college_long.csv")
write_csv(dropout_long, "dropout_long.csv")
write_csv(ai_long, "ai_long.csv")

# Window function and feature engineering results
write_csv(college_features, "college_features.csv")
write_csv(dropout_features, "dropout_features.csv")
write_csv(ai_features, "ai_features.csv")

# Descriptive statistics
write_csv(college_summary, "college_summary.csv")
write_csv(dropout_summary, "dropout_summary.csv")
write_csv(ai_summary, "ai_summary.csv")

cat("Project 2 files exported successfully.\n")

Project 2 files exported successfully.

You can add options to executable code like this

[1] 4

The echo: false option disables the printing of code (only output is displayed).