Loading the data

library(readxl)
Trajectory <- read_excel("20231014_bronbestand_data_onderzoek.xlsx",sheet=2) # import the treatment 
print(paste(length(unique(Trajectory$client_id_pseudo)),"Clients in the Trajectory dataset"))
## [1] "2459 Clients in the Trajectory dataset"
print(paste(length(unique(Trajectory$dossier_id_pseudo)),"Episodes in the Trajectory dataset"))
## [1] "2906 Episodes in the Trajectory dataset"
SQ_48_score  <-read_excel("20231014_bronbestand_data_onderzoek.xlsx",sheet=6) # import the dataset 
print(paste(length(unique(SQ_48_score$dossier_id_pseudo)),"Episodes in the SQ_48 _score dataset"))
## [1] "2731 Episodes in the SQ_48 _score dataset"
Session <- read_excel("20231014_bronbestand_data_onderzoek.xlsx",sheet=3)
print(paste(length(unique(Session$dossier_id_pseudo)),"Episodes in the SQ_48 _score dataset"))
## [1] "2887 Episodes in the SQ_48 _score dataset"
Professionals <- read_excel("20231014_bronbestand_data_onderzoek.xlsx",sheet=5) 

1. Merge connected episodes and reorganize the treatment trajectory

1.1 Detect episodes occurring at small intervals (less than 10 days) and merge them into a single episode

library(dplyr)

interval <- 10 

# select out the 

data1 <- Trajectory %>% 
  select("client_id_pseudo",
         "dossier_id_pseudo",
         "dossier_datum_registratie",
         "dossier_einddatum")%>%
  mutate(
    dossier_einddatum = 
      as.Date(dossier_einddatum, format = "%Y-%m-%d"),
    dossier_datum_registratie = 
      as.Date(dossier_datum_registratie, format = "%Y-%m-%d")
  ) %>%
  group_by(client_id_pseudo) %>% 
  arrange(client_id_pseudo, 
          dossier_datum_registratie) %>%
  mutate(
    time_difference = abs(dossier_einddatum - lead(dossier_datum_registratie)),
    # the first end date minus next register date
    select_current_row = time_difference < interval & !is.na(time_difference),
    #if time iterval less than 10 day, then select current row is ture
    select_next_row = lag(select_current_row, default = FALSE)
    # if current row is True, then next row is True
  ) %>%
  filter(select_current_row | select_next_row)%>%
  # select current row is true of next row is true
  select(dossier_id_pseudo,client_id_pseudo)%>%
  ungroup() 

print(paste(length(unique(data1$client_id_pseudo)),
      "Clients have connecting treatment episodes"))
## [1] "65 Clients have connecting treatment episodes"

1.2 re-organise the treatment trajectory based treatment session

  1. Replace the trajectory date with the treatment session date, using the date of the first treatment session as the start of the trajectory, and the date of the last treatment session as the end of the treatment trajectory.

  2. Merge two records with the identifier ‘dossier_id_pseudo’ when the interval between the end date of the current episode and the registration date of the next episode is less than 10 days.”

For example, if the end date of one episode for a particular ‘dossier_id_pseudo’ is April 1st, and the registration date of the subsequent episode for the same ‘dossier_id_pseudo’ is April 9th, these two episodes should be merged, as the interval between them is only 8 days.

#filter out the individual episodes not in data1 and re-organise
New_Trajectory_1 <- Session %>%
  filter(!is.na(dossier_id_pseudo))%>% # removing the NA data
  filter(!dossier_id_pseudo %in% data1$dossier_id_pseudo) %>% # excluded connecting episodes
  group_by(dossier_id_pseudo) %>%
  arrange(dossier_id_pseudo,sessie_datum_starttijd)%>%
  mutate(
    start_date = min(sessie_datum_starttijd, na.rm = TRUE), 
    # the first treatment as the first date
    end_date = max(sessie_datum_starttijd, na.rm = TRUE),
    # the last treatment session as the last date
    number_sessions = n()
  ) %>%
  select(dossier_id_pseudo, start_date, end_date,number_sessions)%>%
  distinct(,.keep_all = TRUE)%>% 
  ungroup()

# reorganize the treatment trajectory for connecting episode
New_Trajectory_2 <- Session %>%
  filter(dossier_id_pseudo %in% data1$dossier_id_pseudo)%>%
  left_join(data1) %>% 
  group_by(client_id_pseudo)%>%
  arrange(client_id_pseudo,sessie_datum_starttijd)%>%
  mutate(
    start_date = min(sessie_datum_starttijd, na.rm = TRUE), 
    end_date = max(sessie_datum_starttijd, na.rm = TRUE),
    number_sessions = n()
  ) %>%
  mutate(dossier_id_pseudo=first(dossier_id_pseudo))%>% # here combine two different dossiers
  ungroup()%>%
  select(dossier_id_pseudo, start_date, end_date,number_sessions)%>%
  distinct(,.keep_all = TRUE)

# merge trajectory 1 and trajectory 2
New_Trajectory <- rbind(New_Trajectory_1,New_Trajectory_2) %>%
  mutate(
    end_date = as.Date(end_date),
    start_date = as.Date(start_date),
    duration = difftime(end_date+1, start_date, units = "weeks")
  )

1.3 Select the direct treatment session for clients based on information from Maarten

All the treatment session could divided into: direct treatment session for clients and indirect treatment session Here, we filtered all the direct treatment session for the clients

library(dplyr)
# for clients who do not have connecting sessions

Treatment_session_1 <- Session %>%
  filter(!is.na(dossier_id_pseudo)) %>%
  filter(!dossier_id_pseudo %in% data1$dossier_id_pseudo) %>%
  filter(sessie_tijd_direct != 0) %>%
 filter(grepl("^Behandelsessie", sessie_naam, ignore.case = TRUE, perl = TRUE) &
         !grepl("e-mail$", sessie_naam, ignore.case = TRUE, perl = TRUE) | 
         grepl("Follow-Up", sessie_naam, ignore.case = TRUE) &
        !grepl("Tweede afstemmingsoverleg ipv", sessie_naam, ignore.case = TRUE, perl = TRUE) | 
         grepl("verrichting", sessie_naam, ignore.case = TRUE) |
        grepl("Beschrijvende diagnostiek", sessie_naam, ignore.case = TRUE)|
        grepl("Bewustwording", sessie_naam, ignore.case = TRUE)|
        grepl("Consult specialist", sessie_naam, ignore.case = TRUE)| 
        grepl("Diagnostiek", sessie_naam, ignore.case = TRUE)|
        grepl("Dubbele behandelsessies", sessie_naam, ignore.case = TRUE)|
        grepl("Eerste", sessie_naam, ignore.case = TRUE)|
        grepl("Eindgesprek optioneel", sessie_naam, ignore.case = TRUE)|
        grepl("F2F interventie", sessie_naam, ignore.case = TRUE)|
        grepl("Fase 1", sessie_naam, ignore.case = TRUE)|
        grepl("Fase 2", sessie_naam, ignore.case = TRUE)|
        grepl("Fase 3", sessie_naam, ignore.case = TRUE)|
        grepl("Gesprek hoofdbehandelaar", sessie_naam, ignore.case = TRUE)|
        grepl("Grenzen aangeven", sessie_naam, ignore.case = TRUE)|
        grepl("Intake", sessie_naam, ignore.case = TRUE)&
        !grepl("^Opstellen", sessie_naam, ignore.case = TRUE)& 
        !grepl("^Uitwerken", sessie_naam, ignore.case = TRUE)&
        !grepl("werkgever$", sessie_naam, ignore.case = TRUE)|
        grepl("Kennismakingsgesprek", sessie_naam, ignore.case = TRUE)|
        grepl("Korte behandelsessie", sessie_naam, ignore.case = TRUE)|
        grepl("Nameting: Behandelaar leest", sessie_naam, ignore.case = TRUE)|
        grepl("Nameting: Behandelaar leest", sessie_naam, ignore.case = TRUE)|
        grepl("Psychiatrisch", sessie_naam, ignore.case = TRUE)&
         !grepl("^Psychiatrisch.*[^1]5$", sessie_naam, ignore.case = TRUE, perl = TRUE)&
          !grepl("^Psychiatrisch.*e-mail$", sessie_naam, ignore.case = TRUE)&
          !grepl("^Psychiatrisch.*tele consult$", sessie_naam, ignore.case = TRUE, perl = TRUE) &
         !grepl("^Psychiatrisch.*tele consult \\(telefonisch / e-mail\\)$", sessie_naam, ignore.case = TRUE, perl = TRUE)|
        grepl("Psychiatrisch vervolg consult (farmacotherapie)", sessie_naam, ignore.case = TRUE)|  
        grepl("Regie behandelaar", sessie_naam, ignore.case = TRUE)|
        grepl("Regie overdracht kennismaking", sessie_naam, ignore.case = TRUE)|
        grepl("Screening", sessie_naam, ignore.case = TRUE)&
          !grepl("Uitwerktijd", sessie_naam, ignore.case = TRUE)|
        grepl("Tele consult", sessie_naam, ignore.case = TRUE)|
        grepl("Telefonisch consult", sessie_naam, ignore.case = TRUE)|
        grepl("Terugvalpreventie", sessie_naam, ignore.case = TRUE)|
        grepl("Terugvalpreventie", sessie_naam, ignore.case = TRUE)|
        grepl("Time Management", sessie_naam, ignore.case = TRUE)|
        grepl("Tweede gesprek", sessie_naam, ignore.case = TRUE)| 
        grepl("Tweede intake gesprek", sessie_naam, ignore.case = TRUE)| 
        grepl("Zelfwaardering", sessie_naam, ignore.case = TRUE)) %>%
  filter(sessie_naam != "Psychiatrisch tele consult (telefonisch / e-mail)")%>%
    filter(sessie_naam != "Tweede follow-up ipv afstemmingsoverleg")%>%
    filter(sessie_naam != "Psychiatrisch tele consult")%>%
  filter(!grepl("^Tele consult.*nt$", sessie_naam, ignore.case = TRUE))%>%
  filter(sessie_naam != "Uitwerktijd Intakeverslag")%>%
  filter(sessie_naam != "Telefonische follow-up")%>%
  select(dossier_id_pseudo, sessie_datum_starttijd,
         sessie_naam, sessie_tijd_direct,
         sessie_tijd_indirect_indirect,
         sessie_tijd_indirect_uitwerk)
  
# for clients who have connecting session 
Treatment_session_2 <- Session %>%
  filter(dossier_id_pseudo %in% data1$dossier_id_pseudo)%>%
  left_join(data1) %>% 
  group_by(client_id_pseudo)%>%
  mutate(dossier_id_pseudo=first(dossier_id_pseudo))%>% # merge two consecutive treatment episodes
  filter(sessie_tijd_direct != 0)%>%
    filter(grepl("^Behandelsessie", sessie_naam, ignore.case = TRUE, perl = TRUE) &
         !grepl("e-mail$", sessie_naam, ignore.case = TRUE, perl = TRUE) | 
         grepl("Follow-Up", sessie_naam, ignore.case = TRUE) &
        !grepl("Tweede afstemmingsoverleg ipv", sessie_naam, ignore.case = TRUE, perl = TRUE) | 
         grepl("verrichting", sessie_naam, ignore.case = TRUE) |
        grepl("Beschrijvende diagnostiek", sessie_naam, ignore.case = TRUE)|
        grepl("Bewustwording", sessie_naam, ignore.case = TRUE)|
        grepl("Consult specialist", sessie_naam, ignore.case = TRUE)| 
        grepl("Diagnostiek", sessie_naam, ignore.case = TRUE)|
        grepl("Dubbele behandelsessies", sessie_naam, ignore.case = TRUE)|
        grepl("Eerste", sessie_naam, ignore.case = TRUE)|
        grepl("Eindgesprek optioneel", sessie_naam, ignore.case = TRUE)|
        grepl("F2F interventie", sessie_naam, ignore.case = TRUE)|
        grepl("Fase 1", sessie_naam, ignore.case = TRUE)|
        grepl("Fase 2", sessie_naam, ignore.case = TRUE)|
        grepl("Fase 3", sessie_naam, ignore.case = TRUE)|
        grepl("Gesprek hoofdbehandelaar", sessie_naam, ignore.case = TRUE)|
        grepl("Grenzen aangeven", sessie_naam, ignore.case = TRUE)|
        grepl("Intake", sessie_naam, ignore.case = TRUE)&
        !grepl("^Opstellen", sessie_naam, ignore.case = TRUE)& 
        !grepl("^Uitwerken", sessie_naam, ignore.case = TRUE)&
        !grepl("werkgever$", sessie_naam, ignore.case = TRUE)|
        grepl("Kennismakingsgesprek", sessie_naam, ignore.case = TRUE)|
        grepl("Korte behandelsessie", sessie_naam, ignore.case = TRUE)|
        grepl("Nameting: Behandelaar leest", sessie_naam, ignore.case = TRUE)|
        grepl("Nameting: Behandelaar leest", sessie_naam, ignore.case = TRUE)|
        grepl("Psychiatrisch", sessie_naam, ignore.case = TRUE)&
         !grepl("^Psychiatrisch.*[^1]5$", sessie_naam, ignore.case = TRUE, perl = TRUE)&
          !grepl("^Psychiatrisch.*e-mail$", sessie_naam, ignore.case = TRUE)&
          !grepl("^Psychiatrisch.*tele consult$", sessie_naam, ignore.case = TRUE, perl = TRUE) &
         !grepl("^Psychiatrisch.*tele consult \\(telefonisch / e-mail\\)$", sessie_naam, ignore.case = TRUE, perl = TRUE)|
        grepl("Psychiatrisch vervolg consult (farmacotherapie)", sessie_naam, ignore.case = TRUE)|  
        grepl("Regie behandelaar", sessie_naam, ignore.case = TRUE)|
        grepl("Regie overdracht kennismaking", sessie_naam, ignore.case = TRUE)|
        grepl("Screening", sessie_naam, ignore.case = TRUE)&
          !grepl("Uitwerktijd", sessie_naam, ignore.case = TRUE)|
        grepl("Tele consult", sessie_naam, ignore.case = TRUE)|
        grepl("Telefonisch consult", sessie_naam, ignore.case = TRUE)|
        grepl("Terugvalpreventie", sessie_naam, ignore.case = TRUE)|
        grepl("Terugvalpreventie", sessie_naam, ignore.case = TRUE)|
        grepl("Time Management", sessie_naam, ignore.case = TRUE)|
        grepl("Tweede gesprek", sessie_naam, ignore.case = TRUE)| 
        grepl("Tweede intake gesprek", sessie_naam, ignore.case = TRUE)| 
        grepl("Zelfwaardering", sessie_naam, ignore.case = TRUE)) %>%
  filter(sessie_naam != "Psychiatrisch tele consult (telefonisch / e-mail)")%>%
    filter(sessie_naam != "Tweede follow-up ipv afstemmingsoverleg")%>%
    filter(sessie_naam != "Psychiatrisch tele consult")%>%
  filter(!grepl("^Tele consult.*nt$", sessie_naam,ignore.case = TRUE))%>%
  filter(sessie_naam != "Uitwerktijd Intakeverslag")%>%
    filter(sessie_naam != "Telefonische follow-up")%>%
  ungroup()%>%
  select(dossier_id_pseudo, sessie_datum_starttijd,
         sessie_naam, sessie_tijd_direct,
         sessie_tijd_indirect_indirect,
         sessie_tijd_indirect_uitwerk)

Treatment_session <- rbind(Treatment_session_1, Treatment_session_2)%>%
  arrange(dossier_id_pseudo,sessie_datum_starttijd)%>%
  group_by(dossier_id_pseudo)%>%
  mutate(number_sessions=n())%>%
  ungroup() # direct treatment session for all clients

# for double check
# compare the filtered treatment session with session offered by Maarten
Test_name <- Treatment_session %>%
  group_by(sessie_naam) %>% # group by session name
  summarise(count = n()) %>% # count the 
  arrange(desc(count)) 

1.3.1 Visualize the number of session and duration

Also compared the number of indirect treatment session and direct treatment session

2 Transition probabilities

2.1 Find the nearest session and its date for each score

library(dplyr)
library(lubridate)
library(purrr)

#first merge the different episode 
# find the each dossier ID score
SQ_48_1 <- SQ_48_score %>%
  filter(!is.na(dossier_id_pseudo))%>%
  filter(!dossier_id_pseudo %in% data1$dossier_id_pseudo)

SQ_48_2 <- SQ_48_score %>%
  filter(dossier_id_pseudo %in% data1$dossier_id_pseudo)%>%
  group_by(client_id_pseudo)%>%
  arrange(client_id_pseudo,test_datum_ingevuld)%>%
  mutate(dossier_id_pseudo=first(dossier_id_pseudo))
  
SQ_48_total <- rbind(SQ_48_1,SQ_48_2) %>% 
  filter(dossier_id_pseudo %in% New_Trajectory$dossier_id_pseudo)%>%
  filter(grepl("SQ", test_naam))

SQ_48_total$test_datum_ingevuld <- as.Date(SQ_48_total$test_datum_ingevuld, format = "%Y-%m-%d")


# the function for find the nearest session for each score
find_closest_treatment_date <- function(test_datum_ingevuld, dossier_id_pseudo, df_treatments) {
 
  test_datum_ingevuld <- as.Date(test_datum_ingevuld)


  patient_treatment_dates <- as.Date(df_treatments$sessie_datum_starttijd[df_treatments$dossier_id_pseudo == dossier_id_pseudo])

  if (length(patient_treatment_dates) == 0) {
    return(NA_real_)
  } else {
    return(patient_treatment_dates[which.min(abs(patient_treatment_dates - test_datum_ingevuld))])
  }
}

# the function to find which session is nearest
find_treatment_number <- function(test_datum_ingevuld, dossier_id_pseudo, df_treatments) {
 
  test_datum_ingevuld <- as.Date(test_datum_ingevuld)


  patient_treatment_dates <- as.Date(df_treatments$sessie_datum_starttijd[df_treatments$dossier_id_pseudo == dossier_id_pseudo])

  if (length(patient_treatment_dates) == 0) {
    return(NA_integer_)
  } else {
    return(which.min(abs(patient_treatment_dates - test_datum_ingevuld)))
  }
}

# define a function to calculate the T-score

calculate_T_SCORE <- function(RS_SQ48) {
  T_SCORE <- 19.40608979088534 + 
    0.5941681191862558 * RS_SQ48 + 
    -0.001812735748786516 * RS_SQ48^2 +
    6.960825359362865e-006 * RS_SQ48^3
  return(T_SCORE)
}

# apply the function to find the nearest treatment session
SQ_48_total_score <-SQ_48_total %>%
  filter(dossier_id_pseudo %in% Treatment_session$dossier_id_pseudo)%>%
  select(dossier_id_pseudo,test_datum_ingevuld,test_score)%>%
  arrange(dossier_id_pseudo,test_datum_ingevuld)%>%
  rowwise() %>%
  mutate(
   closest_treatment_date = find_closest_treatment_date(test_datum_ingevuld,dossier_id_pseudo,Treatment_session),
   treatment_number =find_treatment_number(test_datum_ingevuld,dossier_id_pseudo,Treatment_session),
   test_score = calculate_T_SCORE(test_score)
   ) %>%
  distinct(dossier_id_pseudo,test_datum_ingevuld,.keep_all = TRUE)
  
print(head(SQ_48_total_score))
## # A tibble: 6 × 5
## # Rowwise: 
##   dossier_id_pseudo        test_datum_ingevuld test_score closest_treatment_date
##   <chr>                    <date>                   <dbl> <date>                
## 1 00000010-9E77-42E0-AA15… 2022-07-11                51.4 2022-07-14            
## 2 00000010-9E77-42E0-AA15… 2022-09-08                60.2 2022-09-06            
## 3 00000010-9E77-42E0-AA15… 2022-11-28                55.4 2022-11-29            
## 4 000DE9B4-080D-41F3-B90F… 2022-02-09                48.2 2022-02-09            
## 5 001E6F1A-E1FE-43AC-AC50… 2022-06-15                55.4 2022-06-15            
## 6 001E6F1A-E1FE-43AC-AC50… 2022-09-21                45.9 2022-09-21            
## # ℹ 1 more variable: treatment_number <int>
# FILTER OUT CLIENTS WHO HAS LESS THAN 2 SCORES AND FIRST SCORE IS LESS THAN 42

SQ_48_filter_mark <- SQ_48_total_score %>% 
  group_by(dossier_id_pseudo) %>%
  arrange(dossier_id_pseudo, treatment_number) %>% # Ensure the order is correct
  mutate(number_score = n(),
         first_score = first(test_score)) %>%
  filter(number_score<=1|first_score<42.5) %>% # filter these clients who has only one score and intake score less than 42.5
  ungroup() %>%
  distinct(dossier_id_pseudo) # N=758 excluded 

2.2 calculate the responsive status for all clients

After excluding those clients who have only one sq48 score and whose intake score is less than 42.5. we calculated the response status based on responder definition provided by Maarten

– Five less than pre-treatment and test < 42.5 Assumption: If a client’s last response status is ‘responsive,’ then this patient is considered a responder.

SQ_48_score_responsive_status <- SQ_48_total_score %>%
  filter(! dossier_id_pseudo %in% SQ_48_filter_mark$dossier_id_pseudo) %>% # N=1904
    dplyr::group_by(dossier_id_pseudo) %>% 
  dplyr::arrange(dossier_id_pseudo, test_datum_ingevuld) %>% # Arrange rows within each group
  # Calculate the first score within each group and broadcast it to all rows in the same group
  dplyr::mutate(row_num = row_number()) %>%
  dplyr::mutate(first_score = first(test_score)) %>% 
  # Perform the calculation: (test_score - first_score)
  dplyr::mutate(calc_result = (first_score - test_score)) %>% 
  # Update the intake_status column based on the calculated result
  dplyr::mutate(responsive_status = case_when(
    row_num == 1 ~ "intake",
    calc_result >= 5 & test_score < 42.5 ~ "response",  # five less than pre-treatment and T-score < 42.5
    TRUE ~ "non-response"
  )) %>% 
  # Select and rename columns (optional)
  dplyr::select(
    dossier_id_pseudo,        # Keep as is     
    test_datum_ingevuld,  
    test_score,     
    calc_result,
    responsive_status            
  ) %>%  ungroup()
  
# Check the final responsive status
Responsive_status <- SQ_48_score_responsive_status %>% 
  # Group the data by dossier_id_pseudo
  dplyr::group_by(dossier_id_pseudo) %>% 
  dplyr::mutate(last_responsive_status = last(responsive_status))%>%
  ungroup()

Responsive_patients <- Responsive_status %>%  # select the responders
  filter(last_responsive_status == "response") %>%
  group_by(dossier_id_pseudo)%>%
  mutate(Test_interval = row_number()) %>%
  ungroup()

# filter the patient who are responsive
Non_Responsive_patients <- Responsive_status %>% # select non-responders
  filter(last_responsive_status == "non-response") %>%
    group_by(dossier_id_pseudo)%>%
  mutate(Test_interval = row_number()) %>%
  ungroup()

# calculate the responsive ratio and non-responsive ratio
Responsive_ratio <- n_distinct(Responsive_patients$dossier_id_pseudo)/n_distinct(Responsive_status$dossier_id_pseudo)
Non_Responsive_ratio <- n_distinct(Non_Responsive_patients$dossier_id_pseudo)/n_distinct(Responsive_status$dossier_id_pseudo)


ratios_data <- data.frame(
  Type = c("Responsive", "Non-Responsive"),
  Ratio = c(Responsive_ratio, Non_Responsive_ratio),
  Clients=c(n_distinct(Responsive_patients$dossier_id_pseudo),
            n_distinct(Non_Responsive_patients$dossier_id_pseudo))
)

print(ratios_data)
##             Type     Ratio Clients
## 1     Responsive 0.6163222    1163
## 2 Non-Responsive 0.3836778     724

2.2.1 visualized the treatment trajectory in both responders and non-responders

2.3 calculate the transition probabilities with complete data

Due to clients delaying in completing the questionnaires sent by therapists, the test scores do not precisely reflect the effects after every five treatment sessions, resulting in some treatment scores being missing.

Discussion point 1: To filter out this kind of missing data, we have established a rule: the interval between any two measurement scores must be more than 3 treatment sessions and less than 7 treatment sessions.

For example, consider the scoring timeline for a client: The initial score is recorded after the first treatment session, which is the intake session. The next score is recorded after the sixth treatment session, indicating a gap of five sessions between the two scores. Subsequently, the third score is noted around the thirteenth treatment session, making a difference of seven sessions since the previous score. Based on our definition, this clients will included in our analysis

Discussion point 2:

In our model, ‘recovered’ is the absorbing state. This means that the ‘recovered’ row in the transition probability matrix is set to 1. Consequently, once a patient reaches the recovered state, they cannot transition to other states. This approach is also consistent with real-world practice. However, in our data set, we saw some patient got relapse after recovered

Can we assumed that: if a client has reached the ‘recovered’ state, they will remain in this state in our data set. Does this make sense ?

# 1. Filtering out scores with excessive time intervals
SQ_48_filter_mark_1 <- SQ_48_total_score %>%
  group_by(dossier_id_pseudo) %>%
  arrange(dossier_id_pseudo, treatment_number) %>% # Ensure the order is correct
  mutate(diff = treatment_number - lead(treatment_number)) %>% # Else calculate the difference between current row and next row
  mutate(new_column = (diff < -2 & diff > -8) | is.na(diff)) %>%
         # marked the consecutive score more than 2 session and less than 8 session
  filter(any(new_column == FALSE)) %>%
  ungroup() %>% distinct(dossier_id_pseudo) # 1066 clients excluded



SQ_t48_complete <- Responsive_status %>% # N=796
  filter(!dossier_id_pseudo %in% SQ_48_filter_mark_1$dossier_id_pseudo)%>%
  mutate(
    states =  case_when(
      test_score < 42.5 ~ "recovered", # lower than 39.26 is recover
      test_score >= 42.5 & test_score < 60 ~ "mild", # between 39.26 to 59.9 is mild
      test_score >= 60 & test_score < 70 ~ "moderate", # between 60-70 is moderate
      test_score >=70 ~ "severe", # over than 70 is severe
      TRUE ~ "UNKOWN" # for any other cases, which ideally should not exits
    )
  ) 


# if the patient recovered than it won't transfer to next state
# change the data
SQ_t48_modified <- SQ_t48_complete %>%
  group_by(dossier_id_pseudo) %>%
  # Create a flag to indicate whether there is a 'recovered' state in the group
  mutate(has_recovered = any(states == "recovered")) %>%
  # If 'recovered' exists, find the position of the last 'recovered'
  mutate(last_recovered = ifelse(has_recovered, min(which(states == "recovered")), NA_integer_)) %>%
  # Mark whether the row is after the last 'recovered'
  mutate(is_after_last_recovered = row_number() > last_recovered) %>%
  # If the row is after the last 'recovered', change 'states' to 'recovered'
  mutate(states = ifelse(has_recovered & is_after_last_recovered, "recovered", states)) %>%
  select(-has_recovered, -last_recovered, -is_after_last_recovered) %>%
  ungroup()




# Function to calculate the transition matrix for a given status
calculate_transition_matrix <- function(data, status){
  data_filtered <- data %>%
    filter(last_responsive_status == status) %>%
    group_by(dossier_id_pseudo) %>%
    arrange(dossier_id_pseudo, test_datum_ingevuld) %>%
    mutate(row = row_number())
  
  states <- c("mild", "moderate", "severe", "recovered")
  transition_matrix <- matrix(0, nrow = length(states), ncol = length(states))
  rownames(transition_matrix) <- states
  colnames(transition_matrix) <- states
  
  for (id in unique(data_filtered$dossier_id_pseudo)){
    patient_data <- data_filtered[data_filtered$dossier_id_pseudo == id, ]
    for (i in 1:(nrow(patient_data) - 1)){
      from_state <- patient_data$states[i]
      to_state <- patient_data$states[i + 1]
      transition_matrix[from_state, to_state] <- transition_matrix[from_state, to_state] + 1
    }
  }
  
  transition_prob_matrix <- transition_matrix / rowSums(transition_matrix)
  return(transition_prob_matrix)
}


# Select relevant columns
Data_transition_probability <- SQ_t48_modified %>%
  select(dossier_id_pseudo, last_responsive_status, test_datum_ingevuld, test_score, states)

# Calculate the transition probabilities
transition_prob_matrix_response <- calculate_transition_matrix(Data_transition_probability, "response")
transition_prob_matrix_nonresponse <- calculate_transition_matrix(Data_transition_probability, "non-response")
print("transition matrix for responders")
## [1] "transition matrix for responders"
print(transition_prob_matrix_response)
##                mild   moderate     severe  recovered
## mild      0.3632708 0.01608579 0.00000000 0.62064343
## moderate  0.5943396 0.12264151 0.01886792 0.26415094
## severe    0.6956522 0.13043478 0.08695652 0.08695652
## recovered 0.0000000 0.00000000 0.00000000 1.00000000
print("transition matrix for non-responders")
## [1] "transition matrix for non-responders"
print(transition_prob_matrix_nonresponse)
##                mild   moderate      severe  recovered
## mild      0.8896882 0.04796163 0.002398082 0.05995204
## moderate  0.5337423 0.41104294 0.042944785 0.01226994
## severe    0.4857143 0.40000000 0.114285714 0.00000000
## recovered 0.0000000 0.00000000 0.000000000 1.00000000
# write matrix into CSV
write.csv(transition_prob_matrix_response, file = "transition_prob_matrix_responder_complete_data.csv", row.names = FALSE)
write.csv(transition_prob_matrix_nonresponse, file = "transition_prob_matrix_Nonresponder_complete_data.csv", row.names = FALSE) 

2.4 calculate the transition probabilities with imputed data

Since we excluded 1066 clients due to the lack of suitable SQ48 scores, we are attempting to impute the missing SQ48 scores using single imputation

  1. Identify the appropriate treatment session for each score. For instance, if the second score is recorded after the 6th session, we assume this score represents the 5th session. Similarly, if a score is recorded after the 11th session, we consider this score as representative of the 10th session, and so on.

  2. Determine the missing scores for every set of five treatment sessions. For example, if a client has scores only for the intake session, the 10th session, and the 20th session, it implies that scores for the 5th and 15th sessions are missing.

  3. imputing the missing score by single imputation

  4. calculate the transition probabilities

## Define a function to find the largest multiple of 5 that is less than a given number
find_nearest_multiple_of_five <- function(x) {
  lower = (x %/% 5) * 5
  upper = lower + 5
  ifelse(abs(x - lower) <= abs(x - upper), lower, upper)
}


SQ_48_score_long <- SQ_48_total_score %>%
  filter(!dossier_id_pseudo %in% SQ_48_filter_mark$dossier_id_pseudo) %>%
  group_by(dossier_id_pseudo) %>%
  mutate(find_nearest_multiple_of_five = ifelse(row_number() > 1, 
                                                 find_nearest_multiple_of_five(treatment_number), 
                                                 1))%>%
  filter(find_nearest_multiple_of_five>0) %>%
  arrange(dossier_id_pseudo, find_nearest_multiple_of_five, desc(treatment_number)) %>%
  distinct(dossier_id_pseudo,find_nearest_multiple_of_five,.keep_all = TRUE)%>%
  ungroup

library(tidyr)
SQ_48_score_wider <- SQ_48_score_long %>%
  group_by(dossier_id_pseudo)%>%
  mutate(column_name = paste0("col_", find_nearest_multiple_of_five)) %>%
  rowwise() %>%
  mutate(value = test_score) %>%
  ungroup() %>%
  pivot_wider(names_from = column_name, values_from = value, values_fill = list(value = NA))%>%
  select(dossier_id_pseudo, test_datum_ingevuld, test_score, 
         closest_treatment_date, treatment_number, 
         find_nearest_multiple_of_five, 
         col_1, col_5, col_10, col_15, col_20, col_25, col_30, col_35,
         col_40, col_45, col_50, col_55, col_60, col_65, col_70)


# combine the same row into one
SQ_48_score_tidy <- SQ_48_score_wider %>%
  group_by(dossier_id_pseudo) %>%
  summarise(across(starts_with("col_"), ~na.omit(.)[1])) %>%
  ungroup()

SQ_48_score_long <- SQ_48_score_tidy %>%
  pivot_longer(
    cols = starts_with("col_"),   
    names_to = "treatment_session",      
    values_to = "test_score"           
  )%>%
  group_by(dossier_id_pseudo) %>%
  mutate(last_non_na = max(which(!is.na(test_score)))) %>%  
  filter(row_number() <= last_non_na) %>%  
  select(-last_non_na)%>%
  ungroup()
 
# inmpute the missing data
library(mice)
Imputate_date <- SQ_48_score_long %>% select(-dossier_id_pseudo)
imputed_model <- mice(Imputate_date, m = 1, method = 'norm', maxit = 50)
## 
##  iter imp variable
##   1   1  test_score
##   2   1  test_score
##   3   1  test_score
##   4   1  test_score
##   5   1  test_score
##   6   1  test_score
##   7   1  test_score
##   8   1  test_score
##   9   1  test_score
##   10   1  test_score
##   11   1  test_score
##   12   1  test_score
##   13   1  test_score
##   14   1  test_score
##   15   1  test_score
##   16   1  test_score
##   17   1  test_score
##   18   1  test_score
##   19   1  test_score
##   20   1  test_score
##   21   1  test_score
##   22   1  test_score
##   23   1  test_score
##   24   1  test_score
##   25   1  test_score
##   26   1  test_score
##   27   1  test_score
##   28   1  test_score
##   29   1  test_score
##   30   1  test_score
##   31   1  test_score
##   32   1  test_score
##   33   1  test_score
##   34   1  test_score
##   35   1  test_score
##   36   1  test_score
##   37   1  test_score
##   38   1  test_score
##   39   1  test_score
##   40   1  test_score
##   41   1  test_score
##   42   1  test_score
##   43   1  test_score
##   44   1  test_score
##   45   1  test_score
##   46   1  test_score
##   47   1  test_score
##   48   1  test_score
##   49   1  test_score
##   50   1  test_score
# join impute data with clients' ID 
SQ_48_score_imputed_data <- complete(imputed_model, 1)%>% 
  cbind(select(SQ_48_score_long,1))


SQ_48_score_responsive_status_imputed <- SQ_48_score_imputed_data %>%
    dplyr::group_by(dossier_id_pseudo) %>% 
  # Calculate the first score within each group and broadcast it to all rows in the same group
  dplyr::mutate(row_num = row_number()) %>%
  dplyr::mutate(first_score = first(test_score)) %>% 
  # Perform the calculation: (test_score - first_score)
  dplyr::mutate(calc_result = (first_score - test_score)) %>% 
  mutate(score_number=n())%>%
  # Update the intake_status column based on the calculated result
  dplyr::mutate(responsive_status = case_when(
    row_num == 1 ~ "intake",
    calc_result >= 5 & test_score < 42.5 ~ "response",  # five less than pre-treatment and T-score >42.5
    TRUE ~ "non-response"
  )) %>% 
  filter(score_number>1)%>%
  # Select and rename columns (optional)
  dplyr::select(
    dossier_id_pseudo,        # Keep as is     
    test_score,  
    treatment_session,
    calc_result,
    responsive_status,
    score_number
  ) %>%  ungroup() # N=1879
  
# Check the final responsive status
Responsive_status_imputed <- SQ_48_score_responsive_status_imputed %>% 
  # Group the data by dossier_id_pseudo
  dplyr::group_by(dossier_id_pseudo) %>% 
  dplyr::mutate(last_responsive_status = last(responsive_status))%>%
  ungroup()

Responsive_patients_impute <- Responsive_status_imputed %>%  # select the responders
  filter(last_responsive_status == "response") 

# filter the patient who are responsive
Non_Responsive_patients_impute <- Responsive_status_imputed %>% # select non-responders
  filter(last_responsive_status == "non-response") 

# responsive ratio
n_distinct(Responsive_patients_impute$dossier_id_pseudo)/n_distinct(Responsive_status_imputed$dossier_id_pseudo)
## [1] 0.6248004
 Depression_states <- Responsive_status_imputed%>%
  select(dossier_id_pseudo, treatment_session, test_score,last_responsive_status)%>%
  mutate(
    states =  case_when(
      test_score < 42.5 ~ "recovered", # lower than 39.26 is recover
      test_score >= 42.5 & test_score < 60 ~ "mild", # between 39.26 to 59.9 is mild
      test_score >= 60 & test_score < 70 ~ "moderate", # between 60-70 is moderate
      test_score >=70 ~ "severe", # over than 70 is severe
      TRUE ~ "UNKOWN" # for any other cases, which ideally should not exits
    )
  )




# if the patient recovered than it won't transfer to next state
# change the data
SQ_t48_imputed_modified <- Depression_states %>%
  group_by(dossier_id_pseudo) %>%
  # Create a flag to indicate whether there is a 'recovered' state in the group
  mutate(has_recovered = any(states == "recovered")) %>%
  # If 'recovered' exists, find the position of the last 'recovered'
  mutate(last_recovered = ifelse(has_recovered, min(which(states == "recovered")), NA_integer_)) %>%
  # Mark whether the row is after the last 'recovered'
  mutate(is_after_last_recovered = row_number() > last_recovered) %>%
  # If the row is after the last 'recovered', change 'states' to 'recovered'
  mutate(states = ifelse(has_recovered & is_after_last_recovered, "recovered", states)) %>%
  mutate(score_number = n())%>%
  select(-has_recovered, -last_recovered, -is_after_last_recovered) %>%
  ungroup()


# Function to calculate the transition matrix for a given status
calculate_transition_matrix_imputed <- function(data, status){
  data_filtered <- data %>%
    filter(last_responsive_status == status) %>%
    group_by(dossier_id_pseudo) %>%
    #arrange(dossier_id_pseudo, test_datum_ingevuld) %>%
    mutate(row = row_number())
  
  states <- c("mild", "moderate", "severe", "recovered")
  transition_matrix <- matrix(0, nrow = length(states), ncol = length(states))
  rownames(transition_matrix) <- states
  colnames(transition_matrix) <- states
  
  for (id in unique(data_filtered$dossier_id_pseudo)){
    patient_data <- data_filtered[data_filtered$dossier_id_pseudo == id, ]
    for (i in 1:(nrow(patient_data) - 1)){
      from_state <- patient_data$states[i]
      to_state <- patient_data$states[i + 1]
      transition_matrix[from_state, to_state] <- transition_matrix[from_state, to_state] + 1
    }
  }
  
  transition_prob_matrix <- transition_matrix / rowSums(transition_matrix)
  return(transition_prob_matrix)
}


# Select relevant columns
Data_transition_probability_imputed <- SQ_t48_imputed_modified %>%
  select(dossier_id_pseudo, last_responsive_status, test_score, states)


# Calculate the transition probabilities
transition_prob_matrix_response_imputed <- calculate_transition_matrix_imputed(Data_transition_probability_imputed, "response")
transition_prob_matrix_nonresponse_imputed <- 
  calculate_transition_matrix_imputed(Data_transition_probability_imputed, "non-response")

print("transition matrix for responders with imputed data")
## [1] "transition matrix for responders with imputed data"
print(transition_prob_matrix_response_imputed)
##                mild   moderate     severe recovered
## mild      0.4267003 0.04937028 0.00604534 0.5178841
## moderate  0.5092838 0.12732095 0.00795756 0.3554377
## severe    0.6000000 0.14545455 0.03636364 0.2181818
## recovered 0.0000000 0.00000000 0.00000000 1.0000000
print("transition matrix for responders with completed data")
## [1] "transition matrix for responders with completed data"
print(transition_prob_matrix_response)
##                mild   moderate     severe  recovered
## mild      0.3632708 0.01608579 0.00000000 0.62064343
## moderate  0.5943396 0.12264151 0.01886792 0.26415094
## severe    0.6956522 0.13043478 0.08695652 0.08695652
## recovered 0.0000000 0.00000000 0.00000000 1.00000000
print("transition matrix for non-responders with imputed data")
## [1] "transition matrix for non-responders with imputed data"
print(transition_prob_matrix_nonresponse_imputed)
##                mild   moderate     severe recovered
## mild      0.7252475 0.09735974 0.02722772 0.1501650
## moderate  0.5630435 0.28478261 0.03695652 0.1152174
## severe    0.5233645 0.21495327 0.10280374 0.1588785
## recovered 0.0000000 0.00000000 0.00000000 1.0000000
print("transition matrix for non-responders with completed")
## [1] "transition matrix for non-responders with completed"
print(transition_prob_matrix_nonresponse)
##                mild   moderate      severe  recovered
## mild      0.8896882 0.04796163 0.002398082 0.05995204
## moderate  0.5337423 0.41104294 0.042944785 0.01226994
## severe    0.4857143 0.40000000 0.114285714 0.00000000
## recovered 0.0000000 0.00000000 0.000000000 1.00000000
# write matrix into CSV
write.csv(transition_prob_matrix_response_imputed, 
          file = "transition_prob_matrix_responder_imputed_data.csv", row.names = FALSE)
write.csv(transition_prob_matrix_nonresponse_imputed, 
          file = "transition_prob_matrix_Nonresponder_imputed_data.csv", row.names = FALSE)

Discussion point 4: which one should be used in main analysis

3. Cost

3.1 calculate the percentage of psychologist