library(readxl)
Trajectory <- read_excel("20231014_bronbestand_data_onderzoek.xlsx",sheet=2) # import the treatment
print(paste(length(unique(Trajectory$client_id_pseudo)),"Clients in the Trajectory dataset"))
## [1] "2459 Clients in the Trajectory dataset"
print(paste(length(unique(Trajectory$dossier_id_pseudo)),"Episodes in the Trajectory dataset"))
## [1] "2906 Episodes in the Trajectory dataset"
SQ_48_score <-read_excel("20231014_bronbestand_data_onderzoek.xlsx",sheet=6) # import the dataset
print(paste(length(unique(SQ_48_score$dossier_id_pseudo)),"Episodes in the SQ_48 _score dataset"))
## [1] "2731 Episodes in the SQ_48 _score dataset"
Session <- read_excel("20231014_bronbestand_data_onderzoek.xlsx",sheet=3)
print(paste(length(unique(Session$dossier_id_pseudo)),"Episodes in the SQ_48 _score dataset"))
## [1] "2887 Episodes in the SQ_48 _score dataset"
Professionals <- read_excel("20231014_bronbestand_data_onderzoek.xlsx",sheet=5)
library(dplyr)
interval <- 10
# select out the
data1 <- Trajectory %>%
select("client_id_pseudo",
"dossier_id_pseudo",
"dossier_datum_registratie",
"dossier_einddatum")%>%
mutate(
dossier_einddatum =
as.Date(dossier_einddatum, format = "%Y-%m-%d"),
dossier_datum_registratie =
as.Date(dossier_datum_registratie, format = "%Y-%m-%d")
) %>%
group_by(client_id_pseudo) %>%
arrange(client_id_pseudo,
dossier_datum_registratie) %>%
mutate(
time_difference = abs(dossier_einddatum - lead(dossier_datum_registratie)),
# the first end date minus next register date
select_current_row = time_difference < interval & !is.na(time_difference),
#if time iterval less than 10 day, then select current row is ture
select_next_row = lag(select_current_row, default = FALSE)
# if current row is True, then next row is True
) %>%
filter(select_current_row | select_next_row)%>%
# select current row is true of next row is true
select(dossier_id_pseudo,client_id_pseudo)%>%
ungroup()
print(paste(length(unique(data1$client_id_pseudo)),
"Clients have connecting treatment episodes"))
## [1] "65 Clients have connecting treatment episodes"
Replace the trajectory date with the treatment session date, using the date of the first treatment session as the start of the trajectory, and the date of the last treatment session as the end of the treatment trajectory.
Merge two records with the identifier ‘dossier_id_pseudo’ when the interval between the end date of the current episode and the registration date of the next episode is less than 10 days.”
For example, if the end date of one episode for a particular ‘dossier_id_pseudo’ is April 1st, and the registration date of the subsequent episode for the same ‘dossier_id_pseudo’ is April 9th, these two episodes should be merged, as the interval between them is only 8 days.
#filter out the individual episodes not in data1 and re-organise
New_Trajectory_1 <- Session %>%
filter(!is.na(dossier_id_pseudo))%>% # removing the NA data
filter(!dossier_id_pseudo %in% data1$dossier_id_pseudo) %>% # excluded connecting episodes
group_by(dossier_id_pseudo) %>%
arrange(dossier_id_pseudo,sessie_datum_starttijd)%>%
mutate(
start_date = min(sessie_datum_starttijd, na.rm = TRUE),
# the first treatment as the first date
end_date = max(sessie_datum_starttijd, na.rm = TRUE),
# the last treatment session as the last date
number_sessions = n()
) %>%
select(dossier_id_pseudo, start_date, end_date,number_sessions)%>%
distinct(,.keep_all = TRUE)%>%
ungroup()
# reorganize the treatment trajectory for connecting episode
New_Trajectory_2 <- Session %>%
filter(dossier_id_pseudo %in% data1$dossier_id_pseudo)%>%
left_join(data1) %>%
group_by(client_id_pseudo)%>%
arrange(client_id_pseudo,sessie_datum_starttijd)%>%
mutate(
start_date = min(sessie_datum_starttijd, na.rm = TRUE),
end_date = max(sessie_datum_starttijd, na.rm = TRUE),
number_sessions = n()
) %>%
mutate(dossier_id_pseudo=first(dossier_id_pseudo))%>% # here combine two different dossiers
ungroup()%>%
select(dossier_id_pseudo, start_date, end_date,number_sessions)%>%
distinct(,.keep_all = TRUE)
# merge trajectory 1 and trajectory 2
New_Trajectory <- rbind(New_Trajectory_1,New_Trajectory_2) %>%
mutate(
end_date = as.Date(end_date),
start_date = as.Date(start_date),
duration = difftime(end_date+1, start_date, units = "weeks")
)
All the treatment session could divided into: direct treatment session for clients and indirect treatment session Here, we filtered all the direct treatment session for the clients
library(dplyr)
# for clients who do not have connecting sessions
Treatment_session_1 <- Session %>%
filter(!is.na(dossier_id_pseudo)) %>%
filter(!dossier_id_pseudo %in% data1$dossier_id_pseudo) %>%
filter(sessie_tijd_direct != 0) %>%
filter(grepl("^Behandelsessie", sessie_naam, ignore.case = TRUE, perl = TRUE) &
!grepl("e-mail$", sessie_naam, ignore.case = TRUE, perl = TRUE) |
grepl("Follow-Up", sessie_naam, ignore.case = TRUE) &
!grepl("Tweede afstemmingsoverleg ipv", sessie_naam, ignore.case = TRUE, perl = TRUE) |
grepl("verrichting", sessie_naam, ignore.case = TRUE) |
grepl("Beschrijvende diagnostiek", sessie_naam, ignore.case = TRUE)|
grepl("Bewustwording", sessie_naam, ignore.case = TRUE)|
grepl("Consult specialist", sessie_naam, ignore.case = TRUE)|
grepl("Diagnostiek", sessie_naam, ignore.case = TRUE)|
grepl("Dubbele behandelsessies", sessie_naam, ignore.case = TRUE)|
grepl("Eerste", sessie_naam, ignore.case = TRUE)|
grepl("Eindgesprek optioneel", sessie_naam, ignore.case = TRUE)|
grepl("F2F interventie", sessie_naam, ignore.case = TRUE)|
grepl("Fase 1", sessie_naam, ignore.case = TRUE)|
grepl("Fase 2", sessie_naam, ignore.case = TRUE)|
grepl("Fase 3", sessie_naam, ignore.case = TRUE)|
grepl("Gesprek hoofdbehandelaar", sessie_naam, ignore.case = TRUE)|
grepl("Grenzen aangeven", sessie_naam, ignore.case = TRUE)|
grepl("Intake", sessie_naam, ignore.case = TRUE)&
!grepl("^Opstellen", sessie_naam, ignore.case = TRUE)&
!grepl("^Uitwerken", sessie_naam, ignore.case = TRUE)&
!grepl("werkgever$", sessie_naam, ignore.case = TRUE)|
grepl("Kennismakingsgesprek", sessie_naam, ignore.case = TRUE)|
grepl("Korte behandelsessie", sessie_naam, ignore.case = TRUE)|
grepl("Nameting: Behandelaar leest", sessie_naam, ignore.case = TRUE)|
grepl("Nameting: Behandelaar leest", sessie_naam, ignore.case = TRUE)|
grepl("Psychiatrisch", sessie_naam, ignore.case = TRUE)&
!grepl("^Psychiatrisch.*[^1]5$", sessie_naam, ignore.case = TRUE, perl = TRUE)&
!grepl("^Psychiatrisch.*e-mail$", sessie_naam, ignore.case = TRUE)&
!grepl("^Psychiatrisch.*tele consult$", sessie_naam, ignore.case = TRUE, perl = TRUE) &
!grepl("^Psychiatrisch.*tele consult \\(telefonisch / e-mail\\)$", sessie_naam, ignore.case = TRUE, perl = TRUE)|
grepl("Psychiatrisch vervolg consult (farmacotherapie)", sessie_naam, ignore.case = TRUE)|
grepl("Regie behandelaar", sessie_naam, ignore.case = TRUE)|
grepl("Regie overdracht kennismaking", sessie_naam, ignore.case = TRUE)|
grepl("Screening", sessie_naam, ignore.case = TRUE)&
!grepl("Uitwerktijd", sessie_naam, ignore.case = TRUE)|
grepl("Tele consult", sessie_naam, ignore.case = TRUE)|
grepl("Telefonisch consult", sessie_naam, ignore.case = TRUE)|
grepl("Terugvalpreventie", sessie_naam, ignore.case = TRUE)|
grepl("Terugvalpreventie", sessie_naam, ignore.case = TRUE)|
grepl("Time Management", sessie_naam, ignore.case = TRUE)|
grepl("Tweede gesprek", sessie_naam, ignore.case = TRUE)|
grepl("Tweede intake gesprek", sessie_naam, ignore.case = TRUE)|
grepl("Zelfwaardering", sessie_naam, ignore.case = TRUE)) %>%
filter(sessie_naam != "Psychiatrisch tele consult (telefonisch / e-mail)")%>%
filter(sessie_naam != "Tweede follow-up ipv afstemmingsoverleg")%>%
filter(sessie_naam != "Psychiatrisch tele consult")%>%
filter(!grepl("^Tele consult.*nt$", sessie_naam, ignore.case = TRUE))%>%
filter(sessie_naam != "Uitwerktijd Intakeverslag")%>%
filter(sessie_naam != "Telefonische follow-up")%>%
select(dossier_id_pseudo, sessie_datum_starttijd,
sessie_naam, sessie_tijd_direct,
sessie_tijd_indirect_indirect,
sessie_tijd_indirect_uitwerk)
# for clients who have connecting session
Treatment_session_2 <- Session %>%
filter(dossier_id_pseudo %in% data1$dossier_id_pseudo)%>%
left_join(data1) %>%
group_by(client_id_pseudo)%>%
mutate(dossier_id_pseudo=first(dossier_id_pseudo))%>% # merge two consecutive treatment episodes
filter(sessie_tijd_direct != 0)%>%
filter(grepl("^Behandelsessie", sessie_naam, ignore.case = TRUE, perl = TRUE) &
!grepl("e-mail$", sessie_naam, ignore.case = TRUE, perl = TRUE) |
grepl("Follow-Up", sessie_naam, ignore.case = TRUE) &
!grepl("Tweede afstemmingsoverleg ipv", sessie_naam, ignore.case = TRUE, perl = TRUE) |
grepl("verrichting", sessie_naam, ignore.case = TRUE) |
grepl("Beschrijvende diagnostiek", sessie_naam, ignore.case = TRUE)|
grepl("Bewustwording", sessie_naam, ignore.case = TRUE)|
grepl("Consult specialist", sessie_naam, ignore.case = TRUE)|
grepl("Diagnostiek", sessie_naam, ignore.case = TRUE)|
grepl("Dubbele behandelsessies", sessie_naam, ignore.case = TRUE)|
grepl("Eerste", sessie_naam, ignore.case = TRUE)|
grepl("Eindgesprek optioneel", sessie_naam, ignore.case = TRUE)|
grepl("F2F interventie", sessie_naam, ignore.case = TRUE)|
grepl("Fase 1", sessie_naam, ignore.case = TRUE)|
grepl("Fase 2", sessie_naam, ignore.case = TRUE)|
grepl("Fase 3", sessie_naam, ignore.case = TRUE)|
grepl("Gesprek hoofdbehandelaar", sessie_naam, ignore.case = TRUE)|
grepl("Grenzen aangeven", sessie_naam, ignore.case = TRUE)|
grepl("Intake", sessie_naam, ignore.case = TRUE)&
!grepl("^Opstellen", sessie_naam, ignore.case = TRUE)&
!grepl("^Uitwerken", sessie_naam, ignore.case = TRUE)&
!grepl("werkgever$", sessie_naam, ignore.case = TRUE)|
grepl("Kennismakingsgesprek", sessie_naam, ignore.case = TRUE)|
grepl("Korte behandelsessie", sessie_naam, ignore.case = TRUE)|
grepl("Nameting: Behandelaar leest", sessie_naam, ignore.case = TRUE)|
grepl("Nameting: Behandelaar leest", sessie_naam, ignore.case = TRUE)|
grepl("Psychiatrisch", sessie_naam, ignore.case = TRUE)&
!grepl("^Psychiatrisch.*[^1]5$", sessie_naam, ignore.case = TRUE, perl = TRUE)&
!grepl("^Psychiatrisch.*e-mail$", sessie_naam, ignore.case = TRUE)&
!grepl("^Psychiatrisch.*tele consult$", sessie_naam, ignore.case = TRUE, perl = TRUE) &
!grepl("^Psychiatrisch.*tele consult \\(telefonisch / e-mail\\)$", sessie_naam, ignore.case = TRUE, perl = TRUE)|
grepl("Psychiatrisch vervolg consult (farmacotherapie)", sessie_naam, ignore.case = TRUE)|
grepl("Regie behandelaar", sessie_naam, ignore.case = TRUE)|
grepl("Regie overdracht kennismaking", sessie_naam, ignore.case = TRUE)|
grepl("Screening", sessie_naam, ignore.case = TRUE)&
!grepl("Uitwerktijd", sessie_naam, ignore.case = TRUE)|
grepl("Tele consult", sessie_naam, ignore.case = TRUE)|
grepl("Telefonisch consult", sessie_naam, ignore.case = TRUE)|
grepl("Terugvalpreventie", sessie_naam, ignore.case = TRUE)|
grepl("Terugvalpreventie", sessie_naam, ignore.case = TRUE)|
grepl("Time Management", sessie_naam, ignore.case = TRUE)|
grepl("Tweede gesprek", sessie_naam, ignore.case = TRUE)|
grepl("Tweede intake gesprek", sessie_naam, ignore.case = TRUE)|
grepl("Zelfwaardering", sessie_naam, ignore.case = TRUE)) %>%
filter(sessie_naam != "Psychiatrisch tele consult (telefonisch / e-mail)")%>%
filter(sessie_naam != "Tweede follow-up ipv afstemmingsoverleg")%>%
filter(sessie_naam != "Psychiatrisch tele consult")%>%
filter(!grepl("^Tele consult.*nt$", sessie_naam,ignore.case = TRUE))%>%
filter(sessie_naam != "Uitwerktijd Intakeverslag")%>%
filter(sessie_naam != "Telefonische follow-up")%>%
ungroup()%>%
select(dossier_id_pseudo, sessie_datum_starttijd,
sessie_naam, sessie_tijd_direct,
sessie_tijd_indirect_indirect,
sessie_tijd_indirect_uitwerk)
Treatment_session <- rbind(Treatment_session_1, Treatment_session_2)%>%
arrange(dossier_id_pseudo,sessie_datum_starttijd)%>%
group_by(dossier_id_pseudo)%>%
mutate(number_sessions=n())%>%
ungroup() # direct treatment session for all clients
# for double check
# compare the filtered treatment session with session offered by Maarten
Test_name <- Treatment_session %>%
group_by(sessie_naam) %>% # group by session name
summarise(count = n()) %>% # count the
arrange(desc(count))
Also compared the number of indirect treatment session and direct treatment session
library(dplyr)
library(lubridate)
library(purrr)
#first merge the different episode
# find the each dossier ID score
SQ_48_1 <- SQ_48_score %>%
filter(!is.na(dossier_id_pseudo))%>%
filter(!dossier_id_pseudo %in% data1$dossier_id_pseudo)
SQ_48_2 <- SQ_48_score %>%
filter(dossier_id_pseudo %in% data1$dossier_id_pseudo)%>%
group_by(client_id_pseudo)%>%
arrange(client_id_pseudo,test_datum_ingevuld)%>%
mutate(dossier_id_pseudo=first(dossier_id_pseudo))
SQ_48_total <- rbind(SQ_48_1,SQ_48_2) %>%
filter(dossier_id_pseudo %in% New_Trajectory$dossier_id_pseudo)%>%
filter(grepl("SQ", test_naam))
SQ_48_total$test_datum_ingevuld <- as.Date(SQ_48_total$test_datum_ingevuld, format = "%Y-%m-%d")
# the function for find the nearest session for each score
find_closest_treatment_date <- function(test_datum_ingevuld, dossier_id_pseudo, df_treatments) {
test_datum_ingevuld <- as.Date(test_datum_ingevuld)
patient_treatment_dates <- as.Date(df_treatments$sessie_datum_starttijd[df_treatments$dossier_id_pseudo == dossier_id_pseudo])
if (length(patient_treatment_dates) == 0) {
return(NA_real_)
} else {
return(patient_treatment_dates[which.min(abs(patient_treatment_dates - test_datum_ingevuld))])
}
}
# the function to find which session is nearest
find_treatment_number <- function(test_datum_ingevuld, dossier_id_pseudo, df_treatments) {
test_datum_ingevuld <- as.Date(test_datum_ingevuld)
patient_treatment_dates <- as.Date(df_treatments$sessie_datum_starttijd[df_treatments$dossier_id_pseudo == dossier_id_pseudo])
if (length(patient_treatment_dates) == 0) {
return(NA_integer_)
} else {
return(which.min(abs(patient_treatment_dates - test_datum_ingevuld)))
}
}
# define a function to calculate the T-score
calculate_T_SCORE <- function(RS_SQ48) {
T_SCORE <- 19.40608979088534 +
0.5941681191862558 * RS_SQ48 +
-0.001812735748786516 * RS_SQ48^2 +
6.960825359362865e-006 * RS_SQ48^3
return(T_SCORE)
}
# apply the function to find the nearest treatment session
SQ_48_total_score <-SQ_48_total %>%
filter(dossier_id_pseudo %in% Treatment_session$dossier_id_pseudo)%>%
select(dossier_id_pseudo,test_datum_ingevuld,test_score)%>%
arrange(dossier_id_pseudo,test_datum_ingevuld)%>%
rowwise() %>%
mutate(
closest_treatment_date = find_closest_treatment_date(test_datum_ingevuld,dossier_id_pseudo,Treatment_session),
treatment_number =find_treatment_number(test_datum_ingevuld,dossier_id_pseudo,Treatment_session),
test_score = calculate_T_SCORE(test_score)
) %>%
distinct(dossier_id_pseudo,test_datum_ingevuld,.keep_all = TRUE)
print(head(SQ_48_total_score))
## # A tibble: 6 × 5
## # Rowwise:
## dossier_id_pseudo test_datum_ingevuld test_score closest_treatment_date
## <chr> <date> <dbl> <date>
## 1 00000010-9E77-42E0-AA15… 2022-07-11 51.4 2022-07-14
## 2 00000010-9E77-42E0-AA15… 2022-09-08 60.2 2022-09-06
## 3 00000010-9E77-42E0-AA15… 2022-11-28 55.4 2022-11-29
## 4 000DE9B4-080D-41F3-B90F… 2022-02-09 48.2 2022-02-09
## 5 001E6F1A-E1FE-43AC-AC50… 2022-06-15 55.4 2022-06-15
## 6 001E6F1A-E1FE-43AC-AC50… 2022-09-21 45.9 2022-09-21
## # ℹ 1 more variable: treatment_number <int>
# FILTER OUT CLIENTS WHO HAS LESS THAN 2 SCORES AND FIRST SCORE IS LESS THAN 42
SQ_48_filter_mark <- SQ_48_total_score %>%
group_by(dossier_id_pseudo) %>%
arrange(dossier_id_pseudo, treatment_number) %>% # Ensure the order is correct
mutate(number_score = n(),
first_score = first(test_score)) %>%
filter(number_score<=1|first_score<42.5) %>% # filter these clients who has only one score and intake score less than 42.5
ungroup() %>%
distinct(dossier_id_pseudo) # N=758 excluded
After excluding those clients who have only one sq48 score and whose intake score is less than 42.5. we calculated the response status based on responder definition provided by Maarten
– Five less than pre-treatment and test < 42.5 Assumption: If a client’s last response status is ‘responsive,’ then this patient is considered a responder.
SQ_48_score_responsive_status <- SQ_48_total_score %>%
filter(! dossier_id_pseudo %in% SQ_48_filter_mark$dossier_id_pseudo) %>% # N=1904
dplyr::group_by(dossier_id_pseudo) %>%
dplyr::arrange(dossier_id_pseudo, test_datum_ingevuld) %>% # Arrange rows within each group
# Calculate the first score within each group and broadcast it to all rows in the same group
dplyr::mutate(row_num = row_number()) %>%
dplyr::mutate(first_score = first(test_score)) %>%
# Perform the calculation: (test_score - first_score)
dplyr::mutate(calc_result = (first_score - test_score)) %>%
# Update the intake_status column based on the calculated result
dplyr::mutate(responsive_status = case_when(
row_num == 1 ~ "intake",
calc_result >= 5 & test_score < 42.5 ~ "response", # five less than pre-treatment and T-score < 42.5
TRUE ~ "non-response"
)) %>%
# Select and rename columns (optional)
dplyr::select(
dossier_id_pseudo, # Keep as is
test_datum_ingevuld,
test_score,
calc_result,
responsive_status
) %>% ungroup()
# Check the final responsive status
Responsive_status <- SQ_48_score_responsive_status %>%
# Group the data by dossier_id_pseudo
dplyr::group_by(dossier_id_pseudo) %>%
dplyr::mutate(last_responsive_status = last(responsive_status))%>%
ungroup()
Responsive_patients <- Responsive_status %>% # select the responders
filter(last_responsive_status == "response") %>%
group_by(dossier_id_pseudo)%>%
mutate(Test_interval = row_number()) %>%
ungroup()
# filter the patient who are responsive
Non_Responsive_patients <- Responsive_status %>% # select non-responders
filter(last_responsive_status == "non-response") %>%
group_by(dossier_id_pseudo)%>%
mutate(Test_interval = row_number()) %>%
ungroup()
# calculate the responsive ratio and non-responsive ratio
Responsive_ratio <- n_distinct(Responsive_patients$dossier_id_pseudo)/n_distinct(Responsive_status$dossier_id_pseudo)
Non_Responsive_ratio <- n_distinct(Non_Responsive_patients$dossier_id_pseudo)/n_distinct(Responsive_status$dossier_id_pseudo)
ratios_data <- data.frame(
Type = c("Responsive", "Non-Responsive"),
Ratio = c(Responsive_ratio, Non_Responsive_ratio),
Clients=c(n_distinct(Responsive_patients$dossier_id_pseudo),
n_distinct(Non_Responsive_patients$dossier_id_pseudo))
)
print(ratios_data)
## Type Ratio Clients
## 1 Responsive 0.6163222 1163
## 2 Non-Responsive 0.3836778 724
Due to clients delaying in completing the questionnaires sent by therapists, the test scores do not precisely reflect the effects after every five treatment sessions, resulting in some treatment scores being missing.
Discussion point 1: To filter out this kind of missing data, we have established a rule: the interval between any two measurement scores must be more than 3 treatment sessions and less than 7 treatment sessions.
For example, consider the scoring timeline for a client: The initial score is recorded after the first treatment session, which is the intake session. The next score is recorded after the sixth treatment session, indicating a gap of five sessions between the two scores. Subsequently, the third score is noted around the thirteenth treatment session, making a difference of seven sessions since the previous score. Based on our definition, this clients will included in our analysis
Discussion point 2:
In our model, ‘recovered’ is the absorbing state. This means that the ‘recovered’ row in the transition probability matrix is set to 1. Consequently, once a patient reaches the recovered state, they cannot transition to other states. This approach is also consistent with real-world practice. However, in our data set, we saw some patient got relapse after recovered
Can we assumed that: if a client has reached the ‘recovered’ state, they will remain in this state in our data set. Does this make sense ?
# 1. Filtering out scores with excessive time intervals
SQ_48_filter_mark_1 <- SQ_48_total_score %>%
group_by(dossier_id_pseudo) %>%
arrange(dossier_id_pseudo, treatment_number) %>% # Ensure the order is correct
mutate(diff = treatment_number - lead(treatment_number)) %>% # Else calculate the difference between current row and next row
mutate(new_column = (diff < -2 & diff > -8) | is.na(diff)) %>%
# marked the consecutive score more than 2 session and less than 8 session
filter(any(new_column == FALSE)) %>%
ungroup() %>% distinct(dossier_id_pseudo) # 1066 clients excluded
SQ_t48_complete <- Responsive_status %>% # N=796
filter(!dossier_id_pseudo %in% SQ_48_filter_mark_1$dossier_id_pseudo)%>%
mutate(
states = case_when(
test_score < 42.5 ~ "recovered", # lower than 39.26 is recover
test_score >= 42.5 & test_score < 60 ~ "mild", # between 39.26 to 59.9 is mild
test_score >= 60 & test_score < 70 ~ "moderate", # between 60-70 is moderate
test_score >=70 ~ "severe", # over than 70 is severe
TRUE ~ "UNKOWN" # for any other cases, which ideally should not exits
)
)
# if the patient recovered than it won't transfer to next state
# change the data
SQ_t48_modified <- SQ_t48_complete %>%
group_by(dossier_id_pseudo) %>%
# Create a flag to indicate whether there is a 'recovered' state in the group
mutate(has_recovered = any(states == "recovered")) %>%
# If 'recovered' exists, find the position of the last 'recovered'
mutate(last_recovered = ifelse(has_recovered, min(which(states == "recovered")), NA_integer_)) %>%
# Mark whether the row is after the last 'recovered'
mutate(is_after_last_recovered = row_number() > last_recovered) %>%
# If the row is after the last 'recovered', change 'states' to 'recovered'
mutate(states = ifelse(has_recovered & is_after_last_recovered, "recovered", states)) %>%
select(-has_recovered, -last_recovered, -is_after_last_recovered) %>%
ungroup()
# Function to calculate the transition matrix for a given status
calculate_transition_matrix <- function(data, status){
data_filtered <- data %>%
filter(last_responsive_status == status) %>%
group_by(dossier_id_pseudo) %>%
arrange(dossier_id_pseudo, test_datum_ingevuld) %>%
mutate(row = row_number())
states <- c("mild", "moderate", "severe", "recovered")
transition_matrix <- matrix(0, nrow = length(states), ncol = length(states))
rownames(transition_matrix) <- states
colnames(transition_matrix) <- states
for (id in unique(data_filtered$dossier_id_pseudo)){
patient_data <- data_filtered[data_filtered$dossier_id_pseudo == id, ]
for (i in 1:(nrow(patient_data) - 1)){
from_state <- patient_data$states[i]
to_state <- patient_data$states[i + 1]
transition_matrix[from_state, to_state] <- transition_matrix[from_state, to_state] + 1
}
}
transition_prob_matrix <- transition_matrix / rowSums(transition_matrix)
return(transition_prob_matrix)
}
# Select relevant columns
Data_transition_probability <- SQ_t48_modified %>%
select(dossier_id_pseudo, last_responsive_status, test_datum_ingevuld, test_score, states)
# Calculate the transition probabilities
transition_prob_matrix_response <- calculate_transition_matrix(Data_transition_probability, "response")
transition_prob_matrix_nonresponse <- calculate_transition_matrix(Data_transition_probability, "non-response")
print("transition matrix for responders")
## [1] "transition matrix for responders"
print(transition_prob_matrix_response)
## mild moderate severe recovered
## mild 0.3632708 0.01608579 0.00000000 0.62064343
## moderate 0.5943396 0.12264151 0.01886792 0.26415094
## severe 0.6956522 0.13043478 0.08695652 0.08695652
## recovered 0.0000000 0.00000000 0.00000000 1.00000000
print("transition matrix for non-responders")
## [1] "transition matrix for non-responders"
print(transition_prob_matrix_nonresponse)
## mild moderate severe recovered
## mild 0.8896882 0.04796163 0.002398082 0.05995204
## moderate 0.5337423 0.41104294 0.042944785 0.01226994
## severe 0.4857143 0.40000000 0.114285714 0.00000000
## recovered 0.0000000 0.00000000 0.000000000 1.00000000
# write matrix into CSV
write.csv(transition_prob_matrix_response, file = "transition_prob_matrix_responder_complete_data.csv", row.names = FALSE)
write.csv(transition_prob_matrix_nonresponse, file = "transition_prob_matrix_Nonresponder_complete_data.csv", row.names = FALSE)
Since we excluded 1066 clients due to the lack of suitable SQ48 scores, we are attempting to impute the missing SQ48 scores using single imputation
Identify the appropriate treatment session for each score. For instance, if the second score is recorded after the 6th session, we assume this score represents the 5th session. Similarly, if a score is recorded after the 11th session, we consider this score as representative of the 10th session, and so on.
Determine the missing scores for every set of five treatment sessions. For example, if a client has scores only for the intake session, the 10th session, and the 20th session, it implies that scores for the 5th and 15th sessions are missing.
imputing the missing score by single imputation
calculate the transition probabilities
## Define a function to find the largest multiple of 5 that is less than a given number
find_nearest_multiple_of_five <- function(x) {
lower = (x %/% 5) * 5
upper = lower + 5
ifelse(abs(x - lower) <= abs(x - upper), lower, upper)
}
SQ_48_score_long <- SQ_48_total_score %>%
filter(!dossier_id_pseudo %in% SQ_48_filter_mark$dossier_id_pseudo) %>%
group_by(dossier_id_pseudo) %>%
mutate(find_nearest_multiple_of_five = ifelse(row_number() > 1,
find_nearest_multiple_of_five(treatment_number),
1))%>%
filter(find_nearest_multiple_of_five>0) %>%
arrange(dossier_id_pseudo, find_nearest_multiple_of_five, desc(treatment_number)) %>%
distinct(dossier_id_pseudo,find_nearest_multiple_of_five,.keep_all = TRUE)%>%
ungroup
library(tidyr)
SQ_48_score_wider <- SQ_48_score_long %>%
group_by(dossier_id_pseudo)%>%
mutate(column_name = paste0("col_", find_nearest_multiple_of_five)) %>%
rowwise() %>%
mutate(value = test_score) %>%
ungroup() %>%
pivot_wider(names_from = column_name, values_from = value, values_fill = list(value = NA))%>%
select(dossier_id_pseudo, test_datum_ingevuld, test_score,
closest_treatment_date, treatment_number,
find_nearest_multiple_of_five,
col_1, col_5, col_10, col_15, col_20, col_25, col_30, col_35,
col_40, col_45, col_50, col_55, col_60, col_65, col_70)
# combine the same row into one
SQ_48_score_tidy <- SQ_48_score_wider %>%
group_by(dossier_id_pseudo) %>%
summarise(across(starts_with("col_"), ~na.omit(.)[1])) %>%
ungroup()
SQ_48_score_long <- SQ_48_score_tidy %>%
pivot_longer(
cols = starts_with("col_"),
names_to = "treatment_session",
values_to = "test_score"
)%>%
group_by(dossier_id_pseudo) %>%
mutate(last_non_na = max(which(!is.na(test_score)))) %>%
filter(row_number() <= last_non_na) %>%
select(-last_non_na)%>%
ungroup()
# inmpute the missing data
library(mice)
Imputate_date <- SQ_48_score_long %>% select(-dossier_id_pseudo)
imputed_model <- mice(Imputate_date, m = 1, method = 'norm', maxit = 50)
##
## iter imp variable
## 1 1 test_score
## 2 1 test_score
## 3 1 test_score
## 4 1 test_score
## 5 1 test_score
## 6 1 test_score
## 7 1 test_score
## 8 1 test_score
## 9 1 test_score
## 10 1 test_score
## 11 1 test_score
## 12 1 test_score
## 13 1 test_score
## 14 1 test_score
## 15 1 test_score
## 16 1 test_score
## 17 1 test_score
## 18 1 test_score
## 19 1 test_score
## 20 1 test_score
## 21 1 test_score
## 22 1 test_score
## 23 1 test_score
## 24 1 test_score
## 25 1 test_score
## 26 1 test_score
## 27 1 test_score
## 28 1 test_score
## 29 1 test_score
## 30 1 test_score
## 31 1 test_score
## 32 1 test_score
## 33 1 test_score
## 34 1 test_score
## 35 1 test_score
## 36 1 test_score
## 37 1 test_score
## 38 1 test_score
## 39 1 test_score
## 40 1 test_score
## 41 1 test_score
## 42 1 test_score
## 43 1 test_score
## 44 1 test_score
## 45 1 test_score
## 46 1 test_score
## 47 1 test_score
## 48 1 test_score
## 49 1 test_score
## 50 1 test_score
# join impute data with clients' ID
SQ_48_score_imputed_data <- complete(imputed_model, 1)%>%
cbind(select(SQ_48_score_long,1))
SQ_48_score_responsive_status_imputed <- SQ_48_score_imputed_data %>%
dplyr::group_by(dossier_id_pseudo) %>%
# Calculate the first score within each group and broadcast it to all rows in the same group
dplyr::mutate(row_num = row_number()) %>%
dplyr::mutate(first_score = first(test_score)) %>%
# Perform the calculation: (test_score - first_score)
dplyr::mutate(calc_result = (first_score - test_score)) %>%
mutate(score_number=n())%>%
# Update the intake_status column based on the calculated result
dplyr::mutate(responsive_status = case_when(
row_num == 1 ~ "intake",
calc_result >= 5 & test_score < 42.5 ~ "response", # five less than pre-treatment and T-score >42.5
TRUE ~ "non-response"
)) %>%
filter(score_number>1)%>%
# Select and rename columns (optional)
dplyr::select(
dossier_id_pseudo, # Keep as is
test_score,
treatment_session,
calc_result,
responsive_status,
score_number
) %>% ungroup() # N=1879
# Check the final responsive status
Responsive_status_imputed <- SQ_48_score_responsive_status_imputed %>%
# Group the data by dossier_id_pseudo
dplyr::group_by(dossier_id_pseudo) %>%
dplyr::mutate(last_responsive_status = last(responsive_status))%>%
ungroup()
Responsive_patients_impute <- Responsive_status_imputed %>% # select the responders
filter(last_responsive_status == "response")
# filter the patient who are responsive
Non_Responsive_patients_impute <- Responsive_status_imputed %>% # select non-responders
filter(last_responsive_status == "non-response")
# responsive ratio
n_distinct(Responsive_patients_impute$dossier_id_pseudo)/n_distinct(Responsive_status_imputed$dossier_id_pseudo)
## [1] 0.6248004
Depression_states <- Responsive_status_imputed%>%
select(dossier_id_pseudo, treatment_session, test_score,last_responsive_status)%>%
mutate(
states = case_when(
test_score < 42.5 ~ "recovered", # lower than 39.26 is recover
test_score >= 42.5 & test_score < 60 ~ "mild", # between 39.26 to 59.9 is mild
test_score >= 60 & test_score < 70 ~ "moderate", # between 60-70 is moderate
test_score >=70 ~ "severe", # over than 70 is severe
TRUE ~ "UNKOWN" # for any other cases, which ideally should not exits
)
)
# if the patient recovered than it won't transfer to next state
# change the data
SQ_t48_imputed_modified <- Depression_states %>%
group_by(dossier_id_pseudo) %>%
# Create a flag to indicate whether there is a 'recovered' state in the group
mutate(has_recovered = any(states == "recovered")) %>%
# If 'recovered' exists, find the position of the last 'recovered'
mutate(last_recovered = ifelse(has_recovered, min(which(states == "recovered")), NA_integer_)) %>%
# Mark whether the row is after the last 'recovered'
mutate(is_after_last_recovered = row_number() > last_recovered) %>%
# If the row is after the last 'recovered', change 'states' to 'recovered'
mutate(states = ifelse(has_recovered & is_after_last_recovered, "recovered", states)) %>%
mutate(score_number = n())%>%
select(-has_recovered, -last_recovered, -is_after_last_recovered) %>%
ungroup()
# Function to calculate the transition matrix for a given status
calculate_transition_matrix_imputed <- function(data, status){
data_filtered <- data %>%
filter(last_responsive_status == status) %>%
group_by(dossier_id_pseudo) %>%
#arrange(dossier_id_pseudo, test_datum_ingevuld) %>%
mutate(row = row_number())
states <- c("mild", "moderate", "severe", "recovered")
transition_matrix <- matrix(0, nrow = length(states), ncol = length(states))
rownames(transition_matrix) <- states
colnames(transition_matrix) <- states
for (id in unique(data_filtered$dossier_id_pseudo)){
patient_data <- data_filtered[data_filtered$dossier_id_pseudo == id, ]
for (i in 1:(nrow(patient_data) - 1)){
from_state <- patient_data$states[i]
to_state <- patient_data$states[i + 1]
transition_matrix[from_state, to_state] <- transition_matrix[from_state, to_state] + 1
}
}
transition_prob_matrix <- transition_matrix / rowSums(transition_matrix)
return(transition_prob_matrix)
}
# Select relevant columns
Data_transition_probability_imputed <- SQ_t48_imputed_modified %>%
select(dossier_id_pseudo, last_responsive_status, test_score, states)
# Calculate the transition probabilities
transition_prob_matrix_response_imputed <- calculate_transition_matrix_imputed(Data_transition_probability_imputed, "response")
transition_prob_matrix_nonresponse_imputed <-
calculate_transition_matrix_imputed(Data_transition_probability_imputed, "non-response")
print("transition matrix for responders with imputed data")
## [1] "transition matrix for responders with imputed data"
print(transition_prob_matrix_response_imputed)
## mild moderate severe recovered
## mild 0.4267003 0.04937028 0.00604534 0.5178841
## moderate 0.5092838 0.12732095 0.00795756 0.3554377
## severe 0.6000000 0.14545455 0.03636364 0.2181818
## recovered 0.0000000 0.00000000 0.00000000 1.0000000
print("transition matrix for responders with completed data")
## [1] "transition matrix for responders with completed data"
print(transition_prob_matrix_response)
## mild moderate severe recovered
## mild 0.3632708 0.01608579 0.00000000 0.62064343
## moderate 0.5943396 0.12264151 0.01886792 0.26415094
## severe 0.6956522 0.13043478 0.08695652 0.08695652
## recovered 0.0000000 0.00000000 0.00000000 1.00000000
print("transition matrix for non-responders with imputed data")
## [1] "transition matrix for non-responders with imputed data"
print(transition_prob_matrix_nonresponse_imputed)
## mild moderate severe recovered
## mild 0.7252475 0.09735974 0.02722772 0.1501650
## moderate 0.5630435 0.28478261 0.03695652 0.1152174
## severe 0.5233645 0.21495327 0.10280374 0.1588785
## recovered 0.0000000 0.00000000 0.00000000 1.0000000
print("transition matrix for non-responders with completed")
## [1] "transition matrix for non-responders with completed"
print(transition_prob_matrix_nonresponse)
## mild moderate severe recovered
## mild 0.8896882 0.04796163 0.002398082 0.05995204
## moderate 0.5337423 0.41104294 0.042944785 0.01226994
## severe 0.4857143 0.40000000 0.114285714 0.00000000
## recovered 0.0000000 0.00000000 0.000000000 1.00000000
# write matrix into CSV
write.csv(transition_prob_matrix_response_imputed,
file = "transition_prob_matrix_responder_imputed_data.csv", row.names = FALSE)
write.csv(transition_prob_matrix_nonresponse_imputed,
file = "transition_prob_matrix_Nonresponder_imputed_data.csv", row.names = FALSE)
Discussion point 4: which one should be used in main analysis