Problem Definition
Estimate survival time of patients with primary biliary cirrhosis (PBC), a rare autoimmune liver disease.
Data
Patient genealogy
Time-series: Bilirubin levels measured over time
Data ETL
dat <- pbc2
dat <- data.frame(lapply(dat, # Using Base R functions
function(x) if(is.numeric(x)) round(x, 3) else x))
PBC Data
Patient genealogy
pbc2.id %>% dplyr::group_by(status) %>% dplyr::summarise(mean_life = mean(years, na.rm = T))
#mean(pbc2.id[pbc2.id$status2 == 1, ]$years, na.rm = T)
pbc2.id$status2 <- as.numeric(pbc2.id$status2)
fit <- survfit(Surv(years, status2) ~ 1, data = pbc2.id)
plot(fit, xlab = "Years",
ylab = "Survival probability")

NA
NA
Bilirubin levels measured over time for each patient
lattice::xyplot(log(serBilir) ~ year | id, group = id,
data = pbc2[pbc2$id %in% c(1:10),],
xlab = "Years", ylab = expression(log("serBilir")), col = 1, type = "b")

Correlation of bilirubin level measurements to patients’ state (dead/alive)
Notice patient-to-patient variations
xyplot(log(serBilir) ~ year | as.factor(status2),
group = id, data = pbc2,
panel = function(x, y, ...){panel.xyplot(x, y, type = "l", col = 1, ...)
panel.loess(x, y, col = 2, lwd = 2)}, main = "",
ylab = expression(log ("serBilir")), xlab = "Years")

Model for Forecast Survival Time
tic("Build survival model...")
use_linear_model <- 1
use_nonlinear_model <- 0
if(use_linear_model){
## Linear model
lmeFit <- lme(log(serBilir) ~ drug*(year), random = ~ year|id, data = pbc2)
coxFit <- coxph(Surv(years, status2) ~ drug + prothrombin, data = pbc2.id, x = TRUE)
jmFit <- jointModel(lmeFit, coxFit, timeVar = "year", method = "weibull-AFT-GH")
}
if(use_nonlinear_model){
# Nonlinear model
lmeFit <-
lme(log(serBilir) ~ drug*(year + I(year^2)), random = ~ year + I(year^2)|id, data = pbc2)
coxFit <- coxph(Surv(years, status2) ~ drug + prothrombin, data = pbc2.id, x = TRUE)
jmFit <- jointModel(lmeFit, coxFit, timeVar = "year", method = "weibull-AFT-GH")
}
toc()
Build survival model...: 52.82 sec elapsed
Validation
Two methods:
Backtesting: Go back in time and assume that time is the present date, Forecast failure time and compare if they agreed with the real ground-truth failure time.
Forward validation: Implement predictions in the field and do post-mortem analysis
Get predictions for an example patient
bRunPred <- 1
bSavePlot <- 0
patient_id <- 2
tic()
ND <- pbc2[pbc2$id == patient_id, ]
survPreds <- vector("list", nrow(ND))
for (i in 1:nrow(ND)) {
set.seed(123)
survPreds[[i]] <- survfitJM(jmFit, newdata = ND[1:i, ])
}
par(mfrow = c(2, 2), oma = c(0, 2, 0, 2))
for (i in c(1,3,5,7)) {
plot(survPreds[[i]], estimator = "median", conf.int = TRUE,
include.y = TRUE, main = paste("Follow-up time:",
round(survPreds[[i]]$last.time, 1)))
}

toc()
4.07 sec elapsed
Result
Patient 180-day risk table
bGeneratePreds <- 1
if(bGeneratePreds){
ids_to_predict <- pbc2.id %>% dplyr::filter(status2 == 0) %>% dplyr::select(id) %>% pull()
surv_preds <- list()
for(i in ids_to_predict){
tmp_pred <-pbc2 %>% dplyr::filter(id == i)
last_time <- unique(tmp_pred$years)
tmp <- JM::survfitJM(jmFit, newdata = tmp_pred, last.time = last_time, idVar = "id", survTimes = last_time + 180/365)
pSurv_180day <- unlist(tmp[1]$summaries)[2]
surv_preds <- surv_preds %>% bind_rows(data.frame("id" = i, "psurv_180day" = pSurv_180day, "years" = last_time))
}
surv_preds <- surv_preds %>% dplyr::arrange(pSurv_180day)
us_cities <- maps::us.cities %>% dplyr::select(name, lat, long)
set.seed(123)
us_cities <- us_cities[sample(nrow(surv_preds)),]
surv_preds <- cbind(surv_preds, us_cities)
write.csv(surv_preds, "surv_preds.csv", row.names = F)
}
surv_preds$psurv_180day <- round(surv_preds$psurv_180day, digits = 4)
surv_preds$risk <- "Medium"
surv_preds$risk[surv_preds$psurv_180day > 0.8] <- "Low"
surv_preds$risk[surv_preds$psurv_180day <= 0.6] <- "High"
DT::datatable(surv_preds, options = list(scrollX = TRUE), class = 'cell-border stripe', rownames = FALSE)
NA
Patient 180-day Risk + Locations Info on a Map
surv_preds$id <- paste0("Patient ", surv_preds$id)
surv_preds_sf <- sf::st_as_sf(
x = surv_preds,
coords = c("long", "lat"), # columns with coordinates
crs = 'ESRI:102003' # coordinate reference system code for eastings/northings
) %>% sf::st_transform(crs = 'ESRI:102003') # the coord ref system code for latlong
saveRDS(surv_preds_sf, "surv_preds_sf.RDS")
sch <- surv_preds_sf
map <- sch %>%
leaflet::leaflet() %>%
leaflet::addProviderTiles(providers$OpenStreetMap) %>%
leaflet::addAwesomeMarkers(
popup = ~paste0(
"<h1>", sch$id, "</h1>",
"<table style='width:100%'>",
"<tr>",
"<th>RISK</th>",
"<th>", sch$risk, "</th>",
"</tr>",
"<tr>",
"<tr>",
"<th>pSurvival </th>",
"<th>", sch$psurv_180day, "</th>",
"</tr>",
"<tr>",
"<tr>",
"<th>city</th>",
"<th>", sch$name, "</th>",
"</tr>"
), # end popup()
icon = awesomeIcons(
library = "ion",
icon = ifelse(
test = sch$risk == "High",
yes = "ion-android-star-outline",
no = "ion-android-star-outline"
),
iconColor = "white",
markerColor = ifelse(
test = sch$risk == "High",
yes = "red",
no = ifelse(sch$risk == "Medium", yes = "orange", no = "green")
)
)
) %>% # end addAwesomeMarkers()
leaflet::addMeasure()
map
NA
Patient 180-day Risk + Ambulance Locations on a Map
- Re-distribute Resources (Ambulance + Labor) proactively
us_cities <- maps::us.cities %>% dplyr::select(name, lat, long) %>%
rename(lng = long)
set.seed(125)
ambulance_locations <- us_cities[sample(10),]
ambulance_locations$id <- 1:nrow(ambulance_locations)
ambulance_locations$id <- paste0("A", ambulance_locations$id)
map %>%
addPulseMarkers(
lng = ambulance_locations$lng,
lat = ambulance_locations$lat,
popup = ~paste0(
"<h1>", ambulance_locations$id, "</h1>"),
icon = makePulseIcon()
)
NA
LS0tDQp0aXRsZTogJ1Byb2dub3N0aWNzOiBQcmltYXJ5IEJpbGlhcnkgQ2lycmhvc2lzIChQQkMpIG9mIHRoZSBMaXZlcicNCmF1dGhvcjogIlNhdGlzaCBJeWVuZ2FyIg0KZGF0ZTogIjQvMTMvMjAyMiINCm91dHB1dDogaHRtbF9ub3RlYm9vaw0KDQotLS0NCg0KYGBge3Igc2V0dXAsIGluY2x1ZGU9RkFMU0V9DQprbml0cjo6b3B0c19jaHVuayRzZXQoZWNobyA9IFRSVUUpDQpgYGANCg0KYGBge3IgaW5zdGFsbF9wYWNrYWdlcywgaW5jbHVkZT1GQUxTRX0NCmxpYnJhcnkoSk0pDQpsaWJyYXJ5KGxhdHRpY2UpDQpsaWJyYXJ5KERUKQ0KbGlicmFyeSh0aWN0b2MpDQpsaWJyYXJ5KHN1cnZpdmFsKQ0KbGlicmFyeShkcGx5cikNCmxpYnJhcnkobGVhZmxldCkNCmxpYnJhcnkoZmxleGRhc2hib2FyZCkNCmxpYnJhcnkoc2YpDQpsaWJyYXJ5KGxlYWZsZXQuZXh0cmFzKQ0KDQpgYGANCg0KDQojIyBQcm9ibGVtIERlZmluaXRpb24NCi0gKipFc3RpbWF0ZSBzdXJ2aXZhbCB0aW1lIG9mIHBhdGllbnRzIHdpdGggcHJpbWFyeSBiaWxpYXJ5IGNpcnJob3NpcyAoUEJDKSwgYSByYXJlIGF1dG9pbW11bmUgbGl2ZXIgZGlzZWFzZS4qKg0KDQotIERhdGENCi0gUGF0aWVudCBnZW5lYWxvZ3kgDQotIFRpbWUtc2VyaWVzOiBCaWxpcnViaW4gbGV2ZWxzIG1lYXN1cmVkIG92ZXIgdGltZQ0KDQoNCiMjIERhdGEgRVRMDQoNCg0KYGBge3IsIGluY2x1ZGU9VFJVRX0NCmRhdCA8LSBwYmMyDQpkYXQgPC0gZGF0YS5mcmFtZShsYXBwbHkoZGF0LCAgICAjIFVzaW5nIEJhc2UgUiBmdW5jdGlvbnMNCiAgICAgICAgICAgICAgICAgICAgICAgICBmdW5jdGlvbih4KSBpZihpcy5udW1lcmljKHgpKSByb3VuZCh4LCAzKSBlbHNlIHgpKQ0KDQpgYGANCg0KDQojIyBQQkMgRGF0YQ0KDQojIyMgUGF0aWVudCBnZW5lYWxvZ3kNCg0KYGBge3Igdmlld19kYXRhLCBpbmNsdWRlPVRSVUUsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0UsIGVjaG89RkFMU0V9DQpEVDo6ZGF0YXRhYmxlKHBiYzIuaWQsIG9wdGlvbnMgPSBsaXN0KHNjcm9sbFggPSBUUlVFKSwgY2xhc3MgPSAnY2VsbC1ib3JkZXIgc3RyaXBlJywgcm93bmFtZXMgPSBGQUxTRSkNCg0KYGBgDQoNCmBgYHtyfQ0KDQpwYmMyLmlkICU+JSBkcGx5cjo6Z3JvdXBfYnkoc3RhdHVzKSAlPiUgZHBseXI6OnN1bW1hcmlzZShtZWFuX2xpZmUgPSBtZWFuKHllYXJzLCBuYS5ybSA9IFQpKQ0KI21lYW4ocGJjMi5pZFtwYmMyLmlkJHN0YXR1czIgPT0gMSwgXSR5ZWFycywgbmEucm0gPSBUKQ0KDQpgYGANCg0KDQoNCmBgYHtyfQ0KcGJjMi5pZCRzdGF0dXMyIDwtIGFzLm51bWVyaWMocGJjMi5pZCRzdGF0dXMyKQ0KZml0IDwtIHN1cnZmaXQoU3Vydih5ZWFycywgc3RhdHVzMikgfiAxLCBkYXRhID0gcGJjMi5pZCkNCnBsb3QoZml0LCB4bGFiID0gIlllYXJzIiwgDQogICAgIHlsYWIgPSAiU3Vydml2YWwgcHJvYmFiaWxpdHkiKQ0KDQoNCmBgYA0KDQoNCiMjIyBCaWxpcnViaW4gbGV2ZWxzIG1lYXN1cmVkIG92ZXIgdGltZSBmb3IgZWFjaCBwYXRpZW50DQoNCg0KYGBge3J9DQpsYXR0aWNlOjp4eXBsb3QobG9nKHNlckJpbGlyKSB+IHllYXIgfCBpZCwgZ3JvdXAgPSBpZCwgDQogICAgICAgICAgICAgICAgZGF0YSA9IHBiYzJbcGJjMiRpZCAlaW4lIGMoMToxMCksXSwgDQogICAgICAgICAgICAgICAgeGxhYiA9ICJZZWFycyIsIHlsYWIgPSBleHByZXNzaW9uKGxvZygic2VyQmlsaXIiKSksIGNvbCA9IDEsIHR5cGUgPSAiYiIpDQoNCmBgYA0KDQojIyMgQ29ycmVsYXRpb24gb2YgYmlsaXJ1YmluIGxldmVsIG1lYXN1cmVtZW50cyB0byBwYXRpZW50cycgc3RhdGUgKGRlYWQvYWxpdmUpDQoNCk5vdGljZSBwYXRpZW50LXRvLXBhdGllbnQgdmFyaWF0aW9ucw0KDQpgYGB7cn0NCnh5cGxvdChsb2coc2VyQmlsaXIpIH4geWVhciB8IGFzLmZhY3RvcihzdGF0dXMyKSwNCiAgICAgICBncm91cCA9IGlkLCBkYXRhID0gcGJjMiwgDQogICAgICAgcGFuZWwgPSBmdW5jdGlvbih4LCB5LCAuLi4pe3BhbmVsLnh5cGxvdCh4LCB5LCB0eXBlID0gImwiLCBjb2wgPSAxLCAuLi4pDQogICAgICAgICBwYW5lbC5sb2Vzcyh4LCB5LCBjb2wgPSAyLCBsd2QgPSAyKX0sIG1haW4gPSAiIiwgDQogICAgICAgeWxhYiA9IGV4cHJlc3Npb24obG9nICgic2VyQmlsaXIiKSksIHhsYWIgPSAiWWVhcnMiKSANCg0KYGBgDQoNCg0KDQoNCg0KDQojIyBNb2RlbCBmb3IgRm9yZWNhc3QgU3Vydml2YWwgVGltZQ0KDQoNCmBgYHtyIGJ1aWxkX2pvaW50X21vZGVsLCBpbmNsdWRlPVRSVUV9DQp0aWMoIkJ1aWxkIHN1cnZpdmFsIG1vZGVsLi4uIikNCnVzZV9saW5lYXJfbW9kZWwgPC0gMQ0KdXNlX25vbmxpbmVhcl9tb2RlbCA8LSAwDQoNCmlmKHVzZV9saW5lYXJfbW9kZWwpew0KICAjIyBMaW5lYXIgbW9kZWwNCiAgbG1lRml0IDwtIGxtZShsb2coc2VyQmlsaXIpIH4gZHJ1ZyooeWVhciksIHJhbmRvbSA9IH4geWVhcnxpZCwgZGF0YSA9IHBiYzIpDQogIGNveEZpdCA8LSBjb3hwaChTdXJ2KHllYXJzLCBzdGF0dXMyKSB+IGRydWcgKyBwcm90aHJvbWJpbiwgZGF0YSA9IHBiYzIuaWQsIHggPSBUUlVFKQ0KICBqbUZpdCA8LSBqb2ludE1vZGVsKGxtZUZpdCwgY294Rml0LCB0aW1lVmFyID0gInllYXIiLCBtZXRob2QgPSAid2VpYnVsbC1BRlQtR0giKQ0KfQ0KDQppZih1c2Vfbm9ubGluZWFyX21vZGVsKXsNCiAgIyBOb25saW5lYXIgbW9kZWwNCiAgbG1lRml0IDwtDQogICAgbG1lKGxvZyhzZXJCaWxpcikgfiBkcnVnKih5ZWFyICsgSSh5ZWFyXjIpKSwgcmFuZG9tID0gfiB5ZWFyICsgSSh5ZWFyXjIpfGlkLCBkYXRhID0gcGJjMikNCiAgY294Rml0IDwtIGNveHBoKFN1cnYoeWVhcnMsIHN0YXR1czIpIH4gZHJ1ZyArIHByb3Rocm9tYmluLCBkYXRhID0gcGJjMi5pZCwgeCA9IFRSVUUpDQogIGptRml0IDwtIGpvaW50TW9kZWwobG1lRml0LCBjb3hGaXQsIHRpbWVWYXIgPSAieWVhciIsIG1ldGhvZCA9ICJ3ZWlidWxsLUFGVC1HSCIpDQp9DQoNCnRvYygpDQoNCmBgYA0KDQojIyBWYWxpZGF0aW9uDQoNClR3byBtZXRob2RzOiANCg0KMS4gQmFja3Rlc3Rpbmc6IEdvIGJhY2sgaW4gdGltZSBhbmQgYXNzdW1lIHRoYXQgdGltZSBpcyB0aGUgcHJlc2VudCBkYXRlLCBGb3JlY2FzdCBmYWlsdXJlIHRpbWUgYW5kIGNvbXBhcmUgaWYgdGhleSBhZ3JlZWQgd2l0aCB0aGUgcmVhbCBncm91bmQtdHJ1dGggZmFpbHVyZSB0aW1lLg0KDQoyLiBGb3J3YXJkIHZhbGlkYXRpb246IEltcGxlbWVudCBwcmVkaWN0aW9ucyBpbiB0aGUgZmllbGQgYW5kIGRvIHBvc3QtbW9ydGVtIGFuYWx5c2lzDQoNCg0KDQojIyBHZXQgcHJlZGljdGlvbnMgZm9yIGFuIGV4YW1wbGUgcGF0aWVudA0KDQpgYGB7cn0NCmJSdW5QcmVkIDwtIDENCmJTYXZlUGxvdCA8LSAwDQpwYXRpZW50X2lkIDwtIDINCg0KdGljKCkNCg0KDQpORCA8LSBwYmMyW3BiYzIkaWQgPT0gcGF0aWVudF9pZCwgXQ0Kc3VydlByZWRzIDwtIHZlY3RvcigibGlzdCIsIG5yb3coTkQpKQ0KZm9yIChpIGluIDE6bnJvdyhORCkpIHsNCiAgc2V0LnNlZWQoMTIzKQ0KICBzdXJ2UHJlZHNbW2ldXSA8LSBzdXJ2Zml0Sk0oam1GaXQsIG5ld2RhdGEgPSBORFsxOmksIF0pDQp9DQoNCnBhcihtZnJvdyA9IGMoMiwgMiksIG9tYSA9IGMoMCwgMiwgMCwgMikpDQpmb3IgKGkgaW4gYygxLDMsNSw3KSkgew0KICBwbG90KHN1cnZQcmVkc1tbaV1dLCBlc3RpbWF0b3IgPSAibWVkaWFuIiwgY29uZi5pbnQgPSBUUlVFLA0KICAgICAgIGluY2x1ZGUueSA9IFRSVUUsIG1haW4gPSBwYXN0ZSgiRm9sbG93LXVwIHRpbWU6IiwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcm91bmQoc3VydlByZWRzW1tpXV0kbGFzdC50aW1lLCAxKSkpDQp9DQoNCnRvYygpDQoNCmBgYA0KDQoNCiMjIFJlc3VsdA0KIyMjIFBhdGllbnQgMTgwLWRheSByaXNrIHRhYmxlDQoNCmBgYHtyfQ0KYkdlbmVyYXRlUHJlZHMgPC0gMQ0KDQppZihiR2VuZXJhdGVQcmVkcyl7DQogIGlkc190b19wcmVkaWN0IDwtIHBiYzIuaWQgJT4lIGRwbHlyOjpmaWx0ZXIoc3RhdHVzMiA9PSAwKSAlPiUgZHBseXI6OnNlbGVjdChpZCkgJT4lIHB1bGwoKQ0KICBzdXJ2X3ByZWRzIDwtIGxpc3QoKQ0KICBmb3IoaSBpbiBpZHNfdG9fcHJlZGljdCl7DQogICAgdG1wX3ByZWQgPC1wYmMyICU+JSBkcGx5cjo6ZmlsdGVyKGlkID09IGkpIA0KICAgIGxhc3RfdGltZSA8LSB1bmlxdWUodG1wX3ByZWQkeWVhcnMpDQogICAgdG1wIDwtIEpNOjpzdXJ2Zml0Sk0oam1GaXQsIG5ld2RhdGEgPSB0bXBfcHJlZCwgbGFzdC50aW1lID0gbGFzdF90aW1lLCBpZFZhciA9ICJpZCIsIHN1cnZUaW1lcyA9IGxhc3RfdGltZSArIDE4MC8zNjUpDQogICAgcFN1cnZfMTgwZGF5IDwtIHVubGlzdCh0bXBbMV0kc3VtbWFyaWVzKVsyXQ0KICAgIHN1cnZfcHJlZHMgPC0gc3Vydl9wcmVkcyAlPiUgYmluZF9yb3dzKGRhdGEuZnJhbWUoImlkIiA9IGksICJwc3Vydl8xODBkYXkiID0gcFN1cnZfMTgwZGF5LCAieWVhcnMiID0gbGFzdF90aW1lKSkNCiAgfQ0KICANCiAgc3Vydl9wcmVkcyA8LSBzdXJ2X3ByZWRzICU+JSBkcGx5cjo6YXJyYW5nZShwU3Vydl8xODBkYXkpDQogIHVzX2NpdGllcyA8LSBtYXBzOjp1cy5jaXRpZXMgJT4lIGRwbHlyOjpzZWxlY3QobmFtZSwgbGF0LCBsb25nKQ0KICBzZXQuc2VlZCgxMjMpDQogIHVzX2NpdGllcyA8LSB1c19jaXRpZXNbc2FtcGxlKG5yb3coc3Vydl9wcmVkcykpLF0NCiAgc3Vydl9wcmVkcyA8LSBjYmluZChzdXJ2X3ByZWRzLCB1c19jaXRpZXMpIA0KICB3cml0ZS5jc3Yoc3Vydl9wcmVkcywgInN1cnZfcHJlZHMuY3N2Iiwgcm93Lm5hbWVzID0gRikNCn0NCmBgYA0KDQoNCmBgYHtyfQ0Kc3Vydl9wcmVkcyRwc3Vydl8xODBkYXkgPC0gcm91bmQoc3Vydl9wcmVkcyRwc3Vydl8xODBkYXksIGRpZ2l0cyA9IDQpDQpzdXJ2X3ByZWRzJHJpc2sgPC0gIk1lZGl1bSINCnN1cnZfcHJlZHMkcmlza1tzdXJ2X3ByZWRzJHBzdXJ2XzE4MGRheSA+IDAuOF0gPC0gIkxvdyINCnN1cnZfcHJlZHMkcmlza1tzdXJ2X3ByZWRzJHBzdXJ2XzE4MGRheSA8PSAwLjZdIDwtICJIaWdoIg0KDQpEVDo6ZGF0YXRhYmxlKHN1cnZfcHJlZHMsIG9wdGlvbnMgPSBsaXN0KHNjcm9sbFggPSBUUlVFKSwgY2xhc3MgPSAnY2VsbC1ib3JkZXIgc3RyaXBlJywgcm93bmFtZXMgPSBGQUxTRSkNCg0KYGBgDQoNCg0KDQojIyBQYXRpZW50IDE4MC1kYXkgUmlzayArIExvY2F0aW9ucyBJbmZvIG9uIGEgTWFwDQoNCmBgYHtyfQ0KDQpzdXJ2X3ByZWRzJGlkIDwtIHBhc3RlMCgiUGF0aWVudCAiLCBzdXJ2X3ByZWRzJGlkKQ0KDQoNCnN1cnZfcHJlZHNfc2YgPC0gc2Y6OnN0X2FzX3NmKA0KICB4ID0gc3Vydl9wcmVkcywNCiAgY29vcmRzID0gYygibG9uZyIsICJsYXQiKSwgICMgY29sdW1ucyB3aXRoIGNvb3JkaW5hdGVzDQogIGNycyA9ICdFU1JJOjEwMjAwMycJICMgY29vcmRpbmF0ZSByZWZlcmVuY2Ugc3lzdGVtIGNvZGUgZm9yIGVhc3RpbmdzL25vcnRoaW5ncw0KKSAlPiUgc2Y6OnN0X3RyYW5zZm9ybShjcnMgPSAnRVNSSToxMDIwMDMnKSAgIyB0aGUgY29vcmQgcmVmIHN5c3RlbSBjb2RlIGZvciBsYXRsb25nDQoNCnNhdmVSRFMoc3Vydl9wcmVkc19zZiwgInN1cnZfcHJlZHNfc2YuUkRTIikNCg0KDQpgYGANCg0KDQoNCmBgYHtyLCB3aWR0aCA9IDgsIGhlaWdodCA9IDgsIHdhcm5pbmc9RkFMU0V9DQpzY2ggPC0gc3Vydl9wcmVkc19zZg0KbWFwIDwtIHNjaCAlPiUgDQogIGxlYWZsZXQ6OmxlYWZsZXQoKSAlPiUNCiAgbGVhZmxldDo6YWRkUHJvdmlkZXJUaWxlcyhwcm92aWRlcnMkT3BlblN0cmVldE1hcCkgJT4lIA0KICBsZWFmbGV0OjphZGRBd2Vzb21lTWFya2VycygNCiAgICBwb3B1cCA9IH5wYXN0ZTAoDQogICAgICAiPGgxPiIsIHNjaCRpZCwgIjwvaDE+IiwNCiAgICAgIA0KICAgICAgIjx0YWJsZSBzdHlsZT0nd2lkdGg6MTAwJSc+IiwNCiAgICAgIA0KICAgICAgIjx0cj4iLA0KICAgICAgIjx0aD5SSVNLPC90aD4iLA0KICAgICAgIjx0aD4iLCBzY2gkcmlzaywgIjwvdGg+IiwNCiAgICAgICI8L3RyPiIsDQogICAgICANCiAgICAgICI8dHI+IiwNCiAgICAgICI8dHI+IiwNCiAgICAgICI8dGg+cFN1cnZpdmFsIDwvdGg+IiwNCiAgICAgICI8dGg+Iiwgc2NoJHBzdXJ2XzE4MGRheSwgIjwvdGg+IiwNCiAgICAgICI8L3RyPiIsDQogICAgICANCiAgICAgICI8dHI+IiwNCiAgICAgICI8dHI+IiwNCiAgICAgICI8dGg+Y2l0eTwvdGg+IiwNCiAgICAgICI8dGg+Iiwgc2NoJG5hbWUsICI8L3RoPiIsDQogICAgICAiPC90cj4iDQogICAgKSwgICMgZW5kIHBvcHVwKCkNCiAgICBpY29uID0gYXdlc29tZUljb25zKA0KICAgICAgbGlicmFyeSA9ICJpb24iLA0KICAgICAgaWNvbiA9IGlmZWxzZSgNCiAgICAgICAgdGVzdCA9IHNjaCRyaXNrID09ICJIaWdoIiwgDQogICAgICAgIHllcyA9ICJpb24tYW5kcm9pZC1zdGFyLW91dGxpbmUiLA0KICAgICAgICBubyA9ICJpb24tYW5kcm9pZC1zdGFyLW91dGxpbmUiDQogICAgICApLA0KICAgICAgaWNvbkNvbG9yID0gIndoaXRlIiwNCiAgICAgIG1hcmtlckNvbG9yID0gaWZlbHNlKA0KICAgICAgICB0ZXN0ID0gc2NoJHJpc2sgPT0gIkhpZ2giLCANCiAgICAgICAgeWVzID0gInJlZCIsDQogICAgICAgIG5vID0gaWZlbHNlKHNjaCRyaXNrID09ICJNZWRpdW0iLCB5ZXMgPSAib3JhbmdlIiwgbm8gPSAiZ3JlZW4iKQ0KICAgICAgKQ0KICAgICkNCiAgKSAlPiUgICAjIGVuZCBhZGRBd2Vzb21lTWFya2VycygpDQogIGxlYWZsZXQ6OmFkZE1lYXN1cmUoKQ0KDQoNCg0KYGBgDQoNCg0KDQpgYGB7ciwgZmlnLndpZHRoPTksIGZpZy5hbGlnbj0nY2VudGVyJywgd2FybmluZz1GQUxTRX0NCm1hcA0KDQpgYGANCg0KDQoNCg0KIyMgUGF0aWVudCAxODAtZGF5IFJpc2sgKyBBbWJ1bGFuY2UgTG9jYXRpb25zIG9uIGEgTWFwDQoNCi0gKipSZS1kaXN0cmlidXRlIFJlc291cmNlcyAoQW1idWxhbmNlICsgTGFib3IpIHByb2FjdGl2ZWx5KioNCg0KYGBge3IsIGZpZy53aWR0aD05LCBmaWcuYWxpZ249J2NlbnRlcicsIHdhcm5pbmc9RkFMU0V9DQp1c19jaXRpZXMgPC0gbWFwczo6dXMuY2l0aWVzICU+JSBkcGx5cjo6c2VsZWN0KG5hbWUsIGxhdCwgbG9uZykgJT4lIA0KICByZW5hbWUobG5nID0gbG9uZykNCg0Kc2V0LnNlZWQoMTI1KQ0KYW1idWxhbmNlX2xvY2F0aW9ucyA8LSB1c19jaXRpZXNbc2FtcGxlKDEwKSxdDQphbWJ1bGFuY2VfbG9jYXRpb25zJGlkIDwtIDE6bnJvdyhhbWJ1bGFuY2VfbG9jYXRpb25zKQ0KYW1idWxhbmNlX2xvY2F0aW9ucyRpZCA8LSBwYXN0ZTAoIkEiLCBhbWJ1bGFuY2VfbG9jYXRpb25zJGlkKQ0KDQptYXAgJT4lDQogIGFkZFB1bHNlTWFya2VycygNCiAgICBsbmcgPSBhbWJ1bGFuY2VfbG9jYXRpb25zJGxuZywNCiAgICBsYXQgPSBhbWJ1bGFuY2VfbG9jYXRpb25zJGxhdCwNCiAgICBwb3B1cCA9IH5wYXN0ZTAoDQogICAgICAiPGgxPiIsIGFtYnVsYW5jZV9sb2NhdGlvbnMkaWQsICI8L2gxPiIpLA0KICAgIA0KICAgIGljb24gPSBtYWtlUHVsc2VJY29uKCkNCiAgKQ0KDQpgYGANCg==