Packages

# library(tidyverse)
# install.packages("mada")
# install.packages("meta")
pacman::p_load(mada, meta, tidyverse, rapportools)
# library(mada)
# library(meta)

Dataset

Filtro solamente pra OPG y que tengan para calcular sens y espec

df <- df %>% 
  filter(test == "OPG", vp != "NA")
glimpse(df)
Observations: 21
Variables: 25
$ autor                  <chr> "H. GHAEMINIA", "H. GHAEMINIA", "H. GHAEMINIA", "H. GHAEMINIA", "H. GHAEMINIA", "H.…
$ rs_seleccion_paciente  <dbl> -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
$ rs_test_indice         <dbl> -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
$ rs_referencia_estandar <dbl> -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
$ a_flujo_y_tiempo       <dbl> -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
$ a_seleccion_pacientes  <dbl> -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
$ a_test_indice          <dbl> -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
$ a_referencia_estandar  <dbl> -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
$ muestra_opg            <dbl> 53, 53, 53, 53, 53, 53, 161, 161, 161, 161, 60, 60, 60, 60, 60, 31, 31, 31, 31, 31,…
$ signo_identificado     <chr> "Interrupción linea blanca", "Oscurecimiento raiz", "Desvio canal alveolar inferior…
$ signo_identificado_2   <dbl> NA, NA, NA, NA, NA, NA, 62, 30, 15, 5, 1, NA, 28, 3, 8, NA, NA, NA, NA, NA, NA
$ muestra_cbct           <dbl> 53, 53, 53, 53, 53, 53, 161, 161, 161, 161, 41, 41, 41, 41, 41, 31, 31, 31, 31, 31,…
$ contacto_cbct          <chr> "45", "45", "45", "45", "45", "45", "27", "27", "27", "27", "41", "41", "41", "41",…
$ contacto_real_iac      <dbl> 23, 23, 23, 23, 23, 23, 25, 25, 25, 25, 41, 41, 41, 41, 41, 3, 3, 3, 3, 3, 3
$ ppv                    <dbl> 0.44, 0.44, 0.44, 0.44, 0.44, 0.00, 0.31, 0.31, 0.31, 0.31, NA, NA, NA, NA, NA, NA,…
$ npv                    <dbl> 1.00, 1.00, 1.00, 1.00, 1.00, 0.00, 0.98, 0.98, 0.98, 0.98, NA, NA, NA, NA, NA, NA,…
$ vp                     <dbl> 23, 23, 23, 23, 23, 0, 19, 19, 19, 19, 40, 40, 40, 40, 40, 9, 9, 9, 9, 9, 9
$ fp                     <dbl> 29, 29, 29, 29, 29, 0, 43, 43, 43, 43, 19, 19, 19, 19, 19, 10, 10, 10, 10, 10, 10
$ vn                     <dbl> 1, 1, 1, 1, 1, 0, 72, 72, 72, 72, 0, 0, 0, 0, 0, 12, 12, 12, 12, 12, 12
$ fn                     <dbl> 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0
$ accuracy               <dbl> 0.45, 0.45, 0.45, 0.45, 0.45, 0.00, 0.64, 0.64, 0.64, 0.64, NA, NA, NA, NA, NA, NA,…
$ test                   <chr> "OPG", "OPG", "OPG", "OPG", "OPG", "OPG", "OPG", "OPG", "OPG", "OPG", "OPG", "OPG",…
$ gold_standard          <chr> "Evaluación clínica", "Evaluación clínica", "Evaluación clínica", "Evaluación clíni…
$ sensibilidad_opg       <dbl> 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 0.70, 0.70, 0.70, 0.70, 0.97, 0.97, 0.97, 0.97,…
$ especificidad          <dbl> 0.03, 0.03, 0.03, 0.03, 0.03, 0.03, 0.77, 0.77, 0.77, 0.77, 0.00, 0.00, 0.00, 0.00,…
df$signo_identificado <- rapportools::tocamel(df$signo_identificado) 
df <- df %>% 
  mutate(signo_identificado = case_when(
    signo_identificado == "InterrupciónDeLaCorticalCanalMandibular" ~ "InterrupciónDeCorticalCanalMandibular",
    signo_identificado == "OscurecimientoDeLaRaiz" ~ "OscurecimientoDeLasRaices", 
    TRUE ~ as.character(signo_identificado)
  ))
table(df$signo_identificado)

        DesvioCanalAlveolarInferior EstrechamientoCanalAlveolarInferior                  EstrechamientoRaiz 
                                  4                                   2                                   3 
            InterrupciónLineaBlanca                  OscurecimientoRaiz                     RaicesDesviadas 
                                  4                                   4                                   2 
                  RaizOscuraYBifida 
                                  2 
aNumero <- function(x, na.rm = FALSE) (as.integer(x))
df <- df %>% 
  mutate_at(c("a_seleccion_pacientes", 
              "a_referencia_estandar", 
              "especificidad"), aNumero)

EDA

table(df$autor)

     A. HASANI        A.JHAMB   H. GHAEMINIA TANTANAPORNKUL 
             5              6              6              4 

Calidad

df %>% 
  mutate(riesgoDeSesgo = rs_seleccion_paciente + rs_test_indice + rs_referencia_estandar) %>% 
  mutate(aplicabilidad = a_flujo_y_tiempo + a_seleccion_pacientes + a_test_indice + a_referencia_estandar) %>% 
  group_by(autor) %>% 
  summarise(sesgo = mean(riesgoDeSesgo), aplicab = mean(aplicabilidad))

Meta-analysis

table(df$signo_identificado)

        DesvioCanalAlveolarInferior EstrechamientoCanalAlveolarInferior                  EstrechamientoRaiz 
                                  4                                   2                                   3 
            InterrupciónLineaBlanca                  OscurecimientoRaiz                     RaicesDesviadas 
                                  4                                   4                                   2 
                  RaizOscuraYBifida 
                                  2 

Creo los totales

df <- df %>% 
  mutate(n1 = vp + fp, 
         n2 = vn + fn)
table(df$signo_identificado)

        DesvioCanalAlveolarInferior EstrechamientoCanalAlveolarInferior                  EstrechamientoRaiz 
                                  4                                   2                                   3 
            InterrupciónLineaBlanca                  OscurecimientoRaiz                     RaicesDesviadas 
                                  4                                   4                                   2 
                  RaizOscuraYBifida 
                                  2 

Desviación raíces / clinica

md1 <- df %>% 
  filter(signo_identificado == "DesvioCanalAlveolarInferior") %>% 
  select(autor, 
         TP = vp, 
         FP = fp, 
         TN = vn, 
         FN = fn) %>% 
  na.omit() %>% 
  madad(., level = 0.95)
Unknown or uninitialised column: 'names'.There are very few primary studies!
md1  
Descriptive summary of . with 4 primary studies.
Confidence level for all calculations set to 95 %
Using a continuity correction of 0.5 if applicable 

Diagnostic accuracies 
      sens  2.5% 97.5%  spec  2.5% 97.5%
[1,] 0.979 0.828 0.998 0.048 0.011 0.185
[2,] 0.696 0.511 0.834 0.625 0.534 0.708
[3,] 0.964 0.859 0.992 0.025 0.003 0.200
[4,] 0.950 0.655 0.995 0.543 0.349 0.726

Test for equality of sensitivities: 
X-squared = 15.9214, df = 3, p-value = 0.00118
Test for equality of specificities: 
X-squared = 49.7698, df = 3, p-value = 8.94e-11


Diagnostic OR and likelihood ratios 
        DOR  2.5%   97.5% posLR  2.5% 97.5% negLR  2.5%  97.5%
[1,]  2.390 0.093  61.399 1.029 0.932 1.135 0.431 0.018 10.107
[2,]  3.824 1.572   9.301 1.857 1.323 2.607 0.486 0.272  0.866
[3,]  0.692 0.027  17.781 0.989 0.903 1.083 1.429 0.061 33.538
[4,] 22.619 1.172 436.417 2.081 1.303 3.323 0.092 0.006  1.407

Correlation of sensitivities and false positive rates: 
   rho  2.5 % 97.5 % 
 0.713 -0.789  0.993 
mada::forest(md1, type = "sens", main = "Sensibilidad DesvioCanalAlveolarInferior", xlab = "Sensibilidad")

mada::forest(md1, type = "spec", main = "Especificidad DesvioCanalAlveolarInferior")

mada::forest(md1, type = "DOR", log = TRUE,  main = "Log DOR DesvioCanalAlveolarInferior")

EstrechamientoCanalAlveolarInferior

md2 <- df %>%
  select(autor,
         signo_identificado, 
         gold_standard, 
         test, 
         TP = vp, 
         FP = fp, 
         TN = vn, 
         FN = fn) %>%
  filter(TP > 0) %>% 
  filter(test == "OPG") %>% 
  filter(signo_identificado == "EstrechamientoCanalAlveolarInferior") 
md2  <- mada::madad(md2, level = 0.95)
Unknown or uninitialised column: 'names'.There are very few primary studies!
mada::forest(md2, type = "sens", main = "Sensibilidad", xlab = "Sensibilidad EstrechamientoCanalAlveolarInferior")

mada::forest(md2, type = "spec", main = "Especificidad EstrechamientoCanalAlveolarInferior")

mada::forest(md2, type = "DOR", log = TRUE,  main = "Log DOR EstrechamientoCanalAlveolarInferior")

EstrechamientoRaiz

md4 <- df %>%
  select(autor,
         signo_identificado, 
         gold_standard, 
         test, 
         TP = vp, 
         FP = fp, 
         TN = vn, 
         FN = fn) %>%
  filter(TP > 0) %>% 
  filter(test == "OPG") %>% 
  filter(signo_identificado == "EstrechamientoRaiz") %>%
  madad(., level = 0.95)
Unknown or uninitialised column: 'names'.There are very few primary studies!
md4  
Descriptive summary of . with 2 primary studies.
Confidence level for all calculations set to 95 %
Using a continuity correction of 0.5 if applicable 

Diagnostic accuracies 
      sens  2.5% 97.5%  spec  2.5% 97.5%
[1,] 0.696 0.511 0.834 0.625 0.534 0.708
[2,] 0.950 0.655 0.995 0.543 0.349 0.726

Test for equality of sensitivities: 
X-squared = 1.406, df = 1, p-value = 0.236
Test for equality of specificities: 
X-squared = 0.2486, df = 1, p-value = 0.618


Diagnostic OR and likelihood ratios 
        DOR  2.5%   97.5% posLR  2.5% 97.5% negLR  2.5% 97.5%
[1,]  3.824 1.572   9.301 1.857 1.323 2.607 0.486 0.272 0.866
[2,] 22.619 1.172 436.417 2.081 1.303 3.323 0.092 0.006 1.407

Correlation of sensitivities and false positive rates: 
Error in CIrho(x$cor_sens_fpr, x$nobs) : rho < 1 is not TRUE

InterrupciónLineaBlanca

md5  
Descriptive summary of . with 4 primary studies.
Confidence level for all calculations set to 95 %
Using a continuity correction of 0.5 if applicable 

Diagnostic accuracies 
      sens  2.5% 97.5%  spec  2.5% 97.5%
[1,] 0.979 0.828 0.998 0.048 0.011 0.185
[2,] 0.696 0.511 0.834 0.625 0.534 0.708
[3,] 0.964 0.859 0.992 0.025 0.003 0.200
[4,] 0.950 0.655 0.995 0.543 0.349 0.726

Test for equality of sensitivities: 
X-squared = 15.9214, df = 3, p-value = 0.00118
Test for equality of specificities: 
X-squared = 49.7698, df = 3, p-value = 8.94e-11


Diagnostic OR and likelihood ratios 
        DOR  2.5%   97.5% posLR  2.5% 97.5% negLR  2.5%  97.5%
[1,]  2.390 0.093  61.399 1.029 0.932 1.135 0.431 0.018 10.107
[2,]  3.824 1.572   9.301 1.857 1.323 2.607 0.486 0.272  0.866
[3,]  0.692 0.027  17.781 0.989 0.903 1.083 1.429 0.061 33.538
[4,] 22.619 1.172 436.417 2.081 1.303 3.323 0.092 0.006  1.407

Correlation of sensitivities and false positive rates: 
   rho  2.5 % 97.5 % 
 0.713 -0.789  0.993 

OscurecimientoRaiz

md6  
Descriptive summary of . with 4 primary studies.
Confidence level for all calculations set to 95 %
Using a continuity correction of 0.5 if applicable 

Diagnostic accuracies 
      sens  2.5% 97.5%  spec  2.5% 97.5%
[1,] 0.979 0.828 0.998 0.048 0.011 0.185
[2,] 0.696 0.511 0.834 0.625 0.534 0.708
[3,] 0.964 0.859 0.992 0.025 0.003 0.200
[4,] 0.950 0.655 0.995 0.543 0.349 0.726

Test for equality of sensitivities: 
X-squared = 15.9214, df = 3, p-value = 0.00118
Test for equality of specificities: 
X-squared = 49.7698, df = 3, p-value = 8.94e-11


Diagnostic OR and likelihood ratios 
        DOR  2.5%   97.5% posLR  2.5% 97.5% negLR  2.5%  97.5%
[1,]  2.390 0.093  61.399 1.029 0.932 1.135 0.431 0.018 10.107
[2,]  3.824 1.572   9.301 1.857 1.323 2.607 0.486 0.272  0.866
[3,]  0.692 0.027  17.781 0.989 0.903 1.083 1.429 0.061 33.538
[4,] 22.619 1.172 436.417 2.081 1.303 3.323 0.092 0.006  1.407

Correlation of sensitivities and false positive rates: 
   rho  2.5 % 97.5 % 
 0.713 -0.789  0.993 

RaicesDesviadas

md7  
Descriptive summary of . with 2 primary studies.
Confidence level for all calculations set to 95 %
Using a continuity correction of 0.5 if applicable 

Diagnostic accuracies 
      sens  2.5% 97.5%  spec  2.5% 97.5%
[1,] 0.979 0.828 0.998 0.048 0.011 0.185
[2,] 0.950 0.655 0.995 0.543 0.349 0.726

Test for equality of sensitivities: 
X-squared = 0, df = 1, p-value = 1
Test for equality of specificities: 
X-squared = 14.3731, df = 1, p-value = 0.00015


Diagnostic OR and likelihood ratios 
        DOR  2.5%   97.5% posLR  2.5% 97.5% negLR  2.5%  97.5%
[1,]  2.390 0.093  61.399 1.029 0.932 1.135 0.431 0.018 10.107
[2,] 22.619 1.172 436.417 2.081 1.303 3.323 0.092 0.006  1.407

Correlation of sensitivities and false positive rates: 
Error in CIrho(x$cor_sens_fpr, x$nobs) : rho < 1 is not TRUE

RaizOscuraYBifida

md8  
Descriptive summary of . with 2 primary studies.
Confidence level for all calculations set to 95 %
Using a continuity correction of 0.5 if applicable 

Diagnostic accuracies 
      sens  2.5% 97.5%  spec  2.5% 97.5%
[1,] 0.964 0.859 0.992 0.025 0.003 0.200
[2,] 0.950 0.655 0.995 0.543 0.349 0.726

Test for equality of sensitivities: 
X-squared = 0, df = 1, p-value = 1
Test for equality of specificities: 
X-squared = 11.2867, df = 1, p-value = 0.000781


Diagnostic OR and likelihood ratios 
        DOR  2.5%   97.5% posLR  2.5% 97.5% negLR  2.5%  97.5%
[1,]  0.692 0.027  17.781 0.989 0.903 1.083 1.429 0.061 33.538
[2,] 22.619 1.172 436.417 2.081 1.303 3.323 0.092 0.006  1.407

Correlation of sensitivities and false positive rates: 
Error in CIrho(x$cor_sens_fpr, x$nobs) : rho < 1 is not TRUE

LS0tCnRpdGxlOiAiMjAxOSBTeXN0ZW1hdGljIHJldmlldyBQYW5vcmFtaWMgQWx2ZW9sYXIgTmVydmUgRGFtYWdlIgpvdXRwdXQ6IAogIGh0bWxfbm90ZWJvb2s6IAogICAgdG9jOiB5ZXMKICAgIHRvY19mbG9hdDogdHJ1ZQogICAgZmlnX2NhcHRpb246IHRydWUKLS0tCiMgUGFja2FnZXMKYGBge3J9CiMgbGlicmFyeSh0aWR5dmVyc2UpCiMgaW5zdGFsbC5wYWNrYWdlcygibWFkYSIpCiMgaW5zdGFsbC5wYWNrYWdlcygibWV0YSIpCnBhY21hbjo6cF9sb2FkKG1hZGEsIG1ldGEsIHRpZHl2ZXJzZSwgcmFwcG9ydG9vbHMpCiMgbGlicmFyeShtYWRhKQojIGxpYnJhcnkobWV0YSkKYGBgCgojIERhdGFzZXQKYGBge3IsIGluY2x1ZGU9RkFMU0V9CgpkZiA8LSByZWFkX2NzdigiaHR0cHM6Ly9kb2NzLmdvb2dsZS5jb20vc3ByZWFkc2hlZXRzL2QvZS8yUEFDWC0xdlFSd3B3UnUxcUhMREh3aFFSZVBwQlhWTzVqTVREM0JZRDZOa1gxdDlJeGlGMUE0aGNpRXdVaUd4dDBoeTZLQy1rbTQ1Y0o1VzlFYWhZOC9wdWI/Z2lkPTU2NzkyODMzNyZzaW5nbGU9dHJ1ZSZvdXRwdXQ9Y3N2IikKCgpkZiA8LSBqYW5pdG9yOjpjbGVhbl9uYW1lcyhkZikKYGBgCgoKRmlsdHJvIHNvbGFtZW50ZSBwcmEgT1BHIHkgcXVlIHRlbmdhbiBwYXJhIGNhbGN1bGFyIHNlbnMgeSBlc3BlYwoKYGBge3J9CmRmIDwtIGRmICU+JSAKICBmaWx0ZXIodGVzdCA9PSAiT1BHIiwgdnAgIT0gIk5BIikKYGBgCgpgYGB7cn0KZ2xpbXBzZShkZikKYGBgCgoKYGBge3J9CmRmJHNpZ25vX2lkZW50aWZpY2FkbyA8LSByYXBwb3J0b29sczo6dG9jYW1lbChkZiRzaWdub19pZGVudGlmaWNhZG8pIApkZiA8LSBkZiAlPiUgCiAgbXV0YXRlKHNpZ25vX2lkZW50aWZpY2FkbyA9IGNhc2Vfd2hlbigKICAgIHNpZ25vX2lkZW50aWZpY2FkbyA9PSAiSW50ZXJydXBjacOzbkRlTGFDb3J0aWNhbENhbmFsTWFuZGlidWxhciIgfiAiSW50ZXJydXBjacOzbkRlQ29ydGljYWxDYW5hbE1hbmRpYnVsYXIiLAogICAgc2lnbm9faWRlbnRpZmljYWRvID09ICJPc2N1cmVjaW1pZW50b0RlTGFSYWl6IiB+ICJPc2N1cmVjaW1pZW50b0RlTGFzUmFpY2VzIiwgCiAgICBUUlVFIH4gYXMuY2hhcmFjdGVyKHNpZ25vX2lkZW50aWZpY2FkbykKICApKQpgYGAKCmBgYHtyfQp0YWJsZShkZiRzaWdub19pZGVudGlmaWNhZG8pCmBgYAoKCmBgYHtyfQphTnVtZXJvIDwtIGZ1bmN0aW9uKHgsIG5hLnJtID0gRkFMU0UpIChhcy5pbnRlZ2VyKHgpKQoKCmRmIDwtIGRmICU+JSAKICBtdXRhdGVfYXQoYygiYV9zZWxlY2Npb25fcGFjaWVudGVzIiwgCiAgICAgICAgICAgICAgImFfcmVmZXJlbmNpYV9lc3RhbmRhciIsIAogICAgICAgICAgICAgICJlc3BlY2lmaWNpZGFkIiksIGFOdW1lcm8pCmBgYAoKCiMgRURBCgpgYGB7cn0KdGFibGUoZGYkYXV0b3IpCmBgYAoKIyMgQ2FsaWRhZAoKYGBge3J9CmRmICU+JSAKICBtdXRhdGUocmllc2dvRGVTZXNnbyA9IHJzX3NlbGVjY2lvbl9wYWNpZW50ZSArIHJzX3Rlc3RfaW5kaWNlICsgcnNfcmVmZXJlbmNpYV9lc3RhbmRhcikgJT4lIAogIG11dGF0ZShhcGxpY2FiaWxpZGFkID0gYV9mbHVqb195X3RpZW1wbyArIGFfc2VsZWNjaW9uX3BhY2llbnRlcyArIGFfdGVzdF9pbmRpY2UgKyBhX3JlZmVyZW5jaWFfZXN0YW5kYXIpICU+JSAKICBncm91cF9ieShhdXRvcikgJT4lIAogIHN1bW1hcmlzZShzZXNnbyA9IG1lYW4ocmllc2dvRGVTZXNnbyksIGFwbGljYWIgPSBtZWFuKGFwbGljYWJpbGlkYWQpKQpgYGAKCgojIE1ldGEtYW5hbHlzaXMKCmBgYHtyfQp0YWJsZShkZiRzaWdub19pZGVudGlmaWNhZG8pCmBgYAoKCkNyZW8gbG9zIHRvdGFsZXMKYGBge3J9CmRmIDwtIGRmICU+JSAKICBtdXRhdGUobjEgPSB2cCArIGZwLCAKICAgICAgICAgbjIgPSB2biArIGZuKQpgYGAKCgoKCmBgYHtyfQp0YWJsZShkZiRzaWdub19pZGVudGlmaWNhZG8pCmBgYAoKIyMgRGVzdmlhY2nDs24gcmHDrWNlcyAvIGNsaW5pY2EKYGBge3J9Cm1kMSA8LSBkZiAlPiUgCiAgZmlsdGVyKHNpZ25vX2lkZW50aWZpY2FkbyA9PSAiRGVzdmlvQ2FuYWxBbHZlb2xhckluZmVyaW9yIikgJT4lIAogIHNlbGVjdChhdXRvciwgCiAgICAgICAgIFRQID0gdnAsIAogICAgICAgICBGUCA9IGZwLCAKICAgICAgICAgVE4gPSB2biwgCiAgICAgICAgIEZOID0gZm4pICU+JSAKICBuYS5vbWl0KCkgJT4lIAogIG1hZGFkKC4sIGxldmVsID0gMC45NSkKbWQxICAKCgpgYGAKYGBge3J9CgptYWRhOjpmb3Jlc3QobWQxLCB0eXBlID0gInNlbnMiLCBtYWluID0gIlNlbnNpYmlsaWRhZCBEZXN2aW9DYW5hbEFsdmVvbGFySW5mZXJpb3IiLCB4bGFiID0gIlNlbnNpYmlsaWRhZCIpCmBgYApgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kMSwgdHlwZSA9ICJzcGVjIiwgbWFpbiA9ICJFc3BlY2lmaWNpZGFkIERlc3Zpb0NhbmFsQWx2ZW9sYXJJbmZlcmlvciIpCmBgYApgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kMSwgdHlwZSA9ICJET1IiLCBsb2cgPSBUUlVFLCAgbWFpbiA9ICJMb2cgRE9SIERlc3Zpb0NhbmFsQWx2ZW9sYXJJbmZlcmlvciIpCmBgYAogIAojIyBFc3RyZWNoYW1pZW50b0NhbmFsQWx2ZW9sYXJJbmZlcmlvcgoKYGBge3J9Cm1kMiA8LSBkZiAlPiUKICBzZWxlY3QoYXV0b3IsCiAgICAgICAgIHNpZ25vX2lkZW50aWZpY2FkbywgCiAgICAgICAgIGdvbGRfc3RhbmRhcmQsIAogICAgICAgICB0ZXN0LCAKICAgICAgICAgVFAgPSB2cCwgCiAgICAgICAgIEZQID0gZnAsIAogICAgICAgICBUTiA9IHZuLCAKICAgICAgICAgRk4gPSBmbikgJT4lCiAgZmlsdGVyKFRQID4gMCkgJT4lIAogIGZpbHRlcih0ZXN0ID09ICJPUEciKSAlPiUgCiAgZmlsdGVyKHNpZ25vX2lkZW50aWZpY2FkbyA9PSAiRXN0cmVjaGFtaWVudG9DYW5hbEFsdmVvbGFySW5mZXJpb3IiKSAKbWQyICA8LSBtYWRhOjptYWRhZChtZDIsIGxldmVsID0gMC45NSkKCgpgYGAKYGBge3J9Cm1hZGE6OmZvcmVzdChtZDIsIHR5cGUgPSAic2VucyIsIG1haW4gPSAiU2Vuc2liaWxpZGFkIiwgeGxhYiA9ICJTZW5zaWJpbGlkYWQgRXN0cmVjaGFtaWVudG9DYW5hbEFsdmVvbGFySW5mZXJpb3IiKQpgYGAKYGBge3J9Cm1hZGE6OmZvcmVzdChtZDIsIHR5cGUgPSAic3BlYyIsIG1haW4gPSAiRXNwZWNpZmljaWRhZCBFc3RyZWNoYW1pZW50b0NhbmFsQWx2ZW9sYXJJbmZlcmlvciIpCmBgYApgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kMiwgdHlwZSA9ICJET1IiLCBsb2cgPSBUUlVFLCAgbWFpbiA9ICJMb2cgRE9SIEVzdHJlY2hhbWllbnRvQ2FuYWxBbHZlb2xhckluZmVyaW9yIikKYGBgCiAKCgojIyBFc3RyZWNoYW1pZW50b1JhaXogIAoKCmBgYHtyfQptZDQgPC0gZGYgJT4lCiAgc2VsZWN0KGF1dG9yLAogICAgICAgICBzaWdub19pZGVudGlmaWNhZG8sIAogICAgICAgICBnb2xkX3N0YW5kYXJkLCAKICAgICAgICAgdGVzdCwgCiAgICAgICAgIFRQID0gdnAsIAogICAgICAgICBGUCA9IGZwLCAKICAgICAgICAgVE4gPSB2biwgCiAgICAgICAgIEZOID0gZm4pICU+JQogIGZpbHRlcihUUCA+IDApICU+JSAKICBmaWx0ZXIodGVzdCA9PSAiT1BHIikgJT4lIAogIGZpbHRlcihzaWdub19pZGVudGlmaWNhZG8gPT0gIkVzdHJlY2hhbWllbnRvUmFpeiIpICU+JQogIG1hZGFkKC4sIGxldmVsID0gMC45NSkKbWQ0ICAKYGBgCgpgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kNCwgdHlwZSA9ICJzZW5zIiwgbWFpbiA9ICJTZW5zaWJpbGlkYWQiLCB4bGFiID0gIlNlbnNpYmlsaWRhZCBFc3RyZWNoYW1pZW50b1JhaXoiKQpgYGAKYGBge3J9Cm1hZGE6OmZvcmVzdChtZDQsIHR5cGUgPSAic3BlYyIsIG1haW4gPSAiRXNwZWNpZmljaWRhZCBFc3RyZWNoYW1pZW50b1JhaXoiKQpgYGAKYGBge3J9Cm1hZGE6OmZvcmVzdChtZDQsIHR5cGUgPSAiRE9SIiwgbG9nID0gVFJVRSwgIG1haW4gPSAiTG9nIERPUiBFc3RyZWNoYW1pZW50b1JhaXoiKQpgYGAKICAKIyMgIEludGVycnVwY2nDs25MaW5lYUJsYW5jYQoKYGBge3J9Cm1kNSA8LSBkZiAlPiUKICBzZWxlY3QoYXV0b3IsCiAgICAgICAgIHNpZ25vX2lkZW50aWZpY2FkbywgCiAgICAgICAgIGdvbGRfc3RhbmRhcmQsIAogICAgICAgICB0ZXN0LCAKICAgICAgICAgVFAgPSB2cCwgCiAgICAgICAgIEZQID0gZnAsIAogICAgICAgICBUTiA9IHZuLCAKICAgICAgICAgRk4gPSBmbikgJT4lCiAgZmlsdGVyKFRQID4gMCkgJT4lIAogIGZpbHRlcih0ZXN0ID09ICJPUEciKSAlPiUgCiAgZmlsdGVyKHNpZ25vX2lkZW50aWZpY2FkbyA9PSAiSW50ZXJydXBjacOzbkxpbmVhQmxhbmNhIikgJT4lCiAgbWFkYWQoLiwgbGV2ZWwgPSAwLjk1KQptZDUgIAoKCmBgYApgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kNSwgdHlwZSA9ICJzZW5zIiwgbWFpbiA9ICJTZW5zaWJpbGlkYWQgSW50ZXJydXBjacOzbkxpbmVhQmxhbmNhIiwgeGxhYiA9ICJTZW5zaWJpbGlkYWQgIikKYGBgCmBgYHtyfQptYWRhOjpmb3Jlc3QobWQ1LCB0eXBlID0gInNwZWMiLCBtYWluID0gIkVzcGVjaWZpY2lkYWQgSW50ZXJydXBjacOzbkxpbmVhQmxhbmNhIikKYGBgCmBgYHtyfQptYWRhOjpmb3Jlc3QobWQ1LCB0eXBlID0gIkRPUiIsIGxvZyA9IFRSVUUsICBtYWluID0gIkxvZyBET1IgSW50ZXJydXBjacOzbkxpbmVhQmxhbmNhIikKYGBgCgoKIyMgIE9zY3VyZWNpbWllbnRvUmFpegoKYGBge3J9Cm1kNiA8LSBkZiAlPiUKICBzZWxlY3QoYXV0b3IsCiAgICAgICAgIHNpZ25vX2lkZW50aWZpY2FkbywgCiAgICAgICAgIGdvbGRfc3RhbmRhcmQsIAogICAgICAgICB0ZXN0LCAKICAgICAgICAgVFAgPSB2cCwgCiAgICAgICAgIEZQID0gZnAsIAogICAgICAgICBUTiA9IHZuLCAKICAgICAgICAgRk4gPSBmbikgJT4lCiAgZmlsdGVyKFRQID4gMCkgJT4lIAogIGZpbHRlcih0ZXN0ID09ICJPUEciKSAlPiUgCiAgZmlsdGVyKHNpZ25vX2lkZW50aWZpY2FkbyA9PSAiT3NjdXJlY2ltaWVudG9SYWl6IikgJT4lCiAgbWFkYWQoLiwgbGV2ZWwgPSAwLjk1KQptZDYgIAoKCmBgYApgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kNiwgdHlwZSA9ICJzZW5zIiwgbWFpbiA9ICJTZW5zaWJpbGlkYWQgT3NjdXJlY2ltaWVudG9SYWl6IiwgeGxhYiA9ICJTZW5zaWJpbGlkYWQiKQpgYGAKYGBge3J9Cm1hZGE6OmZvcmVzdChtZDYsIHR5cGUgPSAic3BlYyIsIG1haW4gPSAiRXNwZWNpZmljaWRhZCBPc2N1cmVjaW1pZW50b1JhaXoiKQpgYGAKYGBge3J9Cm1hZGE6OmZvcmVzdChtZDYsIHR5cGUgPSAiRE9SIiwgbG9nID0gVFJVRSwgIG1haW4gPSAiTG9nIERPUiBPc2N1cmVjaW1pZW50b1JhaXoiKQpgYGAKCgojIyAgUmFpY2VzRGVzdmlhZGFzCgpgYGB7cn0KbWQ3IDwtIGRmICU+JQogIHNlbGVjdChhdXRvciwKICAgICAgICAgc2lnbm9faWRlbnRpZmljYWRvLCAKICAgICAgICAgZ29sZF9zdGFuZGFyZCwgCiAgICAgICAgIHRlc3QsIAogICAgICAgICBUUCA9IHZwLCAKICAgICAgICAgRlAgPSBmcCwgCiAgICAgICAgIFROID0gdm4sIAogICAgICAgICBGTiA9IGZuKSAlPiUKICBmaWx0ZXIoVFAgPiAwKSAlPiUgCiAgZmlsdGVyKHRlc3QgPT0gIk9QRyIpICU+JSAKICBmaWx0ZXIoc2lnbm9faWRlbnRpZmljYWRvID09ICJSYWljZXNEZXN2aWFkYXMiKSAlPiUKICBtYWRhZCguLCBsZXZlbCA9IDAuOTUpCm1kNyAgCgoKYGBgCmBgYHtyfQptYWRhOjpmb3Jlc3QobWQ3LCB0eXBlID0gInNlbnMiLCBtYWluID0gIlNlbnNpYmlsaWRhZCBSYWljZXNEZXN2aWFkYXMiLCB4bGFiID0gIlNlbnNpYmlsaWRhZCIpCmBgYApgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kNywgdHlwZSA9ICJzcGVjIiwgbWFpbiA9ICJFc3BlY2lmaWNpZGFkIFJhaWNlc0Rlc3ZpYWRhcyIpCmBgYApgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kNywgdHlwZSA9ICJET1IiLCBsb2cgPSBUUlVFLCAgbWFpbiA9ICJMb2cgRE9SIFJhaWNlc0Rlc3ZpYWRhcyIpCmBgYAoKIyMgIFJhaXpPc2N1cmFZQmlmaWRhCgpgYGB7cn0KbWQ4IDwtIGRmICU+JQogIHNlbGVjdChhdXRvciwKICAgICAgICAgc2lnbm9faWRlbnRpZmljYWRvLCAKICAgICAgICAgZ29sZF9zdGFuZGFyZCwgCiAgICAgICAgIHRlc3QsIAogICAgICAgICBUUCA9IHZwLCAKICAgICAgICAgRlAgPSBmcCwgCiAgICAgICAgIFROID0gdm4sIAogICAgICAgICBGTiA9IGZuKSAlPiUKICBmaWx0ZXIoVFAgPiAwKSAlPiUgCiAgZmlsdGVyKHRlc3QgPT0gIk9QRyIpICU+JSAKICBmaWx0ZXIoc2lnbm9faWRlbnRpZmljYWRvID09ICJSYWl6T3NjdXJhWUJpZmlkYSIpICU+JQogIG1hZGFkKC4sIGxldmVsID0gMC45NSkKbWQ4ICAKCgpgYGAKYGBge3J9Cm1hZGE6OmZvcmVzdChtZDgsIHR5cGUgPSAic2VucyIsIG1haW4gPSAiU2Vuc2liaWxpZGFkIFJhaXpPc2N1cmFZQmlmaWRhIiwgeGxhYiA9ICJTZW5zaWJpbGlkYWQiKQpgYGAKYGBge3J9Cm1hZGE6OmZvcmVzdChtZDgsIHR5cGUgPSAic3BlYyIsIG1haW4gPSAiRXNwZWNpZmljaWRhZCBSYWl6T3NjdXJhWUJpZmlkYSIpCmBgYApgYGB7cn0KbWFkYTo6Zm9yZXN0KG1kOCwgdHlwZSA9ICJET1IiLCBsb2cgPSBUUlVFLCAgbWFpbiA9ICJMb2cgRE9SIFJhaXpPc2N1cmFZQmlmaWRhIikKYGBgCiAK