Configuration

Volume

{
    "AvailabilityZone": "us-east-1a", 
    "Attachments": [
        {
            "AttachTime": "2018-02-07T15:13:07.000Z", 
            "InstanceId": "i-09a31c8d356966248", 
            "VolumeId": "vol-0d8d10c68176aa97b", 
            "State": "attached", 
            "DeleteOnTermination": false, 
            "Device": "/dev/sdf"
        }
    ], 
    "Encrypted": false, 
    "VolumeType": "gp2", 
    "VolumeId": "vol-0d8d10c68176aa97b", 
    "State": "in-use", 
    "Iops": 900, 
    "SnapshotId": "", 
    "CreateTime": "2018-01-19T20:14:49.325Z", 
    "Size": 300
}

ec2 instance

{
  "IamFleetRole": "arn:aws:iam::385009899373:role/aws-ec2-spot-fleet-tagging-role",
  "AllocationStrategy": "lowestPrice",
  "TargetCapacity": 1,
  "SpotPrice": "0.6",
  "ValidFrom": "2018-01-15T18:27:00Z",
  "ValidUntil": "2019-01-15T18:27:00Z",
  "TerminateInstancesWithExpiration": true,
  "LaunchSpecifications": [
    {
      "ImageId": "ami-3480df4e",
      "InstanceType": "c4.4xlarge",
      "SubnetId": "subnet-d2d5c7a4",
      "KeyName": "CellProfiler",
      "SpotPrice": "0.5",
      "IamInstanceProfile": {
        "Arn": "arn:aws:iam::385009899373:instance-profile/s3-imaging-platform-role"
      },
      "BlockDeviceMappings": [
        {
          "DeviceName": "/dev/sda1",
          "Ebs": {
            "DeleteOnTermination": true,
            "VolumeType": "gp2",
            "VolumeSize": 60,
            "SnapshotId": "snap-0d61c49c2c8ecee7a"
          }
        }
      ],
      "SecurityGroups": [
        {
          "GroupId": "sg-2a88ab51"
        },
        {
          "GroupId": "sg-74b99a0f"
        }
      ],
      "TagSpecifications": [
        {
          "ResourceType": "instance",
          "Tags": [
            {
              "Key": "Name",
              "Value": "Shantanu-cytotools"
            }
          ]
        }
      ]
    }
  ],
  "Type": "request"
}

Software

CP 2.2.1 Pipeline

Setup

  • All images were copied to the 300 Gb EBS volume and processes, per well, using GNU parallel, across 16 cores.

Parallel config

skopy

parallel \
  --no-run-if-empty \
  --eta \
  --results ../../log/${BATCH_ID}/skopy-analysis/{/.} \
  --joblog ../../log/${BATCH_ID}/skopy-analysis.log \
  --keep-order \
  -a ../../scratch/ljosa_2013/skopy_commands_analysis.txt

CellProfiler 2.2.1

parallel   
  --no-run-if-empty \
  --eta \
  --results ../../log/${BATCH_ID}/analysis/{/.} \
  --joblog ../../log/${BATCH_ID}/analysis.log \
  --keep-order \
  -a ../../scratch/${BATCH_ID}/cp_docker_commands_analysis.txt
library(glue)
library(magrittr)
library(stringr)
library(tidyverse)
library(corrplot)
skopy_version <- "31f62607"
cp_log <- read_tsv("timing/cp-221-analysis.log") 
cp_log %<>% 
  rowwise() %>% 
  mutate(
    pws = Command %>% 
      str_split("/status_dir/") %>% 
      extract2(1) %>% 
      extract2(2) %>% 
      str_split("\\.") %>% 
      extract2(1) %>% 
      extract2(1)) %>% 
  ungroup() %>% 
  separate(pws, into = c("Metadata_Plate", "Metadata_Well"), sep = "-") %>% 
  select(Metadata_Plate, Metadata_Well, JobRuntime)
skopy_log <- read_tsv(glue("timing/skopy-{version}-analysis.log", version = skopy_version)) 
skopy_log %<>% 
  rowwise() %>% 
  mutate(
    pws = Command %>% 
      str_split(" && ") %>% extract2(1) %>% extract2(1) %>% str_split("features_") %>% extract2(1) %>% extract2(2)) %>% 
  ungroup() %>% 
  separate(pws, into = c("Metadata_Plate1", "Metadata_Plate2", "Metadata_Well"), sep = "_", extra = "drop") %>% 
  unite(Metadata_Plate, Metadata_Plate1, Metadata_Plate2, remove = T, sep = "_") %>%
  select(Metadata_Plate, Metadata_Well, JobRuntime)
full_log <-
  inner_join(
    skopy_log,
    cp_log,
    by = c("Metadata_Plate", "Metadata_Well"),
    suffix = c("_skopy", "_cp")
  )
maxtime <- 
  full_log %>% 
  gather(sw, time, -Metadata_Plate, -Metadata_Well) %>%
  summarize(max_time = max(time)) %>%
  extract2("max_time")
ggplot(full_log, aes(JobRuntime_skopy, JobRuntime_cp)) + 
  geom_hex(binwidth = 20) +
  geom_abline(slope = 1, intercept = 0, linetype = 2, color = "red", alpha = 0.5) +
  xlim(0, maxtime) +
  ylim(0, maxtime) +
  coord_equal() +
  ggtitle("Run time per well (n = 632)")

full_log %>% 
  gather(sw, time, -Metadata_Plate, -Metadata_Well) %>%
  ggplot(aes(sw, time)) + 
  geom_boxplot() +
  ggtitle("Run time per well (n = 632)")

ggplot(full_log, aes(JobRuntime_cp/JobRuntime_skopy-1)) + 
  scale_x_continuous(labels = scales::percent) +
  xlab("speedup") + 
  geom_histogram(binwidth = .1) +
  ggtitle("Speedup per well (n = 632)")

ggplot(full_log, aes(JobRuntime_cp/JobRuntime_skopy-1)) + 
  scale_x_continuous(labels = scales::percent) +
  xlab("speedup") + 
  stat_ecdf() +
  ggtitle("Speedup per well (n = 632)")

full_log %>% 
  summarise_at(c("JobRuntime_cp", "JobRuntime_skopy"), sum) %>%
  gather(sw, time) %>%
  ggplot(aes(sw, time/3600)) + 
  ylab("hours") +
  geom_bar(stat = "identity") +
  ggtitle("Estimated run time on a single core (n = 632)")

cp_log <- read_tsv("timing/cp-221-analysis.log") 
skopy_log <- read_tsv(glue("timing/skopy-{version}-analysis.log", version = skopy_version)) 
get_wall_time <- function(runlog) {
  
  s1 <- runlog %>% arrange(Seq) %>% extract2("Seq")
  s2 <- seq(nrow(runlog))
  
  stopifnot(all(s1==s2))
  
  t1 <- runlog %>% filter(Seq == s1[[1]]) %>% extract2("Starttime")
    
  t2 <- runlog %>% filter(Seq == s1[[nrow(runlog)]]) %>% extract2("Starttime")
  
  t2 - t1
    
}
tribble(~cp, ~skopy,
        get_wall_time(cp_log), 
        get_wall_time(skopy_log)) %>% 
  gather(sw, time) %>%
  ggplot(aes(sw, time/3600)) + 
  ylab("hours") +
  geom_bar(stat = "identity")

  ggtitle("Estimated run time on a 16 cores, across all n = 632 wells")
$title
[1] "Estimated run time on a 16 cores, across all n = 632 wells"

$subtitle
NULL

attr(,"class")
[1] "labels"
data_frame(wall_time_speedup = 
             (get_wall_time(cp_log) / get_wall_time(skopy_log)) - 1) %>%
  ggplot(aes("-", wall_time_speedup)) + 
  geom_bar(stat = "identity") +
  xlab("") +
  scale_y_continuous(labels = scales::percent, limits = c(0, 1))  +
  ggtitle("Estimated speedup on 16 cores, across all n = 632 wells")

LS0tCnRpdGxlOiAic2tvcHkgdGltaW5nIG9uIEJCQkMwMjEiCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCiMgQ29uZmlndXJhdGlvbgoKIyMgVm9sdW1lCgpgYGAKewogICAgIkF2YWlsYWJpbGl0eVpvbmUiOiAidXMtZWFzdC0xYSIsIAogICAgIkF0dGFjaG1lbnRzIjogWwogICAgICAgIHsKICAgICAgICAgICAgIkF0dGFjaFRpbWUiOiAiMjAxOC0wMi0wN1QxNToxMzowNy4wMDBaIiwgCiAgICAgICAgICAgICJJbnN0YW5jZUlkIjogImktMDlhMzFjOGQzNTY5NjYyNDgiLCAKICAgICAgICAgICAgIlZvbHVtZUlkIjogInZvbC0wZDhkMTBjNjgxNzZhYTk3YiIsIAogICAgICAgICAgICAiU3RhdGUiOiAiYXR0YWNoZWQiLCAKICAgICAgICAgICAgIkRlbGV0ZU9uVGVybWluYXRpb24iOiBmYWxzZSwgCiAgICAgICAgICAgICJEZXZpY2UiOiAiL2Rldi9zZGYiCiAgICAgICAgfQogICAgXSwgCiAgICAiRW5jcnlwdGVkIjogZmFsc2UsIAogICAgIlZvbHVtZVR5cGUiOiAiZ3AyIiwgCiAgICAiVm9sdW1lSWQiOiAidm9sLTBkOGQxMGM2ODE3NmFhOTdiIiwgCiAgICAiU3RhdGUiOiAiaW4tdXNlIiwgCiAgICAiSW9wcyI6IDkwMCwgCiAgICAiU25hcHNob3RJZCI6ICIiLCAKICAgICJDcmVhdGVUaW1lIjogIjIwMTgtMDEtMTlUMjA6MTQ6NDkuMzI1WiIsIAogICAgIlNpemUiOiAzMDAKfQpgYGAKCiMjIGVjMiBpbnN0YW5jZQpgYGAKewogICJJYW1GbGVldFJvbGUiOiAiYXJuOmF3czppYW06OjM4NTAwOTg5OTM3Mzpyb2xlL2F3cy1lYzItc3BvdC1mbGVldC10YWdnaW5nLXJvbGUiLAogICJBbGxvY2F0aW9uU3RyYXRlZ3kiOiAibG93ZXN0UHJpY2UiLAogICJUYXJnZXRDYXBhY2l0eSI6IDEsCiAgIlNwb3RQcmljZSI6ICIwLjYiLAogICJWYWxpZEZyb20iOiAiMjAxOC0wMS0xNVQxODoyNzowMFoiLAogICJWYWxpZFVudGlsIjogIjIwMTktMDEtMTVUMTg6Mjc6MDBaIiwKICAiVGVybWluYXRlSW5zdGFuY2VzV2l0aEV4cGlyYXRpb24iOiB0cnVlLAogICJMYXVuY2hTcGVjaWZpY2F0aW9ucyI6IFsKICAgIHsKICAgICAgIkltYWdlSWQiOiAiYW1pLTM0ODBkZjRlIiwKICAgICAgIkluc3RhbmNlVHlwZSI6ICJjNC40eGxhcmdlIiwKICAgICAgIlN1Ym5ldElkIjogInN1Ym5ldC1kMmQ1YzdhNCIsCiAgICAgICJLZXlOYW1lIjogIkNlbGxQcm9maWxlciIsCiAgICAgICJTcG90UHJpY2UiOiAiMC41IiwKICAgICAgIklhbUluc3RhbmNlUHJvZmlsZSI6IHsKICAgICAgICAiQXJuIjogImFybjphd3M6aWFtOjozODUwMDk4OTkzNzM6aW5zdGFuY2UtcHJvZmlsZS9zMy1pbWFnaW5nLXBsYXRmb3JtLXJvbGUiCiAgICAgIH0sCiAgICAgICJCbG9ja0RldmljZU1hcHBpbmdzIjogWwogICAgICAgIHsKICAgICAgICAgICJEZXZpY2VOYW1lIjogIi9kZXYvc2RhMSIsCiAgICAgICAgICAiRWJzIjogewogICAgICAgICAgICAiRGVsZXRlT25UZXJtaW5hdGlvbiI6IHRydWUsCiAgICAgICAgICAgICJWb2x1bWVUeXBlIjogImdwMiIsCiAgICAgICAgICAgICJWb2x1bWVTaXplIjogNjAsCiAgICAgICAgICAgICJTbmFwc2hvdElkIjogInNuYXAtMGQ2MWM0OWMyYzhlY2VlN2EiCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAiU2VjdXJpdHlHcm91cHMiOiBbCiAgICAgICAgewogICAgICAgICAgIkdyb3VwSWQiOiAic2ctMmE4OGFiNTEiCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiR3JvdXBJZCI6ICJzZy03NGI5OWEwZiIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJUYWdTcGVjaWZpY2F0aW9ucyI6IFsKICAgICAgICB7CiAgICAgICAgICAiUmVzb3VyY2VUeXBlIjogImluc3RhbmNlIiwKICAgICAgICAgICJUYWdzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIktleSI6ICJOYW1lIiwKICAgICAgICAgICAgICAiVmFsdWUiOiAiU2hhbnRhbnUtY3l0b3Rvb2xzIgogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdCiAgICB9CiAgXSwKICAiVHlwZSI6ICJyZXF1ZXN0Igp9CmBgYAoKIyMgU29mdHdhcmUKLSBkb2NrZXIgaW1hZ2Ugb2YgQ2VsbFByb2ZpbGVyIDIuMi4xIHNobnRudS9jZWxscHJvZmlsZXI6Mi4yLjEgaHR0cHM6Ly9odWIuZG9ja2VyLmNvbS9yL3NobnRudS9jZWxscHJvZmlsZXIvdGFncy8KLSBza29weSBodHRwczovL2dpdGh1Yi5jb20vYnJvYWRpbnN0aXR1dGUvc2tvcHkvdHJlZS8zMWY2MjYwNzg1MDkwMzJmNDUzYjMzOGY5OTNhMjA3ZWZhNTYyNjU2LwotIEdOVSBwYXJhbGxlbCB2MjAxODAxMjIKCiMjIENQIDIuMi4xIFBpcGVsaW5lCi0gaHR0cHM6Ly9naXRodWIuY29tL2Jyb2FkaW5zdGl0dXRlL2ltYWdpbmctcGxhdGZvcm0tcGlwZWxpbmVzL2Jsb2IvbWFzdGVyL2JiYmMwMjFfbWNmN18yMHhfaW1hZ2V4cHJlc3MvYW5hbHlzaXMuY3BwaXBlClByb2Nlc3Npbmcgd2FzIGdyb3VwZWQgYnkgTWV0YWRhdGFfUGxhdGUsIE1ldGFkYXRhX1dlbGwuIDYzMiBncm91cHMgaW4gdG90YWwKCiMjIFNldHVwCi0gQWxsIGltYWdlcyB3ZXJlIGNvcGllZCB0byB0aGUgMzAwIEdiIEVCUyB2b2x1bWUgYW5kIHByb2Nlc3NlcywgcGVyIHdlbGwsIHVzaW5nIEdOVSBwYXJhbGxlbCwgYWNyb3NzIDE2IGNvcmVzLgoKIyMgUGFyYWxsZWwgY29uZmlnCgojIyMgc2tvcHkgCmBgYApwYXJhbGxlbCBcCiAgLS1uby1ydW4taWYtZW1wdHkgXAogIC0tZXRhIFwKICAtLXJlc3VsdHMgLi4vLi4vbG9nLyR7QkFUQ0hfSUR9L3Nrb3B5LWFuYWx5c2lzL3svLn0gXAogIC0tam9ibG9nIC4uLy4uL2xvZy8ke0JBVENIX0lEfS9za29weS1hbmFseXNpcy5sb2cgXAogIC0ta2VlcC1vcmRlciBcCiAgLWEgLi4vLi4vc2NyYXRjaC9sam9zYV8yMDEzL3Nrb3B5X2NvbW1hbmRzX2FuYWx5c2lzLnR4dApgYGAKCiMjIyBDZWxsUHJvZmlsZXIgMi4yLjEKYGBgCnBhcmFsbGVsICAgCiAgLS1uby1ydW4taWYtZW1wdHkgXAogIC0tZXRhIFwKICAtLXJlc3VsdHMgLi4vLi4vbG9nLyR7QkFUQ0hfSUR9L2FuYWx5c2lzL3svLn0gXAogIC0tam9ibG9nIC4uLy4uL2xvZy8ke0JBVENIX0lEfS9hbmFseXNpcy5sb2cgXAogIC0ta2VlcC1vcmRlciBcCiAgLWEgLi4vLi4vc2NyYXRjaC8ke0JBVENIX0lEfS9jcF9kb2NrZXJfY29tbWFuZHNfYW5hbHlzaXMudHh0CmBgYAoKYGBge3IgbWVzc2FnZT1GQUxTRX0KbGlicmFyeShnbHVlKQpsaWJyYXJ5KG1hZ3JpdHRyKQpsaWJyYXJ5KHN0cmluZ3IpCmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KGNvcnJwbG90KQpgYGAKCgpgYGB7cn0Kc2tvcHlfdmVyc2lvbiA8LSAiMzFmNjI2MDciCmBgYAoKCmBgYHtyIG1lc3NhZ2U9RkFMU0V9CmNwX2xvZyA8LSByZWFkX3RzdigidGltaW5nL2NwLTIyMS1hbmFseXNpcy5sb2ciKSAKCmNwX2xvZyAlPD4lIAogIHJvd3dpc2UoKSAlPiUgCiAgbXV0YXRlKAogICAgcHdzID0gQ29tbWFuZCAlPiUgCiAgICAgIHN0cl9zcGxpdCgiL3N0YXR1c19kaXIvIikgJT4lIAogICAgICBleHRyYWN0MigxKSAlPiUgCiAgICAgIGV4dHJhY3QyKDIpICU+JSAKICAgICAgc3RyX3NwbGl0KCJcXC4iKSAlPiUgCiAgICAgIGV4dHJhY3QyKDEpICU+JSAKICAgICAgZXh0cmFjdDIoMSkpICU+JSAKICB1bmdyb3VwKCkgJT4lIAogIHNlcGFyYXRlKHB3cywgaW50byA9IGMoIk1ldGFkYXRhX1BsYXRlIiwgIk1ldGFkYXRhX1dlbGwiKSwgc2VwID0gIi0iKSAlPiUgCiAgc2VsZWN0KE1ldGFkYXRhX1BsYXRlLCBNZXRhZGF0YV9XZWxsLCBKb2JSdW50aW1lKQoKc2tvcHlfbG9nIDwtIHJlYWRfdHN2KGdsdWUoInRpbWluZy9za29weS17dmVyc2lvbn0tYW5hbHlzaXMubG9nIiwgdmVyc2lvbiA9IHNrb3B5X3ZlcnNpb24pKSAKCnNrb3B5X2xvZyAlPD4lIAogIHJvd3dpc2UoKSAlPiUgCiAgbXV0YXRlKAogICAgcHdzID0gQ29tbWFuZCAlPiUgCiAgICAgIHN0cl9zcGxpdCgiICYmICIpICU+JSBleHRyYWN0MigxKSAlPiUgZXh0cmFjdDIoMSkgJT4lIHN0cl9zcGxpdCgiZmVhdHVyZXNfIikgJT4lIGV4dHJhY3QyKDEpICU+JSBleHRyYWN0MigyKSkgJT4lIAogIHVuZ3JvdXAoKSAlPiUgCiAgc2VwYXJhdGUocHdzLCBpbnRvID0gYygiTWV0YWRhdGFfUGxhdGUxIiwgIk1ldGFkYXRhX1BsYXRlMiIsICJNZXRhZGF0YV9XZWxsIiksIHNlcCA9ICJfIiwgZXh0cmEgPSAiZHJvcCIpICU+JSAKICB1bml0ZShNZXRhZGF0YV9QbGF0ZSwgTWV0YWRhdGFfUGxhdGUxLCBNZXRhZGF0YV9QbGF0ZTIsIHJlbW92ZSA9IFQsIHNlcCA9ICJfIikgJT4lCiAgc2VsZWN0KE1ldGFkYXRhX1BsYXRlLCBNZXRhZGF0YV9XZWxsLCBKb2JSdW50aW1lKQoKZnVsbF9sb2cgPC0KICBpbm5lcl9qb2luKAogICAgc2tvcHlfbG9nLAogICAgY3BfbG9nLAogICAgYnkgPSBjKCJNZXRhZGF0YV9QbGF0ZSIsICJNZXRhZGF0YV9XZWxsIiksCiAgICBzdWZmaXggPSBjKCJfc2tvcHkiLCAiX2NwIikKICApCgptYXh0aW1lIDwtIAogIGZ1bGxfbG9nICU+JSAKICBnYXRoZXIoc3csIHRpbWUsIC1NZXRhZGF0YV9QbGF0ZSwgLU1ldGFkYXRhX1dlbGwpICU+JQogIHN1bW1hcml6ZShtYXhfdGltZSA9IG1heCh0aW1lKSkgJT4lCiAgZXh0cmFjdDIoIm1heF90aW1lIikKCmdncGxvdChmdWxsX2xvZywgYWVzKEpvYlJ1bnRpbWVfc2tvcHksIEpvYlJ1bnRpbWVfY3ApKSArIAogIGdlb21faGV4KGJpbndpZHRoID0gMjApICsKICBnZW9tX2FibGluZShzbG9wZSA9IDEsIGludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gMiwgY29sb3IgPSAicmVkIiwgYWxwaGEgPSAwLjUpICsKICB4bGltKDAsIG1heHRpbWUpICsKICB5bGltKDAsIG1heHRpbWUpICsKICBjb29yZF9lcXVhbCgpICsKICBnZ3RpdGxlKCJSdW4gdGltZSBwZXIgd2VsbCAobiA9IDYzMikiKQoKZnVsbF9sb2cgJT4lIAogIGdhdGhlcihzdywgdGltZSwgLU1ldGFkYXRhX1BsYXRlLCAtTWV0YWRhdGFfV2VsbCkgJT4lCiAgZ2dwbG90KGFlcyhzdywgdGltZSkpICsgCiAgZ2VvbV9ib3hwbG90KCkgKwogIGdndGl0bGUoIlJ1biB0aW1lIHBlciB3ZWxsIChuID0gNjMyKSIpCgpnZ3Bsb3QoZnVsbF9sb2csIGFlcyhKb2JSdW50aW1lX2NwL0pvYlJ1bnRpbWVfc2tvcHktMSkpICsgCiAgc2NhbGVfeF9jb250aW51b3VzKGxhYmVscyA9IHNjYWxlczo6cGVyY2VudCkgKwogIHhsYWIoInNwZWVkdXAiKSArIAogIGdlb21faGlzdG9ncmFtKGJpbndpZHRoID0gLjEpICsKICBnZ3RpdGxlKCJTcGVlZHVwIHBlciB3ZWxsIChuID0gNjMyKSIpCgpnZ3Bsb3QoZnVsbF9sb2csIGFlcyhKb2JSdW50aW1lX2NwL0pvYlJ1bnRpbWVfc2tvcHktMSkpICsgCiAgc2NhbGVfeF9jb250aW51b3VzKGxhYmVscyA9IHNjYWxlczo6cGVyY2VudCkgKwogIHhsYWIoInNwZWVkdXAiKSArIAogIHN0YXRfZWNkZigpICsKICBnZ3RpdGxlKCJTcGVlZHVwIHBlciB3ZWxsIChuID0gNjMyKSIpCgpmdWxsX2xvZyAlPiUgCiAgc3VtbWFyaXNlX2F0KGMoIkpvYlJ1bnRpbWVfY3AiLCAiSm9iUnVudGltZV9za29weSIpLCBzdW0pICU+JQogIGdhdGhlcihzdywgdGltZSkgJT4lCiAgZ2dwbG90KGFlcyhzdywgdGltZS8zNjAwKSkgKyAKICB5bGFiKCJob3VycyIpICsKICBnZW9tX2JhcihzdGF0ID0gImlkZW50aXR5IikgKwogIGdndGl0bGUoIkVzdGltYXRlZCBydW4gdGltZSBvbiBhIHNpbmdsZSBjb3JlIChuID0gNjMyKSIpCgpjcF9sb2cgPC0gcmVhZF90c3YoInRpbWluZy9jcC0yMjEtYW5hbHlzaXMubG9nIikgCgpza29weV9sb2cgPC0gcmVhZF90c3YoZ2x1ZSgidGltaW5nL3Nrb3B5LXt2ZXJzaW9ufS1hbmFseXNpcy5sb2ciLCB2ZXJzaW9uID0gc2tvcHlfdmVyc2lvbikpIAoKZ2V0X3dhbGxfdGltZSA8LSBmdW5jdGlvbihydW5sb2cpIHsKICAKICBzMSA8LSBydW5sb2cgJT4lIGFycmFuZ2UoU2VxKSAlPiUgZXh0cmFjdDIoIlNlcSIpCiAgczIgPC0gc2VxKG5yb3cocnVubG9nKSkKICAKICBzdG9waWZub3QoYWxsKHMxPT1zMikpCiAgCiAgdDEgPC0gcnVubG9nICU+JSBmaWx0ZXIoU2VxID09IHMxW1sxXV0pICU+JSBleHRyYWN0MigiU3RhcnR0aW1lIikKICAgIAogIHQyIDwtIHJ1bmxvZyAlPiUgZmlsdGVyKFNlcSA9PSBzMVtbbnJvdyhydW5sb2cpXV0pICU+JSBleHRyYWN0MigiU3RhcnR0aW1lIikKICAKICB0MiAtIHQxCiAgICAKfQoKdHJpYmJsZSh+Y3AsIH5za29weSwKICAgICAgICBnZXRfd2FsbF90aW1lKGNwX2xvZyksIAogICAgICAgIGdldF93YWxsX3RpbWUoc2tvcHlfbG9nKSkgJT4lIAogIGdhdGhlcihzdywgdGltZSkgJT4lCiAgZ2dwbG90KGFlcyhzdywgdGltZS8zNjAwKSkgKyAKICB5bGFiKCJob3VycyIpICsKICBnZW9tX2JhcihzdGF0ID0gImlkZW50aXR5IikKICBnZ3RpdGxlKCJFc3RpbWF0ZWQgcnVuIHRpbWUgb24gYSAxNiBjb3JlcywgYWNyb3NzIGFsbCBuID0gNjMyIHdlbGxzIikKCmRhdGFfZnJhbWUod2FsbF90aW1lX3NwZWVkdXAgPSAKICAgICAgICAgICAgIChnZXRfd2FsbF90aW1lKGNwX2xvZykgLyBnZXRfd2FsbF90aW1lKHNrb3B5X2xvZykpIC0gMSkgJT4lCiAgZ2dwbG90KGFlcygiLSIsIHdhbGxfdGltZV9zcGVlZHVwKSkgKyAKICBnZW9tX2JhcihzdGF0ID0gImlkZW50aXR5IikgKwogIHhsYWIoIiIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50LCBsaW1pdHMgPSBjKDAsIDEpKSAgKwogIGdndGl0bGUoIkVzdGltYXRlZCBzcGVlZHVwIG9uIDE2IGNvcmVzLCBhY3Jvc3MgYWxsIG4gPSA2MzIgd2VsbHMiKQoKYGBgCgo=