Prism / Ethereum Nodes | Visualization Notebook

Data

Nodes data import

# Import data from API
library(httr2)
library(dplyr)
library(purrr)
library(readr)
library(here)
library(stringr)
library(glue)
# library(shiny)
# library(DT)

if (!require(leaflet)) install.packages("leaflet")
library(leaflet)

# build_req <- function(
#     limit, offset,
#     api_endpoint = "https://ingest-data-staging.dextrac.com:8081",
#     api_token = paste0(
#       "access_token=",
#       readr::read_lines(here("dt_api_token.txt"))
#     )
# ){
#   req <- request(api_endpoint) %>%
#     req_headers(Cookie = api_token) %>%
#     req_url_path_append("ethereum/nodes", limit, offset)
# }
# 
# reqs <- map(seq(0, 26000, 100), \(x) build_req(100, x))
# 
# resps <- req_perform_parallel(reqs)
# 
# df_resp <- resps %>% resps_successes() %>%
#   map(., \(x) resp_body_json(x, simplifyDataFrame=T)) %>%
#   bind_rows()
# 
# readr::write_csv(
#   df_resp, here("parse_eth_nodes_data", "nodes.csv")
# )

nodes <- readr::read_csv(
  here("parse_eth_nodes_data", "nodes.csv"),
  na = character()
)

Geojson import

# Import geojson data as SpatialPolygons
library(sp)
library(geojsonio)
path_geocountries_geojson <- here("parse_eth_nodes_data", "countries.geojson")
path_geoboundaries_geojson <- here("parse_eth_nodes_data", "geoBoundariesCGAZ_ADM0.geojson")

countries_sp <- geojsonio::geojson_read(path_geocountries_geojson, what = "sp")
# geobound_sp <- geojsonio::geojson_read(path_geoboundaries_geojson, what="sp")

Categorization benchmark

# Perform country assignment on data from API, based on geojson data as SpatialPolygonDataFrame
library(maps)

system.time(map.where(  # Benchmark
  database = countries_sp,
  x = nodes$Longitude,
  y = nodes$Latitude,
  namefield = "ADMIN"
))

##    user  system elapsed 
##   0.483   0.010   0.496

# user  system elapsed 
# 0.411   0.007   0.419

Process nodes data

nodes <- nodes %>%
  filter(
    !(Longitude == 0 & Latitude == 0)
  ) %>%
  mutate(
    geo_country_raw = map.where(
      database = countries_sp, x = Longitude, y = Latitude, namefield="ADMIN"
    ),
    geo_country = geo_country_raw %>%
      str_replace_all(":.+?$", "")
  )

nodes_aggr <- nodes %>%
  transmute(
    lat = Latitude,
    long = Longitude,
    country = geo_country
  ) %>%
  group_by(country, lat, long) %>%
  summarise(
    value = n()
  ) %>%
  ungroup()
# 
# nodes_country_aggr <- nodes_aggr %>% 
#   group_by(country) %>% 
#   summarise(
#     value = sum(value, na.rm=T)
#   )
# 
iso_nodes <- nodes %>%
  filter(
    !(Longitude == 0 & Latitude == 0)
  ) %>%
  mutate(
    ISOa2 = CountryCode
  )

iso_nodes_aggr <- iso_nodes %>%
  group_by(ISOa2) %>%
  summarise(
    value = n()
  )

Assign polygon attributes

# Add processed data as map polygon attributes
csp <- countries_sp
# gsp <- geobound_sp@data

nodePal <- colorRampPalette(c('lightblue', 'darkblue'))

cspdata <- left_join(
  # x=csp@data, y=nodes_country_aggr,
  # by=join_by(ADMIN == country)
  x=csp@data, y=iso_nodes_aggr,
  by=join_by(ISO_A2==ISOa2)
) %>% 
  mutate(
    base_label = paste0(ISO_A2, ' ', ADMIN),
    label = case_when(
      value == 1 ~ paste0(
        base_label, " [", value, " node]"
      ),
      value > 1 ~ paste0(
        base_label, " [", value, " nodes]"
      ),
      T ~ paste0(
        base_label, " [no nodes]"
      )
    ),
    color = nodePal(40)[
      as.numeric(cut(log10(value), breaks = 40))
    ]
  )
  
csp@data <- cspdata

# Alternative color handling
bins <- c(0, 10, 20, 50, 100, 200, 500, 1000, 5000, Inf)
pal <- colorBin("YlOrRd", domain = cspdata$value, bins = bins)

# ISO A2 diff
unique_iso_nodes <- unique(nodes$CountryCode)
unique_iso_geojson <- unique(csp$ISO_A2)

setdiff(unique_iso_nodes, unique_iso_geojson)

## [1] "RE" "XK"

# # ISO A3 diff
# unique_iso3_geobound <- unique(gsp$shapeGroup)

Tables

Node count by attribute

selectInput(
  'nodes_col', label = 'Grouping column:',
  choices = c(
    "City", "Region", "Country", "CountryCode", "ContinentCode", "InEU",
    "Postal", "Timezone", "UTCOffset", "Organization"
  ), selected = "Country"
)

renderDataTable({
  node_count <- nodes %>% 
    group_by_at(input$nodes_col) %>% 
    summarise(
      node_count = n()
    ) %>% 
    arrange(
      desc(node_count)
    )
},
options = list(
  pageLength = 15,
  lengthMenu = c(10, 15, 25, 50, 100),
  selection = "none"
))

Visualization

Leaflet (via leaflet::)

# Create a leaflet map
leaflet(csp) %>% 
  addTiles() %>% 
  addMarkers(
    data = nodes,
    lng = ~Longitude, 
    lat = ~Latitude,
    clusterOptions = markerClusterOptions(
      removeOutsideVisibleBounds = F,
      spiderfyOnMaxZoom = T
    )
  ) %>% 
  addPolygons(
    weight=1,
    label=csp$label,
    color="white",
    dashArray="3",
    fillColor=~pal(csp$value),
    fillOpacity=0.5,
    data=csp
  )

Three.js (via threejs::)

# Create a threejs globe
library(threejs)

globejs(lat = nodes_aggr$lat,
        long = nodes_aggr$long,
        val = 10 * log(nodes_aggr$value),
        pointsize = 0.5,
        atmosphere = TRUE)