OpenMP Tools

Summary

This is a very brief bit on how OpenMP tools work in principle.

Prereq.

Clang11 or better is required, i.e. anything with openmp5.0 and the runtime compiled to support the OMPT interface.

The tool will be a shared lib that is either LD_PRELOADed or set via OMP_TOOL_LIBRARIES env.

Hello world example

This is just the tool init and finalise method:

#include <stdio.h>
#include <omp.h>
#include <ompt.h>

int ompt_initialize(
        ompt_function_lookup_t lookup,
        int initial_device_num,
        ompt_data_t *data)
{
  printf("[INIT] init time: %f\n", omp_get_wtime()-*(double*)(data->ptr));
  return 1;
}

void ompt_finalize(ompt_data_t* data)
{
  printf("[FINAL] application runtime: %f\n", omp_get_wtime()-*(double*)(data->ptr));
}
  
ompt_start_tool_result_t* ompt_start_tool(
  unsigned int omp_version,
  const char *runtime_version)
{
  printf("[START] %s, OMP v. %u\n", runtime_version, omp_version);

  static double time=0;
  time=omp_get_wtime(); // The data pointer of the tool is initialiseD with time

  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize,{.ptr=&time}};
  return &ompt_start_tool_result;
}

Compile as such:

clang -g mytool.c  -g -shared -fPIC -o mytool.so

To run the tool:

OMP_TOOL_LIBRARIES=mytool.so ./application

Explanations

The start_tool function is what the runtime looks for. The return code is important. The runtime will keep trying to find tools — if more than one tool is loaded there is no guarantee which one will be used.

Adding callbacks

The logic is that in the initialize function, we register which events we want the tool to listen to. The method requires two C-macros, the first macro uses the set_callback method:

#define register_callback_t(name, type)                       \
do{                                                           \
  type f_##name = &on_##name;                                 \
  if (ompt_set_callback(name, (ompt_callback_t)f_##name) ==   \
      ompt_set_never)                                         \
    printf("0: Could not register callback '" #name "'\n");   \
}while(0)

#define register_callback(name) register_callback_t(name, name##_t)

N.b. The do while(0) construct ensures that the intended macro logic is preserved.

The canonical way of adding callbacks to the tool is to register them in initialize and define a function.

int ompt_initialize(
        ompt_function_lookup_t lookup,
        int initial_device_num,
        ompt_data_t *data)
{
  printf("[INIT] init time: %f\n", omp_get_wtime()-*(double*)(data->ptr));
  ompt_set_callback_t ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback"); // Used in macro above
  register_callback(ompt_callback_thread_begin);
  return 1;
}

static void
on_ompt_callback_thread_begin(
    ompt_thread_t            thread_type,
    ompt_data_t             *thread
)
{
    int thread_id = omp_get_thread_num();
    printf("Thread: %d thread begin\n", thread_id);
}

Appendix

mytool.c

#include <stdio.h>
#include <omp.h>
#include <ompt.h>

static void
on_ompt_callback_thread_begin(
    ompt_thread_t            thread_type,
    ompt_data_t             *thread
)
{
    int thread_id = omp_get_thread_num();
    printf("Thread: %d thread begin\n", thread_id);
}

#define register_callback_t(name, type)                       \
do{                                                           \
  type f_##name = &on_##name;                                 \
  if (ompt_set_callback(name, (ompt_callback_t)f_##name) ==   \
      ompt_set_never)                                         \
    printf("0: Could not register callback '" #name "'\n");   \
}while(0)

#define register_callback(name) register_callback_t(name, name##_t)

int ompt_initialize(
        ompt_function_lookup_t lookup,
        int initial_device_num,
        ompt_data_t *data)
{
  ompt_set_callback_t ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback"); // Used in macro above
  register_callback(ompt_callback_thread_begin);
  printf("[INIT] init time: %f\n", omp_get_wtime()-*(double*)(data->ptr));
  return 1;
}

void ompt_finalize(ompt_data_t* data)
{
  printf("[FINAL] application runtime: %f\n", omp_get_wtime()-*(double*)(data->ptr));
}
  
ompt_start_tool_result_t* ompt_start_tool(
  unsigned int omp_version,
  const char *runtime_version)
{
  printf("[START] %s, OMP v. %u\n", runtime_version, omp_version);

  static double time=0;
  time=omp_get_wtime(); // The data pointer of the tool is initialiseD with time

  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize,{.ptr=&time}};
  return &ompt_start_tool_result;
}