Example Programs¶

C++ Example¶

Application Level Example:¶

 #include <dlio_profiler/dlio_profiler.h>

 void foo() {
   DLIO_PROFILER_CPP_FUNCTION(); // Add at the begining of each function
   sleep(1);
   {
     DLIO_PROFILER_CPP_REGION(CUSTOM); // Add at the beginning of code block. keep name unique
     sleep(1);
     DLIO_PROFILER_CPP_REGION_START(CUSTOM_BLOCK); // add start. keep name unique
     sleep(1);
     DLIO_PROFILER_CPP_REGION_END(CUSTOM_BLOCK); // add end. Match name from START.
   }
 }

 int main(int argc, char *argv[]) {
   // Basic Bookkeeping
   int init = 0;
   if (argc > 2) {
     if (strcmp(argv[2], "1") == 0) {
       // Initialize Application Profiler
       DLIO_PROFILER_CPP_INIT(nullptr, nullptr, nullptr);
       init = 1;
     }
   }
   char filename[1024];
   sprintf(filename, "%s/demofile.txt", argv[1]);

   // Run functions
   foo();
   // Implicit I/O calls No need for marking.
   FILE *fh = fopen(filename, "w+");
   if (fh != NULL) {
     fwrite("hello", sizeof("hello"), 1, fh);
     fclose(fh);
   }
   if (init == 1) {
     // Finalize Application Profiler
     DLIO_PROFILER_CPP_FINI();
   }
   return 0;
 }

For this example, link with libdlio_profiler.so at compile time. As the DLIO_PROFILER_CPP_INIT do not pass log file or data dir, we need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. By default the DLIO Profiler mode is set to FUNCTION. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset
 # Enable profiler
 DLIO_PROFILER_ENABLE=1

LD_PRELOAD Example:¶

 #include <dlio_profiler/dlio_profiler.h>

 int main(int argc, char *argv[]) {
   char filename[1024];
   sprintf(filename, "%s/demofile.txt", argv[1]);
   foo(); # function will be ignored in pure LD_PRELOAD mode.
   // Implicit I/O calls No need for marking.
   FILE *fh = fopen(filename, "w+");
   if (fh != NULL) {
     fwrite("hello", sizeof("hello"), 1, fh);
     fclose(fh);
   }
   return 0;
 }

For this example, LD_PRELOAD the executable with libdlio_profiler_preload.so at runtime. We need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 export DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 export DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset
 # Set the mode to PRELOAD
 export DLIO_PROFILER_INIT=PRELOAD
 # Enable profiler
 export DLIO_PROFILER_ENABLE=1

Hybrid Example:¶

 #include <dlio_profiler/dlio_profiler.h>

 void foo() {
   DLIO_PROFILER_CPP_FUNCTION(); // Add at the begining of each function
   sleep(1);
   {
     DLIO_PROFILER_CPP_REGION(CUSTOM); // Add at the beginning of code block. keep name unique
     sleep(1);
     DLIO_PROFILER_CPP_REGION_START(CUSTOM_BLOCK); // add start. keep name unique
     sleep(1);
     DLIO_PROFILER_CPP_REGION_END(CUSTOM_BLOCK); // add end. Match name from START.
   }
 }

 int main(int argc, char *argv[]) {
   // Basic Bookkeeping
   int init = 0;
   if (argc > 2) {
     if (strcmp(argv[2], "1") == 0) {
       // Initialize Application Profiler
       DLIO_PROFILER_CPP_INIT(nullptr, nullptr, nullptr);
       init = 1;
     }
   }
   char filename[1024];
   sprintf(filename, "%s/demofile.txt", argv[1]);

   // Run functions
   foo();
   // Implicit I/O calls No need for marking.
   FILE *fh = fopen(filename, "w+");
   if (fh != NULL) {
     fwrite("hello", sizeof("hello"), 1, fh);
     fclose(fh);
   }
   if (init == 1) {
     // Finalize Application Profiler
     DLIO_PROFILER_CPP_FINI();
   }
   return 0;
 }

For this example, link with libdlio_profiler.so at compile time and LD_PRELOAD the executable with libdlio_profiler_preload.soat runtime. As the DLIO_PROFILER_CPP_INIT do not pass log file or data dir, we need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. By default the DLIO Profiler mode is set to FUNCTION. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset
 # Set the mode to PRELOAD
 export DLIO_PROFILER_INIT=PRELOAD
 # Enable profiler
 DLIO_PROFILER_ENABLE=1

C Example¶

Application Level Example:¶

 #include <dlio_profiler/dlio_profiler.h>

 void foo() {
   DLIO_PROFILER_C_FUNCTION_START();
   sleep(1);
   if (<CONDITION>) {
     DLIO_PROFILER_C_FUNCTION_END();
     return; // Define DLIO_PROFILER_C_FUNCTION_END on every branch
   }
   {
     DLIO_PROFILER_C_REGION_START(CUSTOM);
     sleep(1);
     DLIO_PROFILER_C_REGION_END(CUSTOM); // END region CUSTOM.
   }
   DLIO_PROFILER_C_FUNCTION_END(); // Define DLIO_PROFILER_C_FUNCTION_END on every branch
 }

 int main(int argc, char *argv[]) {
   // Basic Bookkeeping
   int init = 0;
   if (argc > 2) {
     if (strcmp(argv[2], "1") == 0) {
       // Initialize Application Profiler
       DLIO_PROFILER_C_INIT(nullptr, nullptr, nullptr);
       init = 1;
     }
   }
   char filename[1024];
   sprintf(filename, "%s/demofile.txt", argv[1]);

   // Run functions
   foo();
   // Implicit I/O calls No need for marking.
   FILE *fh = fopen(filename, "w+");
   if (fh != NULL) {
     fwrite("hello", sizeof("hello"), 1, fh);
     fclose(fh);
   }
   if (init == 1) {
     // Finalize Application Profiler
     DLIO_PROFILER_C_FINI();
   }
   return 0;
 }

For this example, link with libdlio_profiler.so at compile time. As the DLIO_PROFILER_CPP_INIT do not pass log file or data dir, we need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. By default the DLIO Profiler mode is set to FUNCTION. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset
 # Enable profiler
 DLIO_PROFILER_ENABLE=1

LD_PRELOAD Example:¶

 #include <dlio_profiler/dlio_profiler.h>

 int main(int argc, char *argv[]) {
   char filename[1024];
   sprintf(filename, "%s/demofile.txt", argv[1]);
   foo(); # function will be ignored in pure LD_PRELOAD mode.
   // Implicit I/O calls No need for marking.
   FILE *fh = fopen(filename, "w+");
   if (fh != NULL) {
     fwrite("hello", sizeof("hello"), 1, fh);
     fclose(fh);
   }
   return 0;
 }

For this example, LD_PRELOAD the executable with libdlio_profiler_preload.so at runtime. We need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 export DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 export DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset
 # Set the mode to PRELOAD
 export DLIO_PROFILER_INIT=PRELOAD
 # Enable profiler
 export DLIO_PROFILER_ENABLE=1

Hybrid Example:¶

 #include <dlio_profiler/dlio_profiler.h>

 void foo() {
   DLIO_PROFILER_C_FUNCTION_START();
   sleep(1);
   if (<CONDITION>) {
     DLIO_PROFILER_C_FUNCTION_END();
     return; // Define DLIO_PROFILER_C_FUNCTION_END on every branch
   }
   {
     DLIO_PROFILER_C_REGION_START(CUSTOM);
     sleep(1);
     DLIO_PROFILER_C_REGION_END(CUSTOM); // END region CUSTOM.
   }
   DLIO_PROFILER_C_FUNCTION_END(); // Define DLIO_PROFILER_C_FUNCTION_END on every branch
 }

 int main(int argc, char *argv[]) {
   // Basic Bookkeeping
   int init = 0;
   if (argc > 2) {
     if (strcmp(argv[2], "1") == 0) {
       // Initialize Application Profiler
       DLIO_PROFILER_C_INIT(nullptr, nullptr, nullptr);
       init = 1;
     }
   }
   char filename[1024];
   sprintf(filename, "%s/demofile.txt", argv[1]);

   // Run functions
   foo();
   // Implicit I/O calls No need for marking.
   FILE *fh = fopen(filename, "w+");
   if (fh != NULL) {
     fwrite("hello", sizeof("hello"), 1, fh);
     fclose(fh);
   }
   if (init == 1) {
     // Finalize Application Profiler
     DLIO_PROFILER_C_FINI();
   }
   return 0;
 }

For this example, link with libdlio_profiler.so at compile time and LD_PRELOAD the executable with libdlio_profiler_preload.so at runtime. As the DLIO_PROFILER_CPP_INIT do not pass log file or data dir, we need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. By default the DLIO Profiler mode is set to FUNCTION. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset
 # Set the mode to PRELOAD
 export DLIO_PROFILER_INIT=PRELOAD
 # Enable profiler
 DLIO_PROFILER_ENABLE=1

Python Example¶

Application Level Example:¶

 from dlio_profiler.logger import dlio_logger, fn_interceptor
 log_inst = dlio_logger.initialize_log(logfile=None, data_dir=None, process_id=-1)
 dlio_log = fn_interceptor("COMPUTE")

 # Example of using function decorators
 @dlio_log.log
 def log_events(index):
     sleep(1)

 # Example of function spawning and implicit I/O calls
 def posix_calls(val):
     index, is_spawn = val
     path = f"{cwd}/data/demofile{index}.txt"
     f = open(path, "w+")
     f.write("Now the file has more content!")
     f.close()
     if is_spawn:
         print(f"Calling spawn on {index} with pid {os.getpid()}")
         log_inst.finalize() # This need to be called to correctly finalize DLIO Profiler.
     else:
         print(f"Not calling spawn on {index} with pid {os.getpid()}")

 # NPZ calls internally calls POSIX calls.
 def npz_calls(index):
     # print(f"{cwd}/data/demofile2.npz")
     path = f"{cwd}/data/demofile{index}.npz"
     if os.path.exists(path):
         os.remove(path)
     records = np.random.randint(255, size=(8, 8, 1024), dtype=np.uint8)
     record_labels = [0] * 1024
     np.savez(path, x=records, y=record_labels)

 def main():
     log_events(0)
     npz_calls(1)
     with get_context('spawn').Pool(1, initializer=init) as pool:
         pool.map(posix_calls, ((2, True),))
     log_inst.finalize()


 if __name__ == "__main__":
     main()

For this example, as the DLIO_PROFILER_CPP_INIT do not pass log file or data dir, we need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. By default the DLIO Profiler mode is set to FUNCTION. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset:$PWD/data
 # Enable profiler
 DLIO_PROFILER_ENABLE=1

LD_PRELOAD Example:¶

 # Example of function spawning and implicit I/O calls
 def posix_calls(val):
     index, is_spawn = val
     path = f"{cwd}/data/demofile{index}.txt"
     f = open(path, "w+")
     f.write("Now the file has more content!")
     f.close()
     if is_spawn:
         print(f"Calling spawn on {index} with pid {os.getpid()}")
     else:
         print(f"Not calling spawn on {index} with pid {os.getpid()}")

 # NPZ calls internally calls POSIX calls.
 def npz_calls(index):
     # print(f"{cwd}/data/demofile2.npz")
     path = f"{cwd}/data/demofile{index}.npz"
     if os.path.exists(path):
         os.remove(path)
     records = np.random.randint(255, size=(8, 8, 1024), dtype=np.uint8)
     record_labels = [0] * 1024
     np.savez(path, x=records, y=record_labels)

 def main():
     npz_calls(1)
     with get_context('spawn').Pool(1, initializer=init) as pool:
         pool.map(posix_calls, ((2, True),))

 if __name__ == "__main__":
     main()

For this example, LD_PRELOAD the executable with libdlio_profiler_preload.so at runtime. We need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 export DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 export DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset
 # Set the mode to PRELOAD
 export DLIO_PROFILER_INIT=PRELOAD
 # Enable profiler
 export DLIO_PROFILER_ENABLE=1

Hybrid Example:¶

 from dlio_profiler.logger import dlio_logger, fn_interceptor
 log_inst = dlio_logger.initialize_log(logfile=None, data_dir=None, process_id=-1)
 dlio_log = fn_interceptor("COMPUTE")

 # Example of using function decorators
 @dlio_log.log
 def log_events(index):
     sleep(1)

 # Example of function spawning and implicit I/O calls
 def posix_calls(val):
     index, is_spawn = val
     path = f"{cwd}/data/demofile{index}.txt"
     f = open(path, "w+")
     f.write("Now the file has more content!")
     f.close()
     if is_spawn:
         print(f"Calling spawn on {index} with pid {os.getpid()}")
         log_inst.finalize() # This need to be called to correctly finalize DLIO Profiler.
     else:
         print(f"Not calling spawn on {index} with pid {os.getpid()}")

 # NPZ calls internally calls POSIX calls.
 def npz_calls(index):
     # print(f"{cwd}/data/demofile2.npz")
     path = f"{cwd}/data/demofile{index}.npz"
     if os.path.exists(path):
         os.remove(path)
     records = np.random.randint(255, size=(8, 8, 1024), dtype=np.uint8)
     record_labels = [0] * 1024
     np.savez(path, x=records, y=record_labels)

 def main():
     log_events(0)
     npz_calls(1)
     with get_context('spawn').Pool(1, initializer=init) as pool:
         pool.map(posix_calls, ((2, True),))
     log_inst.finalize()


 if __name__ == "__main__":
     main()

For this example, use LD_PRELOAD the executable with libdlio_profiler_preload.soat runtime. As the DLIO_PROFILER_CPP_INIT do not pass log file or data dir, we need to set DLIO_PROFILER_LOG_FILE and DLIO_PROFILER_DATA_DIR. By default the DLIO Profiler mode is set to FUNCTION. Example of running this configurations are:

 # the process id, app_name and .pfw will be appended by the profiler for each app and process.
 # name of final log file is ~/log_file-<APP_NAME>-<PID>.pfw
 DLIO_PROFILER_LOG_FILE=~/log_file
 # Colon separated paths for including for profiler
 DLIO_PROFILER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset
 # Set the mode to PRELOAD
 export DLIO_PROFILER_INIT=PRELOAD
 # Enable profiler
 DLIO_PROFILER_ENABLE=1

Resnet50 with pytorch and torchvision example from ALCF Polaris:¶

Create a separate conda environment for the application and install dlio-profiler

  #!/bin/bash +x
  set -e
  set -x
  export MODULEPATH=/soft/modulefiles/conda/:$MODULEPATH
  module load 2023-10-04  # This is the latest conda module on Polaris

  export ML_ENV=$PWD/PolarisAT/conda-envs/ml_workload_latest_conda_2 # Please change the following path accordingly

  if [[ -e $ML_ENV ]]; then
      conda activate $ML_ENV
  else
      conda create  -p $ML_ENV --clone  /soft/datascience/conda/2023-10-04/mconda3/
      conda activate $ML_ENV
      yes | MPICC="cc -shared -target-accel=nvidia80" pip install --force-reinstall --no-cache-dir --no-binary=mpi4py mpi4py
      yes | pip install --no-cache-dir git+https://github.com/hariharan-devarajan/dlio-profiler.git
      pip uninstall -y torch horovod
      yes | pip install --no-cache-dir horovod
      #INSTALL OTHER MISSING FILES
  fi

Since, torchvision.datasets.ImageFolder spawns separate python processes to help the parallel data loading in torch, we will be using the HYBRID MODE of the DLIO Profiler (e.g., see Python Hybrid mode), so that the application can use both APP and PRELOAD Mode to log I/O from all dynamically spawned processes and function profiling from application.

The following dlio_profiler code is added to profile the application at the function level. Note: dlio-profiler python level log file location is provided inside the python code in the dlio_logger.initialize_log() function and the POSIX or STDIO calls level log file location is provided in the job scirpt environment variable DLIO_PROFILER_LOG_FILE

  ...
  # From the preamble
  from dlio_profiler.logger import dlio_logger as logger, fn_interceptor as dlp_event_logging
  dlp_pid=os.getpid()
  log_inst=logger.initialize_log(f"./resnet50/dlio_log_py_level-{dlp_pid}.pfw", "", dlp_pid)
  compute_dlp = dlp_event_logging("Compute")
  io_dlp = dlp_event_logging("IO", name="real_IO")
  ...
  # From the train() function
  for i, (images, target) in io_dlp.iter(enumerate(train_loader)):
        with dlp_event_logging("communication-except-io", name="cpu-gpu-transfer", step=i, epoch=epoch) as transfer:
            images = images.to(device)
            target = target.to(device)
        with dlp_event_logging("compute", name="model-compute-forward-prop", step=i, epoch=epoch) as compute:
            output = model(images)
            loss = criterion(output, target)
        with dlp_event_logging("compute", name="model-compute-backward-prop", step=i, epoch=epoch) as compute:
            acc1, acc5 = accuracy(output, target, topk=(1, 5))
            losses.update(loss.item(), images.size(0))
            top1.update(acc1[0], images.size(0))
            top5.update(acc5[0], images.size(0))

  ...
  # At the end of main function
  log_inst.finalize()

Job submition script

  export MODULEPATH=/soft/modulefiles/conda/:$MODULEPATH
  module load 2023-10-04
  conda activate./dlio_ml_workloads/PolarisAT/conda-envs/ml_workload_latest_conda

  export LD_LIBRARY_PATH=$env_path/lib/:$LD_LIBRARY_PATH
  export DLIO_PROFILER_LOG_LEVEL=ERROR
  export DLIO_PROFILER_ENABLE=1
  export DLIO_PROFILER_INC_METADATA=1
  export DLIO_PROFILER_INIT=PRELOAD
  export DLIO_PROFILER_DATA_DIR=./resnet_original_data #Path to the orignal resnet 50 dataset
  export DLIO_PROFILER_LOG_FILE=./dlio_log_posix_level.pfw

  LD_PRELOAD=./dlio_ml_workloads/PolarisAT/conda-envs/ml_workload_latest_conda/lib/python*/site-packages/dlio_profiler/lib/libdlio_profiler_preload.so aprun -n 4 -N 4 python resnet_hvd_dlio.py --batch-size 64 --epochs 1 > dlio_log 2>&1

  cat *.pfw > combined_logs.pfw # To combine to a single pfw file.

Integrated Applications¶

Here is the list applications that currently use DLIO Profiler.

DLIO Benchmark
MuMMI
Resnet50 with pytorch and torchvision

Example Chrome Tracing Plots¶

Example of Unet3D application with DLIO Benchmark. This trace shows the first few steps of the benchmark. Here, we can see that we can get application level calls (e.g., train and TorchDataset) as well as low-level I/O calls (dark green color).