[rocprofiler-systems]: Add OpenSHMEM API tracing (#3189)

Add optional OpenSHMEM API tracing using the GOTCHA-based backend. Tracing is off by default and must be enabled via config.

Author: sputhala-amd

projects/rocprofiler-systems/CHANGELOG.md

@@ -4,28 +4,27 @@
 
 Full documentation for ROCm Systems Profiler is available at [https://rocm.docs.amd.com/projects/rocprofiler-systems/en/latest/](https://rocm.docs.amd.com/projects/rocprofiler-systems/en/latest/).
 
-## ROCm Systems Profiler 1.5.0 for ROCm x.y.z (unreleased)
-
-### Changed
-
-- Simplify categorizing like pmc_info events by removing the _<idx> from the "symbol" field. ie., "JpegAct_0" -> "JpegAct".
-- Added `libhsa-runtime64.so` and `libomp.so` to the internal library exclusion list for runtime instrumentation to prevent instrumenting of runtime library internals.
-- RCCL implementation refactored with `production_pmc_registrar` for improved testability and separation of concerns.
-- Unsupported RCCL datatypes now gracefully return 0 with `LOG_WARNING` instead of aborting profiler, allowing continued profiling with newer RCCL versions.
+## ROCm Systems Profiler 1.5.0 for ROCm 7.12.0
 
 ### Added
 
 - Per-GPU RCCL communication data counters (Send/Recv) in `rocpd` output with multi-GPU device attribution using `ncclCommCuDevice` API.
 - Presets profiles that configure the rocprofiler-system tools for common profiling scenarios, offering optimized configurations for specific use cases.
 - `rocprof-sys-attach` CLI tool for attaching to and profiling running processes via rocprofiler-sdk rocattach API (experimental).
+- Support for OpenSHMEM API tracing via `ROCPROFSYS_USE_SHMEM=ON` configuration setting.
 
-### Resolved issues
+### Changed
 
-- Fixed an issue where JPEG engine activity PMC events were not being collected for MI35X systems. Only the first 32 JPEG engines were being collected.
+- Simplify categorizing like pmc_info events by removing the _<idx> from the "symbol" field. ie., "JpegAct_0" -> "JpegAct".
+- Added `libhsa-runtime64.so` and `libomp.so` to the internal library exclusion list for runtime instrumentation to prevent instrumenting of runtime library internals.
+- RCCL implementation refactored with `production_pmc_registrar` for improved testability and separation of concerns.
+- Unsupported RCCL datatypes now gracefully return 0 with `LOG_WARNING` instead of aborting profiler, allowing continued profiling with newer RCCL versions.
 
 ### Resolved issues
 
-- Fixed MPI perfetto trace file merging when using trace cache mode with `ROCPROFSYS_PERFETTO_COMBINE_TRACES=ON`. Previously, each MPI rank would produce a separate trace file; now all ranks' traces are correctly merged into a single output file.
+- Fixed an issue where JPEG engine activity PMC events were not being collected for MI35X systems. Only the first 32 JPEG engines were being collected.
+- Fixed MPI perfetto trace file merging when using trace cache mode with `ROCPROFSYS_PERFETTO_COMBINE_TRACES=ON`.
+  Previously, each MPI rank would produce a separate trace file; now all ranks' traces are correctly merged into a single output file.
 
 ## ROCm Systems Profiler 1.4.0 for ROCm 7.11.0
 

projects/rocprofiler-systems/examples/CMakeLists.txt

@@ -90,4 +90,5 @@ add_subdirectory(jpegdecode)
 add_subdirectory(roctx)
 add_subdirectory(thread-limit)
 add_subdirectory(transferBench)
+add_subdirectory(shmem)
 add_subdirectory(scratch-memory)

projects/rocprofiler-systems/examples/shmem/CMakeLists.txt

@@ -0,0 +1,82 @@
+# Copyright (c) Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+cmake_minimum_required(VERSION 3.21 FATAL_ERROR)
+
+if(ROCPROFSYS_DISABLE_EXAMPLES)
+    get_filename_component(_DIR ${CMAKE_CURRENT_LIST_DIR} NAME)
+    if(
+        "rocprofiler-systems-shmem-examples" IN_LIST ROCPROFSYS_DISABLE_EXAMPLES
+        OR ${_DIR} IN_LIST ROCPROFSYS_DISABLE_EXAMPLES
+    )
+        return()
+    endif()
+endif()
+
+# oshcc is the OpenSHMEM C compiler wrapper (e.g. from Open MPI); it provides
+# the correct include paths and libraries for shmem.h and OpenSHMEM runtime.
+# Prefer /usr/bin/oshcc (system package) to avoid version mismatch with oshrun.
+find_program(OSHCC_EXECUTABLE NAMES oshcc PATHS /usr/bin NO_DEFAULT_PATH)
+if(NOT OSHCC_EXECUTABLE)
+    find_program(OSHCC_EXECUTABLE NAMES oshcc)
+endif()
+if(NOT OSHCC_EXECUTABLE)
+    if("${CMAKE_PROJECT_NAME}" STREQUAL "rocprofiler-systems" AND "$ENV{ROCPROFSYS_CI}")
+        set(_MSG_TYPE STATUS) # don't generate warnings during CI
+    else()
+        set(_MSG_TYPE AUTHOR_WARNING)
+    endif()
+    message(
+        ${_MSG_TYPE}
+        "oshcc not found. Skipping SHMEM examples (shmem_hello, shmem_pingpong)."
+    )
+    return()
+else()
+    message(STATUS "oshcc found: ${OSHCC_EXECUTABLE}")
+endif()
+
+# Build with oshcc so shmem.h and OpenSHMEM libs are found. Output to project build root.
+set(_SHMEM_BINDIR ${CMAKE_BINARY_DIR})
+set(_SHMEM_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(_SHMEM_HELLO_EXE ${_SHMEM_BINDIR}/shmem_hello${CMAKE_EXECUTABLE_SUFFIX})
+set(_SHMEM_PINGPONG_EXE ${_SHMEM_BINDIR}/shmem_pingpong${CMAKE_EXECUTABLE_SUFFIX})
+
+add_custom_command(
+    OUTPUT ${_SHMEM_HELLO_EXE}
+    COMMAND ${OSHCC_EXECUTABLE} ${_SHMEM_SRCDIR}/shmem_hello.c -o ${_SHMEM_HELLO_EXE}
+    DEPENDS ${_SHMEM_SRCDIR}/shmem_hello.c
+    COMMENT "Building shmem_hello with oshcc"
+    VERBATIM
+)
+add_custom_target(shmem_hello ALL DEPENDS ${_SHMEM_HELLO_EXE})
+
+add_custom_command(
+    OUTPUT ${_SHMEM_PINGPONG_EXE}
+    COMMAND
+        ${OSHCC_EXECUTABLE} ${_SHMEM_SRCDIR}/shmem_pingpong.c -o ${_SHMEM_PINGPONG_EXE}
+    DEPENDS ${_SHMEM_SRCDIR}/shmem_pingpong.c
+    COMMENT "Building shmem_pingpong with oshcc"
+    VERBATIM
+)
+add_custom_target(shmem_pingpong ALL DEPENDS ${_SHMEM_PINGPONG_EXE})
+
+# Export paths for rocprof-sys-shmem-tests.cmake (same project)
+set(ROCPROFSYS_SHMEM_HELLO_EXE
+    "${_SHMEM_HELLO_EXE}"
+    CACHE INTERNAL
+    "Path to shmem_hello executable"
+)
+set(ROCPROFSYS_SHMEM_PINGPONG_EXE
+    "${_SHMEM_PINGPONG_EXE}"
+    CACHE INTERNAL
+    "Path to shmem_pingpong executable"
+)
+
+if(ROCPROFSYS_INSTALL_EXAMPLES)
+    install(
+        PROGRAMS ${_SHMEM_HELLO_EXE} ${_SHMEM_PINGPONG_EXE}
+        DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/rocprofiler-systems/examples
+        COMPONENT rocprofiler-systems-examples
+        OPTIONAL
+    )
+endif()

projects/rocprofiler-systems/examples/shmem/shmem_hello.c

@@ -0,0 +1,54 @@
+// Copyright (c) Advanced Micro Devices, Inc.
+// SPDX-License-Identifier: MIT
+
+#include <shmem.h>
+#include <stdio.h>
+
+int
+main(void)
+{
+    // 1. Initialize the SHMEM environment
+    shmem_init();
+
+    // 2. Get basic information about my place in the world
+    int me   = shmem_my_pe();  // my Processing Element (like rank)
+    int npes = shmem_n_pes();  // total number of PEs (like size)
+
+    // Simple output from every process
+    printf("Hello from PE %d of %d\n", me, npes);
+
+    // -------------------------------------------------------------------------
+    // A bit more interesting: simple point-to-point communication
+    // -------------------------------------------------------------------------
+
+    // Allocate one integer in the symmetric heap (visible to all PEs)
+    int* value = (int*) shmem_malloc(sizeof(int));
+
+    // Everyone initializes their own slot to their PE number
+    *value = me;
+
+    // Barrier so everyone has written their value
+    shmem_barrier_all();
+
+    // Each PE reads the value from the next PE (with wrap-around)
+    int next_pe = (me + 1) % npes;
+    int received;
+
+    // Blocking get from next PE
+    shmem_int_get(&received, value, 1, next_pe);
+
+    printf("PE %d received value %d from PE %d\n", me, received, next_pe);
+
+    // Optional: make sure remote memory operations are completed
+    shmem_quiet();
+
+    // -------------------------------------------------------------------------
+    // Cleanup
+    // -------------------------------------------------------------------------
+    shmem_free(value);
+
+    // Finalize SHMEM (required in modern versions)
+    shmem_finalize();
+
+    return 0;
+}