rapids-cmake can generate pinned versions file

cmake-format-rapids-cmake.json

@@ -76,10 +76,21 @@
           "PATCH_COMMAND": "1"
         }
       },
-      "rapids_cpm_init": {
+      "rapids_cpm_generate_pinned_versions": {
         "pargs": {
           "nargs": 0
         },
+        "kwargs": {
+          "OUTPUT": 1
+        }
+      },
+      "rapids_cpm_init": {
+        "pargs": {
+          "nargs": 0,
+          "flags": [
+            "GENERATE_PINNED_VERSIONS"
+          ]
+        },
         "kwargs": {
           "OVERRIDE": 1
         }

docs/api.rst

@@ -33,6 +33,7 @@ tracking of these dependencies for correct export support.
 
    /command/rapids_cpm_init
    /command/rapids_cpm_find
+   /command/rapids_cpm_generate_pinned_versions
    /command/rapids_cpm_package_override
 
 .. _`cpm_pre-configured_packages`:

docs/command/rapids_cpm_generate_pinned_versions.rst

@@ -0,0 +1 @@
+.. cmake-module:: ../../rapids-cmake/cpm/generate_pinned_versions.cmake

rapids-cmake/cpm/detail/pinning_root_dir_hook.cmake

@@ -0,0 +1,31 @@
+#=============================================================================
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+include_guard(GLOBAL)
+
+# Make sure we always have CMake 3.23 policies when executing this file since we can be executing in
+# directories of users of rapids-cmake which have a lower minimum cmake version and therefore
+# different policies
+#
+cmake_policy(PUSH)
+cmake_policy(VERSION 3.23)
+
+# Include the needed functions that write out the the pinned versions file
+include("${rapids-cmake-dir}/cpm/detail/pinning_write_file.cmake")
+
+# Compute and write out the pinned versions file
+rapids_cpm_pinning_write_file()
+
+cmake_policy(POP)

rapids-cmake/cpm/detail/pinning_write_file.cmake

@@ -0,0 +1,206 @@
+#=============================================================================
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+include_guard(GLOBAL)
+
+#[=======================================================================[.rst:
+rapids_cpm_pinning_extract_source_git_info
+------------------------------------------
+
+.. versionadded:: v24.04.00
+
+
+#]=======================================================================]
+function(rapids_cpm_pinning_extract_source_git_info package git_url_var git_sha_var)
+  set(source_dir "${CPM_PACKAGE_${package}_SOURCE_DIR}")
+  set(_RAPIDS_URL)
+  set(_RAPIDS_SHA)
+  if(EXISTS "${source_dir}")
+    execute_process(COMMAND ${GIT_EXECUTABLE} ls-remote --get-url
+                    WORKING_DIRECTORY ${source_dir}
+                    ERROR_QUIET
+                    OUTPUT_VARIABLE _RAPIDS_URL
+                    OUTPUT_STRIP_TRAILING_WHITESPACE)
+    # Need to handle when we have applied N patch sets to the git repo and therefore the latest
+    # commit is just local
+    #
+    # Find all commits on our branch back to the common parent ( what we cloned )
+    #
+    execute_process(COMMAND ${GIT_EXECUTABLE} show-branch --current --sha1-name
+                    WORKING_DIRECTORY ${source_dir}
+                    ERROR_QUIET
+                    OUTPUT_VARIABLE _rapids_commit_stack
+                    OUTPUT_STRIP_TRAILING_WHITESPACE)
+    # The last entry in the output that has "* [" is our commit
+    #
+    # Find that line and convert the `* [short-sha1] Commit Message` to a list that is ` *
+    # ;short-sha1;Commit Message` and extract the short sha1
+    string(FIND "${_rapids_commit_stack}" "* [" position REVERSE)
+    if(position LESS 0)
+      # No changes to the repo so use the `HEAD` keyword
+      set(short_sha HEAD)
+    else()
+      string(SUBSTRING "${_rapids_commit_stack}" ${position} -1 _rapids_commit_stack)
+      string(REGEX REPLACE "(\\[|\\])" ";" _rapids_commit_stack "${_rapids_commit_stack}")
+      list(GET _rapids_commit_stack 1 short_sha)
+    endif()
+
+    # Convert from the short sha1 ( could be keyword `HEAD` ) to a full SHA1
+    execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse ${short_sha}
+                    WORKING_DIRECTORY ${source_dir}
+                    ERROR_QUIET
+                    OUTPUT_VARIABLE _RAPIDS_SHA
+                    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  endif()
+  # Only set the provided variables if we extracted the information
+  if(_RAPIDS_URL)
+    set(${git_url_var} "${_RAPIDS_URL}" PARENT_SCOPE)
+  endif()
+  if(_RAPIDS_SHA)
+    set(${git_sha_var} "${_RAPIDS_SHA}" PARENT_SCOPE)
+  endif()
+
+endfunction()
+
+#[=======================================================================[.rst:
+rapids_cpm_pinning_create_and_set_member
+----------------------------------------
+
+.. versionadded:: v24.04.00
+
+Insert the given json key value pair into the provided json data variable
+
+#]=======================================================================]
+function(rapids_cpm_pinning_create_and_set_member json_blob_var key value)
+
+  # Identify special values types that shouldn't be treated as a string
+  # https://gitlab.kitware.com/cmake/cmake/-/issues/25716
+  if(value MATCHES "(^true$|^false$|^null$|^\\{|^\\[)")
+    # value is a json type that doesn't need quotes
+    string(JSON json_blob ERROR_VARIABLE err_var SET "${${json_blob_var}}" ${key} ${value})
+  else()
+    # We need to quote 'value' so that it is a valid string json element.
+    string(JSON json_blob ERROR_VARIABLE err_var SET "${${json_blob_var}}" ${key} "\"${value}\"")
+  endif()
+  set(${json_blob_var} "${json_blob}" PARENT_SCOPE)
+endfunction()
+
+#[=======================================================================[.rst:
+rapids_cpm_pinning_add_json_entry
+---------------------------------
+
+.. versionadded:: v24.04.00
+
+#]=======================================================================]
+function(rapids_cpm_pinning_add_json_entry json_var package_name url_var sha_var)
+  include("${rapids-cmake-dir}/cpm/detail/get_default_json.cmake")
+  include("${rapids-cmake-dir}/cpm/detail/get_override_json.cmake")
+  get_default_json(${package_name} json_data)
+  get_override_json(${package_name} override_json_data)
+
+  set(url_string)
+  set(sha_string)
+  if(${url_var})
+    string(CONFIGURE [=["git_url": "${${url_var}}",]=] url_string)
+  endif()
+  if(${sha_var})
+    string(CONFIGURE [=["git_tag": "${${sha_var}}",]=] sha_string)
+  endif()
+  # We start with a default template, and only add members that don't exist
+  string(CONFIGURE [=[{
+  "version": "${CPM_PACKAGE_${package_name}_VERSION}",
+  ${url_string}
+  ${sha_string}
+  "git_shallow": false,
+  "always_download": true
+  }]=]
+                   pinned_json_entry)
+
+  foreach(data IN LISTS override_json_data json_data)
+    if(NOT data)
+      # Need to handle both json_data and the override being empty
+      continue()
+    endif()
+    string(JSON entry_count LENGTH "${data}")
+    math(EXPR entry_count "${entry_count} - 1")
+    # cmake-lint: disable=E1120
+    foreach(index RANGE ${entry_count})
+      string(JSON member MEMBER "${data}" ${index})
+      string(JSON existing_value ERROR_VARIABLE dont_have GET "${pinned_json_entry}" ${member})
+      if(dont_have)
+        string(JSON value GET "${data}" ${member})
+        rapids_cpm_pinning_create_and_set_member(pinned_json_entry ${member} ${value})
+      endif()
+    endforeach()
+  endforeach()
+  set(${json_var} "\"${package_name}\": ${pinned_json_entry}" PARENT_SCOPE)
+endfunction()
+
+#[=======================================================================[.rst:
+rapids_cpm_pinning_write_file
+-----------------------------
+
+.. versionadded:: v24.04.00
+
+This function will write out the pinned version info to the provided files when we are in the root
+CMakeLists.txt
+#]=======================================================================]
+function(rapids_cpm_pinning_write_file)
+
+  find_package(Git QUIET REQUIRED)
+
+  set(_rapids_json
+      [=[
+{
+"root": {
+"packages": {
+]=])
+
+  list(POP_BACK CPM_PACKAGES last_package)
+  foreach(package IN LISTS CPM_PACKAGES last_package)
+    # Clear variables so we don't re-use them between packages when one package doesn't have a git
+    # url or sha
+    set(git_url)
+    set(git_sha)
+    set(not_last_package TRUE)
+    if(package STREQUAL last_package)
+      set(not_last_package FALSE)
+    endif()
+    rapids_cpm_pinning_extract_source_git_info(${package} git_url git_sha)
+    rapids_cpm_pinning_add_json_entry(_rapids_entry ${package} git_url git_sha)
+    if(not_last_package)
+      string(APPEND _rapids_entry [=[,
+]=])
+    else()
+      string(APPEND _rapids_entry [=[
+]=])
+    endif()
+    string(APPEND _rapids_json "${_rapids_entry}")
+  endforeach()
+
+  set(post_amble [=[
+}}}]=])
+  string(APPEND _rapids_json "${post_amble}")
+
+  # We extract everything out of the fake `root` element so that we get a pretty JSON format from
+  # CMake.
+  string(JSON _rapids_json GET "${_rapids_json}" root)
+
+  get_property(write_paths GLOBAL PROPERTY rapids_cpm_generate_pin_files)
+  foreach(path IN LISTS write_paths)
+    file(WRITE "${path}" "${_rapids_json}")
+  endforeach()
+
+endfunction()

rapids-cmake/cpm/generate_pinned_versions.cmake

@@ -0,0 +1,107 @@
+#=============================================================================
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+include_guard(GLOBAL)
+
+#[=======================================================================[.rst:
+rapids_cpm_generate_pinned_versions
+-----------------------------------
+
+.. versionadded:: v24.04.00
+
+Generate a json file with all dependencies with pinned version values
+
+.. code-block:: cmake
+
+  rapids_cpm_generate_pinned_versions( OUTPUT <json_verions_output_path> )
+
+Generates a json file with all `CPM` dependencies with pinned version values.
+This allows for subsequent reproducible builds using the exact same state.
+
+The rapids-cmake default `versions.json` uses branch names or git tag names
+for dependencies. This is done so that projects can 'live at head' of dependencies.
+By using :cmake:command:`rapids_cpm_package_override` a project can specify a custom
+`versions.json` that specifies exact git SHA's so that projects have reproducible builds.
+
+:cmake:command:`rapids_cpm_generate_pinned_versions` can be used to transform a set of
+rapids-cmake dependencies from branch names to pinned values. This can be used in subsequent
+builds, e.g:
+
+  1. Have CI run with `versions.json` which tracks dependency by branch name
+  2. Store the generated pinned `versions.json` from the CI builds
+  3. If build is good, create the release branch and commit the generated pinned `versions.json`
+     to have reproducible builds for that release
+
+``OUTPUT``
+Specify a file path where the pinned versions information will be written. Can be called multiple
+times and each unique path will be written to.
+
+The generated json file will have the following entries for each package:
+
+.. code-block:: json
+
+  {
+    "version": "<CPM_PACKAGE_<package_name>_VERSION>",
+    "git_url": "<deduced>",
+    "git_tag": "<deduced>",
+    "git_shallow": false,
+    "always_download": true
+  }
+
+
+If the original package (or override) also had any `patches`, or `proprietary_binary`
+fields those will be propagated to the generated entry.
+
+.. note::
+  The git SHA1 computed for each package is found by finding the most recent
+  commit that can be cloned from the url.
+
+  This means that for proper reproducible builds, all patches must be encapsulated
+  in the input json files or as CPM `PATCH_COMMAND`.
+
+#]=======================================================================]
+function(rapids_cpm_generate_pinned_versions)
+  list(APPEND CMAKE_MESSAGE_CONTEXT "rapids.cpm.generate_pinned_versions")
+
+  set(_rapids_options)
+  set(_rapids_one_value OUTPUT)
+  set(_rapids_multi_value)
+  cmake_parse_arguments(_RAPIDS "${_rapids_options}" "${_rapids_one_value}"
+                        "${_rapids_multi_value}" ${ARGN})
+
+  if(NOT _RAPIDS_OUTPUT)
+    message(FATAL_ERROR "rapids_cpm_generate_pinned_versions requires an `OUTPUT` argument")
+  endif()
+
+  find_package(Git QUIET)
+  if(NOT Git_FOUND)
+    message(FATAL_ERROR "rapids_cpm_generate_pinned_versions requires 'git' to exist")
+  endif()
+
+  # Append the requested write path for `detail/write_pinned_versions.cmake`
+  set_property(GLOBAL APPEND PROPERTY rapids_cpm_generate_pin_files "${_RAPIDS_OUTPUT}")
+
+  get_property(already_hooked GLOBAL PROPERTY rapids_cpm_generate_pin_hook SET)
+  if(NOT already_hooked)
+    # install a hook that writes out the pinned versions at the end of the root level CMakeLists.txt
+    # execution so we get all CPM packages added. Plus we can compute the paths once, and write out
+    # `N` times if needed
+    set(root_dir "${${CMAKE_PROJECT_NAME}_SOURCE_DIR}")
+    cmake_language(DEFER DIRECTORY ${root_dir} ID rapids_cpm_generate_pinned_versions CALL include
+                   "${rapids-cmake-dir}/cpm/detail/pinning_root_dir_hook.cmake")
+    set_property(GLOBAL PROPERTY rapids_cpm_generate_pin_hook "ON")
+  endif()
+
+endfunction()