From f4092989d8d32fec6092ab45ab57e0c50773aa5c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 2 Jul 2024 15:56:26 -0500 Subject: [PATCH 01/14] split up CUDA-suffixed dependencies in dependencies.yaml --- dependencies.yaml | 93 +++++++++++++++++++++++++++++++------- python/cudf/pyproject.toml | 19 +++++++- 2 files changed, 94 insertions(+), 18 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index e3f8a72e76c..99635b83189 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -342,13 +342,27 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} - packages: &build_python_packages_cu12 + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: - rmm-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} - packages: &build_python_packages_cu11 + - matrix: + cuda: "12.*" + cuda_suffixed: "false" + packages: &build_python_cudf_cu12_unsuffixed + - *rmm_conda + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: - rmm-cu11==24.8.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_conda] } + - matrix: + cuda: "11.*" + cuda_suffixed: "false" + packages: + - *rmm_conda + - {matrix: null, packages: *build_python_cudf_cu12_unsuffixed} libarrow_build: common: - output_types: conda @@ -582,23 +596,42 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - pynvjitlink>=0.0.0a0 + - &pynvjitlink_conda pynvjitlink>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - cubinlinker - - ptxcompiler + - &cubinlinker_conda cubinlinker + - &ptxcompiler_conda ptxcompiler - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - rmm-cu12==24.8.*,>=0.0.0a0 - pynvjitlink-cu12>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "false" + packages: &run_cudf_cu12_unsuffixed + - *rmm_conda + - *pynvjitlink_conda + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - rmm-cu11==24.8.*,>=0.0.0a0 - cubinlinker-cu11 - ptxcompiler-cu11 - - {matrix: null, packages: [cubinlinker, ptxcompiler, *rmm_conda]} + - matrix: + cuda: "11.*" + cuda_suffixed: "false" + packages: + - *cubinlinker_conda + - *ptxcompiler_conda + - *rmm_conda + # when no CUDA version is specified, fall back to + # CUDA 12 dependencies (but without -cuXY suffixes) + - {matrix: null, packages: *run_cudf_cu12_unsuffixed} run_cudf_polars: common: - output_types: [conda, requirements, pyproject] @@ -710,13 +743,27 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - cudf-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "false" + packages: &depends_on_cudf_cu12_unsuffixed + - *cudf_conda + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - cudf-cu11==24.8.*,>=0.0.0a0 - - {matrix: null, packages: [*cudf_conda]} + - matrix: + cuda: "11.*" + cuda_suffixed: "false" + packages: + - *cudf_conda + - {matrix: null, packages: *depends_on_cudf_cu12_unsuffixed} depends_on_cudf_kafka: common: - output_types: conda @@ -731,12 +778,26 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - cudf_kafka-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "false" + packages: + - *cudf_conda + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - cudf_kafka-cu11==24.8.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "false" + packages: + - *cudf_kafka_conda - {matrix: null, packages: [*cudf_kafka_conda]} depends_on_cupy: common: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 20b731624df..77fac9ddcd0 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -19,7 +19,6 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cachetools", - "cubinlinker", "cuda-python>=11.7.1,<12.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", @@ -28,8 +27,8 @@ dependencies = [ "nvtx>=0.2.1", "packaging", "pandas>=2.0,<2.2.3dev0", - "ptxcompiler", "pyarrow>=16.1.0,<16.2.0a0", + "pynvjitlink>=0.0.0a0", "rich", "rmm==24.8.*,>=0.0.0a0", "typing_extensions>=4.0.0", @@ -120,6 +119,22 @@ skip = [ [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" +# This project's dependencies.yaml separates CUDA-suffixed dependencies into 2 lists, +# like this: +# +# rmm-cu12 [cuda_suffixed = "true"] +# rmm [cuda_suffixed = "false"] +# +# So it's possible to separate the choices "use CUDA-suffixed names in wheel metadata" +# and "depend on different sets of libraries based on CUDA version". +# +# Setting matrix selector "cuda_suffixed=true" here tells rapids-build-backend that by default, +# it should produce wheels for redistribution which have CUDA-suffixed dependency names. +# +# This is intended to be overriden with --config-settings passed to 'pip wheel' or 'build' +# in environments which prefer all CUDA-suffix-less dependency names. +# +matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4", "cython>=3.0.3", From ecb59b231f381493bd43176a001131c62d5d5517 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 2 Jul 2024 16:41:35 -0500 Subject: [PATCH 02/14] typo --- python/cudf/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 77fac9ddcd0..08014319d90 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -131,7 +131,7 @@ dependencies-file = "../../dependencies.yaml" # Setting matrix selector "cuda_suffixed=true" here tells rapids-build-backend that by default, # it should produce wheels for redistribution which have CUDA-suffixed dependency names. # -# This is intended to be overriden with --config-settings passed to 'pip wheel' or 'build' +# This is intended to be overridden with --config-settings passed to 'pip wheel' or 'build' # in environments which prefer all CUDA-suffix-less dependency names. # matrix-entry = "cuda_suffixed=true" From 694a07f6e2996d38845c745d3172ca6545aeb9cc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 3 Jul 2024 09:18:32 -0500 Subject: [PATCH 03/14] pynvjitlink is not built from source in devcontainers --- dependencies.yaml | 9 ++++++--- python/cudf/pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 99635b83189..b473fbaf420 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -596,7 +596,7 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - &pynvjitlink_conda pynvjitlink>=0.0.0a0 + - pynvjitlink>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - &cubinlinker_conda cubinlinker @@ -608,13 +608,16 @@ dependencies: cuda_suffixed: "true" packages: - rmm-cu12==24.8.*,>=0.0.0a0 - - pynvjitlink-cu12>=0.0.0a0 + - &pynvjitlink_cu12 pynvjitlink-cu12>=0.0.0a0 - matrix: cuda: "12.*" cuda_suffixed: "false" packages: &run_cudf_cu12_unsuffixed - *rmm_conda - - *pynvjitlink_conda + # for as long as 'pynvjitlink' is not built from source in + # RAPIDS devcontainers / DLFW, it needs to be the fully-suffixed name + # here so it'll be found from package indices + - *pynvjitlink_cu12 - matrix: cuda: "11.*" cuda_suffixed: "true" diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 08014319d90..a91d667081f 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "packaging", "pandas>=2.0,<2.2.3dev0", "pyarrow>=16.1.0,<16.2.0a0", - "pynvjitlink>=0.0.0a0", + "pynvjitlink-cu12>=0.0.0a0", "rich", "rmm==24.8.*,>=0.0.0a0", "typing_extensions>=4.0.0", From 49827812352779979c69533c091434c8d5233113 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 3 Jul 2024 10:31:23 -0500 Subject: [PATCH 04/14] all the projects depending on something suffixed need this matrix entry --- python/cudf_kafka/pyproject.toml | 1 + python/cudf_polars/pyproject.toml | 1 + python/custreamz/pyproject.toml | 1 + python/dask_cudf/pyproject.toml | 1 + 4 files changed, 4 insertions(+) diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 11e18cd4f32..34287c2b0e2 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -100,6 +100,7 @@ regex = "(?P.*)" [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4", "cython>=3.0.3", diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index bf4673fcc50..bda775fe2cf 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -184,3 +184,4 @@ build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" # Pure python disable-cuda = true +matrix-entry = "cuda_suffixed=true" diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 7b99e041b54..70d16c4b07f 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -49,6 +49,7 @@ Homepage = "https://github.com/rapidsai/cudf" [tool.rapids-build-backend] build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" [tool.setuptools] license-files = ["LICENSE"] diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 9b2e3a5a7b1..16e07428d6b 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -58,6 +58,7 @@ Homepage = "https://github.com/rapidsai/cudf" [tool.rapids-build-backend] build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" [tool.setuptools] license-files = ["LICENSE"] From dd5c3e394ebc2a413aa0b9f05b522a7f0d7cf19a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 11:29:11 -0500 Subject: [PATCH 05/14] try overridding matrix entry via containerEnv --- .devcontainer/cuda11.8-pip/devcontainer.json | 3 +++ .devcontainer/cuda12.5-pip/devcontainer.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 4945d6cf753..44f833ed0bc 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -8,6 +8,9 @@ "BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-ubuntu22.04" } }, + "containerEnv": { + "RAPIDS_MATRIX_ENTRY": "cuda=${localEnv:CUDA_VERSION};cuda_suffixed=true", + }, "runArgs": [ "--rm", "--name", diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 026eb540952..9899d4134c0 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -8,6 +8,9 @@ "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.5-ubuntu22.04" } }, + "containerEnv": { + "RAPIDS_MATRIX_ENTRY": "cuda=${localEnv:CUDA_VERSION};cuda_suffixed=true", + }, "runArgs": [ "--rm", "--name", From de682df99b19774598309211e4617a79082fd049 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 11:44:33 -0500 Subject: [PATCH 06/14] fix templating --- .devcontainer/cuda11.8-pip/devcontainer.json | 2 +- .devcontainer/cuda12.5-pip/devcontainer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 44f833ed0bc..301ca9862c9 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -9,7 +9,7 @@ } }, "containerEnv": { - "RAPIDS_MATRIX_ENTRY": "cuda=${localEnv:CUDA_VERSION};cuda_suffixed=true", + "RAPIDS_MATRIX_ENTRY": "cuda=11.8;cuda_suffixed=true" }, "runArgs": [ "--rm", diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 9899d4134c0..de22ca3e4c6 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -9,7 +9,7 @@ } }, "containerEnv": { - "RAPIDS_MATRIX_ENTRY": "cuda=${localEnv:CUDA_VERSION};cuda_suffixed=true", + "RAPIDS_MATRIX_ENTRY": "cuda=12.5;cuda_suffixed=true" }, "runArgs": [ "--rm", From e98f8a193086d34e6b07dfe920e32ec47fbeea37 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 16:18:03 -0500 Subject: [PATCH 07/14] revert some changes --- dependencies.yaml | 14 ++++++-------- python/cudf/pyproject.toml | 18 ++---------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 7e948077542..1f4e575d0d8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -618,7 +618,7 @@ dependencies: - matrix: cuda: "12.*" cuda_suffixed: "false" - packages: &run_cudf_cu12_unsuffixed + packages: - *rmm_conda # for as long as 'pynvjitlink' is not built from source in # RAPIDS devcontainers / DLFW, it needs to be the fully-suffixed name @@ -634,13 +634,11 @@ dependencies: - matrix: cuda: "11.*" cuda_suffixed: "false" - packages: + packages: &run_cudf_cu11_unsuffixed - *cubinlinker_conda - *ptxcompiler_conda - *rmm_conda - # when no CUDA version is specified, fall back to - # CUDA 12 dependencies (but without -cuXY suffixes) - - {matrix: null, packages: *run_cudf_cu12_unsuffixed} + - {matrix: null, packages: *run_cudf_cu11_unsuffixed} run_cudf_polars: common: - output_types: [conda, requirements, pyproject] @@ -760,7 +758,7 @@ dependencies: - matrix: cuda: "12.*" cuda_suffixed: "false" - packages: &depends_on_cudf_cu12_unsuffixed + packages: - *cudf_conda - matrix: cuda: "11.*" @@ -770,9 +768,9 @@ dependencies: - matrix: cuda: "11.*" cuda_suffixed: "false" - packages: + packages: &depends_on_cudf_cu11_unsuffixed - *cudf_conda - - {matrix: null, packages: *depends_on_cudf_cu12_unsuffixed} + - {matrix: null, packages: *depends_on_cudf_cu11_unsuffixed} depends_on_cudf_kafka: common: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index f88df735c1f..30b0f6249f9 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -19,6 +19,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cachetools", + "cubinlinker", "cuda-python>=11.7.1,<12.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", @@ -27,8 +28,8 @@ dependencies = [ "nvtx>=0.2.1", "packaging", "pandas>=2.0,<2.2.3dev0", + "ptxcompiler", "pyarrow>=16.1.0,<16.2.0a0", - "pynvjitlink-cu12>=0.0.0a0", "rich", "rmm==24.8.*,>=0.0.0a0", "typing_extensions>=4.0.0", @@ -119,21 +120,6 @@ skip = [ [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" -# This project's dependencies.yaml separates CUDA-suffixed dependencies into 2 lists, -# like this: -# -# rmm-cu12 [cuda_suffixed = "true"] -# rmm [cuda_suffixed = "false"] -# -# So it's possible to separate the choices "use CUDA-suffixed names in wheel metadata" -# and "depend on different sets of libraries based on CUDA version". -# -# Setting matrix selector "cuda_suffixed=true" here tells rapids-build-backend that by default, -# it should produce wheels for redistribution which have CUDA-suffixed dependency names. -# -# This is intended to be overridden with --config-settings passed to 'pip wheel' or 'build' -# in environments which prefer all CUDA-suffix-less dependency names. -# matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", From 255f529a94803b616f73c899e17df3da9ff54d6c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 23 Jul 2024 13:08:47 -0500 Subject: [PATCH 08/14] no need to override config settings, libraries are installed with --no-build-isolation --no-deps --- .devcontainer/cuda11.8-pip/devcontainer.json | 3 --- .devcontainer/cuda12.5-pip/devcontainer.json | 3 --- 2 files changed, 6 deletions(-) diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 301ca9862c9..4945d6cf753 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -8,9 +8,6 @@ "BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-ubuntu22.04" } }, - "containerEnv": { - "RAPIDS_MATRIX_ENTRY": "cuda=11.8;cuda_suffixed=true" - }, "runArgs": [ "--rm", "--name", diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index de22ca3e4c6..026eb540952 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -8,9 +8,6 @@ "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.5-ubuntu22.04" } }, - "containerEnv": { - "RAPIDS_MATRIX_ENTRY": "cuda=12.5;cuda_suffixed=true" - }, "runArgs": [ "--rm", "--name", From 627d9a3731a55e4dba6c0500ea5ee13cd3602cfe Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 23 Jul 2024 22:43:22 -0700 Subject: [PATCH 09/14] fix comment on pynvjitlink --- dependencies.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 1f4e575d0d8..81c85e0a88c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -620,9 +620,12 @@ dependencies: cuda_suffixed: "false" packages: - *rmm_conda - # for as long as 'pynvjitlink' is not built from source in - # RAPIDS devcontainers / DLFW, it needs to be the fully-suffixed name - # here so it'll be found from package indices + # pynvjitlink still has a '-cu12' suffix here because: + # + # * it's suffixed when built from source in DLFW builds + # * it needs to be 'pip install'-ed from package repositories + # in RAPIDS devcontainer builds + # - *pynvjitlink_cu12 - matrix: cuda: "11.*" From 035213e88c365a318af2062d761994c144f4d186 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 10:37:09 -0500 Subject: [PATCH 10/14] remove pynvjitlink special case --- dependencies.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 81c85e0a88c..85ab8507f9a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -614,19 +614,13 @@ dependencies: cuda_suffixed: "true" packages: - rmm-cu12==24.8.*,>=0.0.0a0 - - &pynvjitlink_cu12 pynvjitlink-cu12>=0.0.0a0 + - pynvjitlink-cu12>=0.0.0a0 - matrix: cuda: "12.*" cuda_suffixed: "false" packages: - *rmm_conda - # pynvjitlink still has a '-cu12' suffix here because: - # - # * it's suffixed when built from source in DLFW builds - # * it needs to be 'pip install'-ed from package repositories - # in RAPIDS devcontainer builds - # - - *pynvjitlink_cu12 + - pynvjitlink>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" From f0f90d063033c202524b02959dbf9bbe41113db7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 12:00:05 -0500 Subject: [PATCH 11/14] consolidate cuda_suffixed=false conditions, stop naming anchors _conda --- dependencies.yaml | 56 +++++++++++------------------------------------ 1 file changed, 13 insertions(+), 43 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 85ab8507f9a..2abbdba6450 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -329,7 +329,7 @@ dependencies: common: - output_types: conda packages: - - &rmm_conda rmm==24.8.*,>=0.0.0a0 + - &rmm_unsuffixed rmm==24.8.*,>=0.0.0a0 - pip - pip: - git+https://github.com/python-streamz/streamz.git@master @@ -348,22 +348,12 @@ dependencies: cuda_suffixed: "true" packages: - rmm-cu12==24.8.*,>=0.0.0a0 - - matrix: - cuda: "12.*" - cuda_suffixed: "false" - packages: &build_python_cudf_cu12_unsuffixed - - *rmm_conda - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - rmm-cu11==24.8.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "false" - packages: - - *rmm_conda - - {matrix: null, packages: *build_python_cudf_cu12_unsuffixed} + - {matrix: null, packages: [*rmm_unsuffixed]} libarrow_build: common: - output_types: conda @@ -581,7 +571,7 @@ dependencies: - typing_extensions>=4.0.0 - output_types: conda packages: - - *rmm_conda + - *rmm_unsuffixed - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -605,8 +595,8 @@ dependencies: - pynvjitlink>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - &cubinlinker_conda cubinlinker - - &ptxcompiler_conda ptxcompiler + - &cubinlinker_unsuffixed cubinlinker + - &ptxcompiler_unsuffixed ptxcompiler - output_types: [requirements, pyproject] matrices: - matrix: @@ -619,7 +609,7 @@ dependencies: cuda: "12.*" cuda_suffixed: "false" packages: - - *rmm_conda + - *rmm_unsuffixed - pynvjitlink>=0.0.0a0 - matrix: cuda: "11.*" @@ -632,9 +622,9 @@ dependencies: cuda: "11.*" cuda_suffixed: "false" packages: &run_cudf_cu11_unsuffixed - - *cubinlinker_conda - - *ptxcompiler_conda - - *rmm_conda + - *cubinlinker_unsuffixed + - *ptxcompiler_unsuffixed + - *rmm_unsuffixed - {matrix: null, packages: *run_cudf_cu11_unsuffixed} run_cudf_polars: common: @@ -737,7 +727,7 @@ dependencies: common: - output_types: conda packages: - - &cudf_conda cudf==24.8.*,>=0.0.0a0 + - &cudf_unsuffixed cudf==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -752,27 +742,17 @@ dependencies: cuda_suffixed: "true" packages: - cudf-cu12==24.8.*,>=0.0.0a0 - - matrix: - cuda: "12.*" - cuda_suffixed: "false" - packages: - - *cudf_conda - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - cudf-cu11==24.8.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "false" - packages: &depends_on_cudf_cu11_unsuffixed - - *cudf_conda - - {matrix: null, packages: *depends_on_cudf_cu11_unsuffixed} + - {matrix: null, packages: [*cudf_unsuffixed]} depends_on_cudf_kafka: common: - output_types: conda packages: - - &cudf_kafka_conda cudf_kafka==24.8.*,>=0.0.0a0 + - &cudf_kafka_unsuffixed cudf_kafka==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -787,22 +767,12 @@ dependencies: cuda_suffixed: "true" packages: - cudf_kafka-cu12==24.8.*,>=0.0.0a0 - - matrix: - cuda: "12.*" - cuda_suffixed: "false" - packages: - - *cudf_conda - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - cudf_kafka-cu11==24.8.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "false" - packages: - - *cudf_kafka_conda - - {matrix: null, packages: [*cudf_kafka_conda]} + - {matrix: null, packages: [*cudf_kafka_unsuffixed]} depends_on_cupy: common: - output_types: conda From 034aac826901da5f310d00a062a3ed4334c98a2d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 12:18:21 -0500 Subject: [PATCH 12/14] one more anchor --- dependencies.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 2abbdba6450..48433d8e5c1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -592,7 +592,7 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - pynvjitlink>=0.0.0a0 + - &pynvjitlink_unsuffixed pynvjitlink>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - &cubinlinker_unsuffixed cubinlinker @@ -610,7 +610,7 @@ dependencies: cuda_suffixed: "false" packages: - *rmm_unsuffixed - - pynvjitlink>=0.0.0a0 + - *pynvjitlink_unsuffixed - matrix: cuda: "11.*" cuda_suffixed: "true" From 2061a24e02f146602f83532baedf0b6fc6924956 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 12:24:17 -0500 Subject: [PATCH 13/14] catch more cases in update-version --- ci/release/update-version.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index f629de64905..a65086fda28 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -68,15 +68,17 @@ done # README.md update sed_runner "s/version == ${CURRENT_SHORT_TAG}/version == ${NEXT_SHORT_TAG}/g" README.md sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" README.md +sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" python/cudf_polars/docs/overview.md # Libcudf examples update sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/versions.cmake # CI files -for FILE in .github/workflows/*.yaml; do +for FILE in .github/workflows/*.yaml .github/workflows/*.yml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE}; done +sed_runner "s/branch-[0-9]+\.[0-9]+/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_cudf_polars.sh # Java files NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}-SNAPSHOT" From 1416fa7b6153d944e95340e6c10a4008d07e6098 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 12:27:41 -0500 Subject: [PATCH 14/14] more update-version changes --- ci/release/update-version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index a65086fda28..ad96aff3930 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -69,6 +69,7 @@ done sed_runner "s/version == ${CURRENT_SHORT_TAG}/version == ${NEXT_SHORT_TAG}/g" README.md sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" README.md sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" python/cudf_polars/docs/overview.md +sed_runner "s/branch-${CURRENT_SHORT_TAG}/branch-${NEXT_SHORT_TAG}/g" python/cudf_polars/docs/overview.md # Libcudf examples update sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/versions.cmake