From f775d32b0137fabfef6067089b0e51bf4a92705b Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 16:18:59 +0200 Subject: [PATCH 01/16] bump: try `deepspeed >=0.14.1,<=0.15.0` --- requirements/fabric/strategies.txt | 2 +- requirements/pytorch/strategies.txt | 2 +- src/lightning/fabric/strategies/deepspeed.py | 6 +----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/requirements/fabric/strategies.txt b/requirements/fabric/strategies.txt index bea30b37fa5f8..7856db1df2eec 100644 --- a/requirements/fabric/strategies.txt +++ b/requirements/fabric/strategies.txt @@ -5,5 +5,5 @@ # note: is a bug around 0.10 with `MPS_Accelerator must implement all abstract methods` # shall be resolved by https://github.com/microsoft/DeepSpeed/issues/4372 -deepspeed >=0.9.3, <=0.9.3; platform_system != "Windows" and platform_system != "Darwin" # strict +deepspeed >=0.14.1,<=0.15.0; platform_system != "Windows" and platform_system != "Darwin" # strict bitsandbytes >=0.45.2,<0.47.0; platform_system != "Darwin" diff --git a/requirements/pytorch/strategies.txt b/requirements/pytorch/strategies.txt index 1f7296798b551..89392d6006d38 100644 --- a/requirements/pytorch/strategies.txt +++ b/requirements/pytorch/strategies.txt @@ -3,4 +3,4 @@ # note: is a bug around 0.10 with `MPS_Accelerator must implement all abstract methods` # shall be resolved by https://github.com/microsoft/DeepSpeed/issues/4372 -deepspeed >=0.9.3, <=0.9.3; platform_system != "Windows" and platform_system != "Darwin" # strict +deepspeed >=0.14.1,<=0.15.0; platform_system != "Windows" and platform_system != "Darwin" # strict diff --git a/src/lightning/fabric/strategies/deepspeed.py b/src/lightning/fabric/strategies/deepspeed.py index 48333455240cf..c11ae8589d1ff 100644 --- a/src/lightning/fabric/strategies/deepspeed.py +++ b/src/lightning/fabric/strategies/deepspeed.py @@ -47,7 +47,6 @@ from torch.optim.lr_scheduler import _LRScheduler _DEEPSPEED_AVAILABLE = RequirementCache("deepspeed") -_DEEPSPEED_GREATER_EQUAL_0_14_1 = RequirementCache("deepspeed>=0.14.1") # TODO(fabric): Links in the docstrings to PL-specific deepspeed user docs need to be replaced. @@ -503,10 +502,7 @@ def load_checkpoint( ) engine = engines[0] - if _DEEPSPEED_GREATER_EQUAL_0_14_1: - from deepspeed.runtime.base_optimizer import DeepSpeedOptimizer - else: - from deepspeed.runtime import DeepSpeedOptimizer + from deepspeed.runtime.base_optimizer import DeepSpeedOptimizer optimzer_state_requested = any(isinstance(item, (Optimizer, DeepSpeedOptimizer)) for item in state.values()) From 311982280358817a9bb7f160b9bfe735f2621a9a Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 18:24:31 +0200 Subject: [PATCH 02/16] drop from oldest --- .actions/assistant.py | 26 -------------------------- .azure/gpu-tests-fabric.yml | 3 +++ .azure/gpu-tests-pytorch.yml | 3 +++ 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 7b2d49423d622..f8cda67a74c92 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -341,32 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N class AssistantCLI: - @staticmethod - def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None: - """Remove some packages from given requirement files.""" - if isinstance(req_files, str): - req_files = [req_files] - for req in req_files: - AssistantCLI._prune_packages(req, packages) - - @staticmethod - def _prune_packages(req_file: str, packages: Sequence[str]) -> None: - """Remove some packages from given requirement files.""" - path = Path(req_file) - assert path.exists() - text = path.read_text() - lines = text.splitlines() - final = [] - for line in lines: - ln_ = line.strip() - if not ln_ or ln_.startswith("#"): - final.append(line) - continue - req = list(_parse_requirements([ln_]))[0] - if req.name not in packages: - final.append(line) - print(final) - path.write_text("\n".join(final) + "\n") @staticmethod def _replace_min(fname: str) -> None: diff --git a/.azure/gpu-tests-fabric.yml b/.azure/gpu-tests-fabric.yml index c2d492e5b9564..21bed58d6a869 100644 --- a/.azure/gpu-tests-fabric.yml +++ b/.azure/gpu-tests-fabric.yml @@ -101,6 +101,9 @@ jobs: - bash: | python .actions/assistant.py replace_oldest_ver pip install "cython<3.0" wheel # for compatibility + pip install -U "lightning-utilities[cli]" + # drop deepspeed since it is not supported by our minimal Torch requirements + python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/fabric/strategies.txt condition: contains(variables['Agent.JobName'], 'oldest') displayName: "setting oldest dependencies" diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 16ced045ddade..e0411278b6215 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -105,6 +105,9 @@ jobs: - bash: | python .actions/assistant.py replace_oldest_ver pip install "cython<3.0" wheel # for compatibility + pip install -U "lightning-utilities[cli]" + # drop deepspeed since it is not supported by our minimal Torch requirements + python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/pytorch/strategies.txt condition: contains(variables['Agent.JobName'], 'oldest') displayName: "setting oldest dependencies" From a5fc71f9397cc0343fbf895187bcc426b3b43e44 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 16:24:56 +0000 Subject: [PATCH 03/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .actions/assistant.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index f8cda67a74c92..a49041932a2df 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -341,7 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N class AssistantCLI: - @staticmethod def _replace_min(fname: str) -> None: with open(fname, encoding="utf-8") as fopen: From a83102c1b705a866363abe7a564627abe4520ec0 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 18:34:12 +0200 Subject: [PATCH 04/16] docker --- dockers/release/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile index c7a7a1e0c8470..6e660e865caa0 100644 --- a/dockers/release/Dockerfile +++ b/dockers/release/Dockerfile @@ -40,6 +40,12 @@ RUN \ # otherwise there is collision with folder name and pkg name on Pypi cd pytorch-lightning && \ pip install setuptools==75.6.0 && \ + pip install -U "lightning-utilities[cli]" && \ + # drop deepspeed since it is not supported by our minimal Torch requirements \ + if [[ "$PYTORCH_VERSION" =~ ^(2\.1|2\.2|2\.3|2\.4)$ ]]; then \ + python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/fabric/strategies.txt ; \ + python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/pytorch/strategies.txt ; \ + fi && \ PACKAGE_NAME=lightning pip install '.[extra,loggers,strategies]' --no-cache-dir && \ PACKAGE_NAME=pytorch pip install '.[extra,loggers,strategies]' --no-cache-dir && \ cd .. && \ From 5bd96ecb4e23d7bdf7970b2208be53f5268a10d3 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 18:59:31 +0200 Subject: [PATCH 05/16] bash --- dockers/release/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile index 6e660e865caa0..f920893befe0b 100644 --- a/dockers/release/Dockerfile +++ b/dockers/release/Dockerfile @@ -23,6 +23,7 @@ LABEL maintainer="Lightning-AI " ARG LIGHTNING_VERSION="" COPY ./ /home/pytorch-lightning/ +SHELL ["/bin/bash", "-c"] # install dependencies RUN \ @@ -39,7 +40,7 @@ RUN \ fi && \ # otherwise there is collision with folder name and pkg name on Pypi cd pytorch-lightning && \ - pip install setuptools==75.6.0 && \ + # pip install setuptools==75.6.0 && \ pip install -U "lightning-utilities[cli]" && \ # drop deepspeed since it is not supported by our minimal Torch requirements \ if [[ "$PYTORCH_VERSION" =~ ^(2\.1|2\.2|2\.3|2\.4)$ ]]; then \ From 64e6cc8aaa7d90366bbcf0c5c00de96d0f5de14a Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 19:19:24 +0200 Subject: [PATCH 06/16] echo --- dockers/release/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile index f920893befe0b..406aacb9eac8c 100644 --- a/dockers/release/Dockerfile +++ b/dockers/release/Dockerfile @@ -23,7 +23,6 @@ LABEL maintainer="Lightning-AI " ARG LIGHTNING_VERSION="" COPY ./ /home/pytorch-lightning/ -SHELL ["/bin/bash", "-c"] # install dependencies RUN \ @@ -43,6 +42,7 @@ RUN \ # pip install setuptools==75.6.0 && \ pip install -U "lightning-utilities[cli]" && \ # drop deepspeed since it is not supported by our minimal Torch requirements \ + echo "PYTORCH_VERSION is: '$PYTORCH_VERSION'" && \ if [[ "$PYTORCH_VERSION" =~ ^(2\.1|2\.2|2\.3|2\.4)$ ]]; then \ python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/fabric/strategies.txt ; \ python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/pytorch/strategies.txt ; \ From 17328efd0a2b0603d5838e5252d309ddaa69b246 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 19:22:29 +0200 Subject: [PATCH 07/16] arg --- dockers/release/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile index 406aacb9eac8c..b80c23dfc73f3 100644 --- a/dockers/release/Dockerfile +++ b/dockers/release/Dockerfile @@ -21,6 +21,7 @@ FROM pytorchlightning/pytorch_lightning:base-cuda${CUDA_VERSION}-py${PYTHON_VERS LABEL maintainer="Lightning-AI " ARG LIGHTNING_VERSION="" +ARG PYTORCH_VERSION COPY ./ /home/pytorch-lightning/ From 8abb07faa06f2d3fbff3abd17b205d3eb059f5d5 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 19:29:52 +0200 Subject: [PATCH 08/16] flaky test_memory_sharing_disabled --- .../strategies/launchers/test_multiprocessing_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests_fabric/strategies/launchers/test_multiprocessing_integration.py b/tests/tests_fabric/strategies/launchers/test_multiprocessing_integration.py index 2abfe73c92dec..1bb86d797a02f 100644 --- a/tests/tests_fabric/strategies/launchers/test_multiprocessing_integration.py +++ b/tests/tests_fabric/strategies/launchers/test_multiprocessing_integration.py @@ -30,6 +30,7 @@ def __init__(self): @RunIf(skip_windows=True) +@pytest.mark.flaky(reruns=3) @pytest.mark.parametrize("strategy", ["ddp_spawn", "ddp_fork"]) def test_memory_sharing_disabled(strategy): """Test that the multiprocessing launcher disables memory sharing on model parameters and buffers to avoid race From a30d5bf327560b49ec067e55744c6733e7dfc66f Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 19:49:23 +0200 Subject: [PATCH 09/16] pip uninstall -y deepspeed --- .azure/gpu-tests-fabric.yml | 2 ++ .azure/gpu-tests-pytorch.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.azure/gpu-tests-fabric.yml b/.azure/gpu-tests-fabric.yml index 21bed58d6a869..322b9fa3a5cdb 100644 --- a/.azure/gpu-tests-fabric.yml +++ b/.azure/gpu-tests-fabric.yml @@ -104,6 +104,8 @@ jobs: pip install -U "lightning-utilities[cli]" # drop deepspeed since it is not supported by our minimal Torch requirements python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/fabric/strategies.txt + # uninstall deepspeed since some older docker images have it pre-installed + pip uninstall -y deepspeed condition: contains(variables['Agent.JobName'], 'oldest') displayName: "setting oldest dependencies" diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index e0411278b6215..91616348b1217 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -108,6 +108,8 @@ jobs: pip install -U "lightning-utilities[cli]" # drop deepspeed since it is not supported by our minimal Torch requirements python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/pytorch/strategies.txt + # uninstall deepspeed since some older docker images have it pre-installed + pip uninstall -y deepspeed condition: contains(variables['Agent.JobName'], 'oldest') displayName: "setting oldest dependencies" From b73c4cb3e0e6fa27d1e9b03ded11917124f7188c Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 20:15:24 +0200 Subject: [PATCH 10/16] error::DeprecationWarning --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a63da5f246392..3cd199e61c16f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -179,7 +179,8 @@ markers = [ "cloud: Run the cloud tests for example", ] filterwarnings = [ - "error::FutureWarning", + "error::DeprecationWarning", + # "error::FutureWarning", "ignore::FutureWarning:onnxscript", # Temporary ignore until onnxscript is updated ] xfail_strict = true From 302447134134146fbafb3469d54d6cf647eb54c5 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 20:21:05 +0200 Subject: [PATCH 11/16] warns --- tests/tests_pytorch/utilities/test_compile.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/tests_pytorch/utilities/test_compile.py b/tests/tests_pytorch/utilities/test_compile.py index a053c847dfd6c..e1b4949b8701f 100644 --- a/tests/tests_pytorch/utilities/test_compile.py +++ b/tests/tests_pytorch/utilities/test_compile.py @@ -32,7 +32,7 @@ # https://github.com/pytorch/pytorch/issues/95708 @pytest.mark.skipif(sys.platform == "darwin", reason="fatal error: 'omp.h' file not found") -@RunIf(dynamo=True) +@RunIf(dynamo=True, deepspeed=True) @mock.patch("lightning.pytorch.trainer.call._call_and_handle_interrupt") def test_trainer_compiled_model(_, tmp_path, monkeypatch, mps_count_0): trainer_kwargs = { @@ -74,13 +74,7 @@ def test_trainer_compiled_model(_, tmp_path, monkeypatch, mps_count_0): mock_cuda_count(monkeypatch, 2) # TODO: Update deepspeed to avoid deprecation warning for `torch.cuda.amp.custom_fwd` on import - warn_context = ( - pytest.warns(FutureWarning, match="torch.cuda.amp.*is deprecated") - if _TORCH_GREATER_EQUAL_2_4 - else nullcontext() - ) - - with warn_context: + with pytest.warns(FutureWarning, match="torch.cuda.amp.*is deprecated"): trainer = Trainer(strategy="deepspeed", accelerator="cuda", **trainer_kwargs) with pytest.raises(RuntimeError, match="Using a compiled model is incompatible with the current strategy.*"): From 123a12220481882f4b9be99439d69f591878577a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 18:21:33 +0000 Subject: [PATCH 12/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_pytorch/utilities/test_compile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tests_pytorch/utilities/test_compile.py b/tests/tests_pytorch/utilities/test_compile.py index e1b4949b8701f..fb9feec6d27dc 100644 --- a/tests/tests_pytorch/utilities/test_compile.py +++ b/tests/tests_pytorch/utilities/test_compile.py @@ -13,14 +13,13 @@ # limitations under the License. import os import sys -from contextlib import nullcontext from unittest import mock import pytest import torch from lightning_utilities.core.imports import RequirementCache -from lightning.fabric.utilities.imports import _TORCH_GREATER_EQUAL_2_2, _TORCH_GREATER_EQUAL_2_4 +from lightning.fabric.utilities.imports import _TORCH_GREATER_EQUAL_2_2 from lightning.pytorch import LightningModule, Trainer from lightning.pytorch.demos.boring_classes import BoringModel from lightning.pytorch.utilities.compile import from_compiled, to_uncompiled From 1837dbd8b5d3aecf66c2536a31754f710ee6f2ef Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 15 Aug 2025 20:30:07 +0200 Subject: [PATCH 13/16] warns --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3cd199e61c16f..b4d5d0b1638f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -179,8 +179,8 @@ markers = [ "cloud: Run the cloud tests for example", ] filterwarnings = [ - "error::DeprecationWarning", - # "error::FutureWarning", + # "error::DeprecationWarning", + "error::FutureWarning", "ignore::FutureWarning:onnxscript", # Temporary ignore until onnxscript is updated ] xfail_strict = true From 0eea2967a71081d8cc1727dd8d5187c42c2eac68 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Mon, 18 Aug 2025 14:01:17 +0200 Subject: [PATCH 14/16] warn --- tests/tests_pytorch/utilities/test_compile.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/tests_pytorch/utilities/test_compile.py b/tests/tests_pytorch/utilities/test_compile.py index fb9feec6d27dc..f90cd5e3ef3fa 100644 --- a/tests/tests_pytorch/utilities/test_compile.py +++ b/tests/tests_pytorch/utilities/test_compile.py @@ -13,13 +13,14 @@ # limitations under the License. import os import sys +from contextlib import nullcontext from unittest import mock import pytest import torch from lightning_utilities.core.imports import RequirementCache -from lightning.fabric.utilities.imports import _TORCH_GREATER_EQUAL_2_2 +from lightning.fabric.utilities.imports import _TORCH_GREATER_EQUAL_2_2, _TORCH_GREATER_EQUAL_2_4 from lightning.pytorch import LightningModule, Trainer from lightning.pytorch.demos.boring_classes import BoringModel from lightning.pytorch.utilities.compile import from_compiled, to_uncompiled @@ -73,7 +74,13 @@ def test_trainer_compiled_model(_, tmp_path, monkeypatch, mps_count_0): mock_cuda_count(monkeypatch, 2) # TODO: Update deepspeed to avoid deprecation warning for `torch.cuda.amp.custom_fwd` on import - with pytest.warns(FutureWarning, match="torch.cuda.amp.*is deprecated"): + warn_context = ( + pytest.warns(FutureWarning, match="torch.cuda.amp.*is deprecated") + if _TORCH_GREATER_EQUAL_2_4 + else nullcontext() + ) + + with warn_context: trainer = Trainer(strategy="deepspeed", accelerator="cuda", **trainer_kwargs) with pytest.raises(RuntimeError, match="Using a compiled model is incompatible with the current strategy.*"): From fba45865dfc23c5e6072dca882dbd7d433d42bc0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Aug 2025 11:17:31 +0000 Subject: [PATCH 15/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .actions/assistant.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 60e75b143db0f..e54e69e4860e7 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -341,7 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N class AssistantCLI: - @staticmethod def copy_replace_imports( source_dir: str, From 47167b65efcd0214feea1418f7c93506b1ce2d51 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Tue, 19 Aug 2025 13:47:40 +0200 Subject: [PATCH 16/16] update --- .azure/gpu-tests-fabric.yml | 9 +++++---- .azure/gpu-tests-pytorch.yml | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.azure/gpu-tests-fabric.yml b/.azure/gpu-tests-fabric.yml index 557918b90aab7..b2f8ab0447a20 100644 --- a/.azure/gpu-tests-fabric.yml +++ b/.azure/gpu-tests-fabric.yml @@ -99,13 +99,14 @@ jobs: displayName: "Image info & NVIDIA" - bash: | - cd requirements/fabric - pip install -U "lightning-utilities[cli]" - python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']" + set -ex pip install "cython<3.0" wheel # for compatibility pip install -U "lightning-utilities[cli]" + cd requirements/fabric + # replace range by pin minimal requirements + python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']" # drop deepspeed since it is not supported by our minimal Torch requirements - python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/fabric/strategies.txt + python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt # uninstall deepspeed since some older docker images have it pre-installed pip uninstall -y deepspeed condition: contains(variables['Agent.JobName'], 'oldest') diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index ed20024adc088..d3c4951a22336 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -103,13 +103,14 @@ jobs: displayName: "Image info & NVIDIA" - bash: | - cd requirements/pytorch - pip install -U "lightning-utilities[cli]" - python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']" + set -ex pip install "cython<3.0" wheel # for compatibility pip install -U "lightning-utilities[cli]" + cd requirements/pytorch + # replace range by pin minimal requirements + python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']" # drop deepspeed since it is not supported by our minimal Torch requirements - python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/pytorch/strategies.txt + python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt # uninstall deepspeed since some older docker images have it pre-installed pip uninstall -y deepspeed condition: contains(variables['Agent.JobName'], 'oldest')