From 841de2cc899696e537ec827ef1fc7797852fad44 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 8 Feb 2023 09:38:52 +0100 Subject: [PATCH 01/35] consolidate Linux workflows on CPU and GPU --- .github/unittest.sh | 84 ++++++++++++++++++++++++++++ .github/workflows/test-linux-cpu.yml | 57 ------------------- .github/workflows/test-linux-gpu.yml | 61 -------------------- .github/workflows/test-linux.yml | 38 +++++++++++++ 4 files changed, 122 insertions(+), 118 deletions(-) create mode 100755 .github/unittest.sh delete mode 100644 .github/workflows/test-linux-cpu.yml delete mode 100644 .github/workflows/test-linux-gpu.yml create mode 100644 .github/workflows/test-linux.yml diff --git a/.github/unittest.sh b/.github/unittest.sh new file mode 100755 index 00000000000..98889286685 --- /dev/null +++ b/.github/unittest.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo '::group::Prepare conda' +CONDA_PATH=$(which conda) +eval "$(${CONDA_PATH} shell.bash hook)" +# The `setuptools` package installed through `conda` includes a patch that errors if something is installed +# through `setuptools` while the `CONDA_BUILD` environment variable is set. +# https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch +# (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for +# `setuptools` from the `-c defaults` channel) +# Since we aren't using `conda build` here, we unset it to avoid installation problems later +# TODO: investigate where `CONDA_BUILD` is set and maybe fix unset it there +unset CONDA_BUILD +echo '::endgroup::' + +echo '::group::Set PyTorch conda channel' +# TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. +if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + POSTFIX=test +else + POSTFIX=nightly +fi +PYTORCH_CHANNEL=pytorch-"${POSTFIX}" +echo "${PYTORCH_CHANNEL}" +echo '::endgroup::' + +echo '::group::Set PyTorch GPU mutex' +case $GPU_ARCH_TYPE in + cpu) + PYTORCH_MUTEX=cpuonly + ;; + cuda) + PYTORCH_MUTEX="pytorch-cuda=${GPU_ARCH_VERSION}" + ;; + *) + echo "Unknown GPU_ARCH_TYPE=${GPU_ARCH_TYPE}" + exit 1 + ;; +esac +echo "${PYTORCH_MUTEX}" +echo '::endgroup::' + +echo '::group::Create build environment' +conda create \ + --name ci \ + --quiet --yes \ + python="${PYTHON_VERSION}" pip \ + setuptools ninja \ + libpng jpeg \ + Pillow numpy requests +conda activate ci +pip install 'av<10' +echo '::endgroup::' + +echo '::group::Install PyTorch' +conda install \ + --quiet --yes \ + -c "${PYTORCH_CHANNEL}" \ + -c nvidia \ + pytorch \ + "${PYTORCH_MUTEX}" + +if [[ $GPU_ARCH_TYPE = 'cuda' ]]; then + python3 -c "import torch; exit(not torch.cuda.is_available())" +fi +echo '::endgroup::' + +echo '::group::Install TorchVision' +python setup.py develop +echo '::endgroup::' + +echo '::group::Collect PyTorch environment information' +python -m torch.utils.collect_env +echo '::endgroup::' + +echo '::group::Install testing utilities' +pip install --progress-bar=off pytest pytest-mock pytest-cov +echo '::endgroup::' + +echo '::group::Run tests' +pytest --durations=25 +echo '::endgroup::' diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml deleted file mode 100644 index 5dc7550d868..00000000000 --- a/.github/workflows/test-linux-cpu.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Unit-tests on Linux CPU - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -env: - CHANNEL: "nightly" - -jobs: - tests: - strategy: - matrix: - python_version: ["3.8", "3.9", "3.10"] - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: linux.12xlarge - repository: pytorch/vision - script: | - # Mark Build Directory Safe - git config --global --add safe.directory /__w/vision/vision - - # Set up Environment Variables - export PYTHON_VERSION="${{ matrix.python_version }}" - export VERSION="cpu" - export CUDATOOLKIT="cpuonly" - - # Set CHANNEL - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export CHANNEL=test - else - export CHANNEL=nightly - fi - - # Create Conda Env - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy - conda activate /work/ci_env - - # Install PyTorch, Torchvision, and testing libraries - set -ex - conda install \ - --yes \ - -c "pytorch-${CHANNEL}" \ - -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ - "${CUDATOOLKIT}" - python3 setup.py develop - python3 -m pip install pytest pytest-mock 'av<10' - - # Run Tests - python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml deleted file mode 100644 index 831de27e350..00000000000 --- a/.github/workflows/test-linux-gpu.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Unit-tests on Linux GPU - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -env: - CHANNEL: "nightly" - -jobs: - tests: - strategy: - matrix: - python_version: ["3.8"] - cuda_arch_version: ["11.7"] - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: linux.g5.4xlarge.nvidia.gpu - repository: pytorch/vision - gpu-arch-type: cuda - gpu-arch-version: ${{ matrix.cuda_arch_version }} - timeout: 120 - script: | - # Mark Build Directory Safe - git config --global --add safe.directory /__w/vision/vision - - # Set up Environment Variables - export PYTHON_VERSION="${{ matrix.python_version }}" - export VERSION="${{ matrix.cuda_arch_version }}" - export CUDATOOLKIT="pytorch-cuda=${VERSION}" - - # Set CHANNEL - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export CHANNEL=test - else - export CHANNEL=nightly - fi - - # Create Conda Env - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy - conda activate /work/ci_env - - # Install PyTorch, Torchvision, and testing libraries - set -ex - conda install \ - --yes \ - -c "pytorch-${CHANNEL}" \ - -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ - "${CUDATOOLKIT}" - python3 setup.py develop - python3 -m pip install pytest pytest-mock 'av<10' - - # Run Tests - python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml new file mode 100644 index 00000000000..f34ad299483 --- /dev/null +++ b/.github/workflows/test-linux.yml @@ -0,0 +1,38 @@ +name: Unit-tests on Linux + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + tests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + runner: ["linux.12xlarge"] + gpu-arch-type: ["cpu"] + include: + - python-version: 3.8 + runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + timeout: 120 + script: | + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/unittest.sh From 7e9083822061529cf04be34916368eab7d61db57 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 8 Feb 2023 14:42:38 +0100 Subject: [PATCH 02/35] add sleep for ssh --- .github/unittest.sh | 30 ++++++------- .github/workflows/test-linux-gpu.yml | 64 ++++++++++++++++++++++++++++ .github/workflows/test-linux.yml | 2 + 3 files changed, 81 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/test-linux-gpu.yml diff --git a/.github/unittest.sh b/.github/unittest.sh index 98889286685..d64441dbcc9 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -67,18 +67,18 @@ if [[ $GPU_ARCH_TYPE = 'cuda' ]]; then fi echo '::endgroup::' -echo '::group::Install TorchVision' -python setup.py develop -echo '::endgroup::' - -echo '::group::Collect PyTorch environment information' -python -m torch.utils.collect_env -echo '::endgroup::' - -echo '::group::Install testing utilities' -pip install --progress-bar=off pytest pytest-mock pytest-cov -echo '::endgroup::' - -echo '::group::Run tests' -pytest --durations=25 -echo '::endgroup::' +#echo '::group::Install TorchVision' +#python setup.py develop +#echo '::endgroup::' +# +#echo '::group::Collect PyTorch environment information' +#python -m torch.utils.collect_env +#echo '::endgroup::' +# +#echo '::group::Install testing utilities' +#pip install --progress-bar=off pytest pytest-mock pytest-cov +#echo '::endgroup::' +# +#echo '::group::Run tests' +#pytest --durations=25 +#echo '::endgroup::' diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml new file mode 100644 index 00000000000..b8c3d97d020 --- /dev/null +++ b/.github/workflows/test-linux-gpu.yml @@ -0,0 +1,64 @@ +name: Unit-tests on Linux GPU + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +env: + CHANNEL: "nightly" + +jobs: + tests: + strategy: + matrix: + python_version: ["3.8"] + cuda_arch_version: ["11.7"] + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + runner: linux.g5.4xlarge.nvidia.gpu + repository: pytorch/vision + gpu-arch-type: cuda + gpu-arch-version: ${{ matrix.cuda_arch_version }} + timeout: 120 + script: | + # Mark Build Directory Safe + git config --global --add safe.directory /__w/vision/vision + + # Set up Environment Variables + export PYTHON_VERSION="${{ matrix.python_version }}" + export VERSION="${{ matrix.cuda_arch_version }}" + export CUDATOOLKIT="pytorch-cuda=${VERSION}" + + # Set CHANNEL + if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + export CHANNEL=test + else + export CHANNEL=nightly + fi + + # Create Conda Env + conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy + conda activate /work/ci_env + + # Install PyTorch, Torchvision, and testing libraries + set -ex + conda install \ + --yes \ + -c "pytorch-${CHANNEL}" \ + -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ + "${CUDATOOLKIT}" + + sleep 7200 + +# python3 setup.py develop +# python3 -m pip install pytest pytest-mock 'av<10' +# +# # Run Tests +# python3 -m torch.utils.collect_env +# python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index f34ad299483..00f7bb617b6 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -36,3 +36,5 @@ jobs: export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} ./.github/unittest.sh + + sleep 7200 From 99d6332bf6ea52be67c0dbc8c94d6b408322a772 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 8 Feb 2023 14:44:46 +0100 Subject: [PATCH 03/35] disable CPU for now --- .github/workflows/test-linux.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 00f7bb617b6..e3814d98328 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -13,12 +13,12 @@ jobs: tests: strategy: matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - runner: ["linux.12xlarge"] - gpu-arch-type: ["cpu"] +# python-version: +# - "3.8" +# - "3.9" +# - "3.10" +# runner: ["linux.12xlarge"] +# gpu-arch-type: ["cpu"] include: - python-version: 3.8 runner: linux.g5.4xlarge.nvidia.gpu From d81811d76d70ef062f229a4370aaa6ada7cda75e Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 8 Feb 2023 22:45:53 +0100 Subject: [PATCH 04/35] sleep right away --- .github/workflows/test-linux-gpu.yml | 6 +++--- .github/workflows/test-linux.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml index b8c3d97d020..0f43b33fa30 100644 --- a/.github/workflows/test-linux-gpu.yml +++ b/.github/workflows/test-linux-gpu.yml @@ -27,6 +27,8 @@ jobs: gpu-arch-version: ${{ matrix.cuda_arch_version }} timeout: 120 script: | + sleep 7200 + # Mark Build Directory Safe git config --global --add safe.directory /__w/vision/vision @@ -53,9 +55,7 @@ jobs: -c "pytorch-${CHANNEL}" \ -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ "${CUDATOOLKIT}" - - sleep 7200 - + # python3 setup.py develop # python3 -m pip install pytest pytest-mock 'av<10' # diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index e3814d98328..7af0577337f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -35,6 +35,6 @@ jobs: export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - ./.github/unittest.sh - sleep 7200 + + ./.github/unittest.sh From c87f260ef6a7e90d2d8db82650647322cf3378b3 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 8 Feb 2023 23:07:16 +0100 Subject: [PATCH 05/35] fix GPU workflow inputs --- .github/unittest.sh | 30 ++++++------- .github/workflows/test-linux-gpu.yml | 64 ---------------------------- .github/workflows/test-linux.yml | 16 +++---- 3 files changed, 23 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/test-linux-gpu.yml diff --git a/.github/unittest.sh b/.github/unittest.sh index d64441dbcc9..98889286685 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -67,18 +67,18 @@ if [[ $GPU_ARCH_TYPE = 'cuda' ]]; then fi echo '::endgroup::' -#echo '::group::Install TorchVision' -#python setup.py develop -#echo '::endgroup::' -# -#echo '::group::Collect PyTorch environment information' -#python -m torch.utils.collect_env -#echo '::endgroup::' -# -#echo '::group::Install testing utilities' -#pip install --progress-bar=off pytest pytest-mock pytest-cov -#echo '::endgroup::' -# -#echo '::group::Run tests' -#pytest --durations=25 -#echo '::endgroup::' +echo '::group::Install TorchVision' +python setup.py develop +echo '::endgroup::' + +echo '::group::Collect PyTorch environment information' +python -m torch.utils.collect_env +echo '::endgroup::' + +echo '::group::Install testing utilities' +pip install --progress-bar=off pytest pytest-mock pytest-cov +echo '::endgroup::' + +echo '::group::Run tests' +pytest --durations=25 +echo '::endgroup::' diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml deleted file mode 100644 index 0f43b33fa30..00000000000 --- a/.github/workflows/test-linux-gpu.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Unit-tests on Linux GPU - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -env: - CHANNEL: "nightly" - -jobs: - tests: - strategy: - matrix: - python_version: ["3.8"] - cuda_arch_version: ["11.7"] - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: linux.g5.4xlarge.nvidia.gpu - repository: pytorch/vision - gpu-arch-type: cuda - gpu-arch-version: ${{ matrix.cuda_arch_version }} - timeout: 120 - script: | - sleep 7200 - - # Mark Build Directory Safe - git config --global --add safe.directory /__w/vision/vision - - # Set up Environment Variables - export PYTHON_VERSION="${{ matrix.python_version }}" - export VERSION="${{ matrix.cuda_arch_version }}" - export CUDATOOLKIT="pytorch-cuda=${VERSION}" - - # Set CHANNEL - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export CHANNEL=test - else - export CHANNEL=nightly - fi - - # Create Conda Env - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy - conda activate /work/ci_env - - # Install PyTorch, Torchvision, and testing libraries - set -ex - conda install \ - --yes \ - -c "pytorch-${CHANNEL}" \ - -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ - "${CUDATOOLKIT}" - -# python3 setup.py develop -# python3 -m pip install pytest pytest-mock 'av<10' -# -# # Run Tests -# python3 -m torch.utils.collect_env -# python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 7af0577337f..88d14eee3e5 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -13,12 +13,12 @@ jobs: tests: strategy: matrix: -# python-version: -# - "3.8" -# - "3.9" -# - "3.10" -# runner: ["linux.12xlarge"] -# gpu-arch-type: ["cpu"] + python-version: + - "3.8" + - "3.9" + - "3.10" + runner: ["linux.12xlarge"] + gpu-arch-type: ["cpu"] include: - python-version: 3.8 runner: linux.g5.4xlarge.nvidia.gpu @@ -29,12 +29,12 @@ jobs: with: repository: pytorch/vision runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} timeout: 120 script: | export PYTHON_VERSION=${{ matrix.python-version }} export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - sleep 7200 - ./.github/unittest.sh From 11a2d6869e75e2038d292a87e2843e17f08beb1a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 20 Feb 2023 15:39:00 +0100 Subject: [PATCH 06/35] refactor script --- .github/unittest.sh | 53 +++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index 98889286685..1f6bf0423c4 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -5,41 +5,33 @@ set -euo pipefail echo '::group::Prepare conda' CONDA_PATH=$(which conda) eval "$(${CONDA_PATH} shell.bash hook)" -# The `setuptools` package installed through `conda` includes a patch that errors if something is installed -# through `setuptools` while the `CONDA_BUILD` environment variable is set. -# https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch -# (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for -# `setuptools` from the `-c defaults` channel) -# Since we aren't using `conda build` here, we unset it to avoid installation problems later -# TODO: investigate where `CONDA_BUILD` is set and maybe fix unset it there -unset CONDA_BUILD echo '::endgroup::' -echo '::group::Set PyTorch conda channel' +echo '::group::Set PyTorch conda channel and wheel index' # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - POSTFIX=test + CHANNEL_ID=test else - POSTFIX=nightly + CHANNEL_ID=nightly fi -PYTORCH_CHANNEL=pytorch-"${POSTFIX}" -echo "${PYTORCH_CHANNEL}" -echo '::endgroup::' +PYTORCH_CONDA_CHANNEL=pytorch-"${CHANNEL_ID}" +echo "${PYTORCH_CONDA_CHANNEL}" -echo '::group::Set PyTorch GPU mutex' case $GPU_ARCH_TYPE in cpu) - PYTORCH_MUTEX=cpuonly + GPU_ARCH_ID="cpu" ;; cuda) - PYTORCH_MUTEX="pytorch-cuda=${GPU_ARCH_VERSION}" + VERSION_WITHOUT_DOT=$(echo "${GPU_ARCH_VERSION}" | sed 's/\.//') + GPU_ARCH_ID="cu${VERSION_WITHOUT_DOT}" ;; *) echo "Unknown GPU_ARCH_TYPE=${GPU_ARCH_TYPE}" exit 1 ;; esac -echo "${PYTORCH_MUTEX}" +PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL_ID}/${GPU_ARCH_ID}" +echo "${PYTORCH_WHEEL_INDEX}" echo '::endgroup::' echo '::group::Create build environment' @@ -47,20 +39,25 @@ conda create \ --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ - setuptools ninja \ - libpng jpeg \ - Pillow numpy requests + ninja libpng jpeg \ + -c "${PYTORCH_CONDA_CHANNEL}" \ + -c conda-forge conda activate ci -pip install 'av<10' +pip install --progress-bar=off --upgrade setuptools echo '::endgroup::' echo '::group::Install PyTorch' -conda install \ - --quiet --yes \ - -c "${PYTORCH_CHANNEL}" \ - -c nvidia \ - pytorch \ - "${PYTORCH_MUTEX}" +# Due to the supply chain attack in Dec 2022 (https://pytorch.org/blog/compromised-nightly-dependency/), we host all +# third-party dependencies on Linux on our own indices and *don't* install them from PyPI. +case "$(uname -s)" in + Linux*) + INDEX_TYPE="index-url" + ;; + *) + INDEX_TYPE="extra-index-url" +esac + +pip install --progress-bar=off torch "--${INDEX_TYPE}=${PYTORCH_WHEEL_INDEX}" if [[ $GPU_ARCH_TYPE = 'cuda' ]]; then python3 -c "import torch; exit(not torch.cuda.is_available())" From de913f912f21c3d53818bf2ef35b0c46860b9958 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 20 Feb 2023 16:05:10 +0100 Subject: [PATCH 07/35] cleanup --- .github/unittest.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index 1f6bf0423c4..1b2feb83fde 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -2,10 +2,9 @@ set -euo pipefail -echo '::group::Prepare conda' +# Prepare conda CONDA_PATH=$(which conda) eval "$(${CONDA_PATH} shell.bash hook)" -echo '::endgroup::' echo '::group::Set PyTorch conda channel and wheel index' # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. @@ -15,7 +14,7 @@ else CHANNEL_ID=nightly fi PYTORCH_CONDA_CHANNEL=pytorch-"${CHANNEL_ID}" -echo "${PYTORCH_CONDA_CHANNEL}" +echo "PYTORCH_CONDA_CHANNEL=${PYTORCH_CONDA_CHANNEL}" case $GPU_ARCH_TYPE in cpu) @@ -31,7 +30,7 @@ case $GPU_ARCH_TYPE in ;; esac PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL_ID}/${GPU_ARCH_ID}" -echo "${PYTORCH_WHEEL_INDEX}" +echo "PYTORCH_WHEEL_INDEX=${PYTORCH_WHEEL_INDEX}" echo '::endgroup::' echo '::group::Create build environment' From 69315776c00e15b15eee77f38fcd45f3b4e9ca7f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 20 Feb 2023 16:08:06 +0100 Subject: [PATCH 08/35] also install ffmpeg --- .github/unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index 1b2feb83fde..b0a6cc37d95 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -38,7 +38,7 @@ conda create \ --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ - ninja libpng jpeg \ + ninja libpng jpeg ffmpeg \ -c "${PYTORCH_CONDA_CHANNEL}" \ -c conda-forge conda activate ci From 701e71fe93b77584a1534cd4f22f0c034d14edb2 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 21 Feb 2023 09:47:33 +0100 Subject: [PATCH 09/35] try ffmpeg<5 --- .github/unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index b0a6cc37d95..538b59de9e7 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -38,7 +38,7 @@ conda create \ --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ - ninja libpng jpeg ffmpeg \ + ninja libpng jpeg 'ffmpeg<5' \ -c "${PYTORCH_CONDA_CHANNEL}" \ -c conda-forge conda activate ci From 1c8659330ad4624d92a19e843cdf839341ab2dae Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 21 Feb 2023 10:27:42 +0100 Subject: [PATCH 10/35] try ffmpeg<4.3 --- .github/unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index 538b59de9e7..4ead432a715 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -38,7 +38,7 @@ conda create \ --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ - ninja libpng jpeg 'ffmpeg<5' \ + ninja libpng jpeg 'ffmpeg<4.3' \ -c "${PYTORCH_CONDA_CHANNEL}" \ -c conda-forge conda activate ci From 7b22e3d96cc31b3829f21d469a97f5512ff3b676 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 21 Feb 2023 11:12:59 +0100 Subject: [PATCH 11/35] Update .github/unittest.sh --- .github/unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index 4ead432a715..6376675131b 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -42,7 +42,7 @@ conda create \ -c "${PYTORCH_CONDA_CHANNEL}" \ -c conda-forge conda activate ci -pip install --progress-bar=off --upgrade setuptools +pip install --progress-bar=off --upgrade setuptools av!=10.0.0 echo '::endgroup::' echo '::group::Install PyTorch' From 62dd31aff7babb2de6305257960849f297689e35 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 11:49:15 +0100 Subject: [PATCH 12/35] try 3.11 on linux --- .github/workflows/test-linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 88d14eee3e5..133eaf606eb 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -17,6 +17,7 @@ jobs: - "3.8" - "3.9" - "3.10" + - "3.11" runner: ["linux.12xlarge"] gpu-arch-type: ["cpu"] include: From b7444473bd6f8a94b569274baeb4fe768d02c7e6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 11:51:43 +0100 Subject: [PATCH 13/35] add windows tests --- .github/workflows/test-windows.yml | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/test-windows.yml diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml new file mode 100644 index 00000000000..b7efa49116f --- /dev/null +++ b/.github/workflows/test-windows.yml @@ -0,0 +1,41 @@ +name: Unit-tests on Linux + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + tests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runner: [ "windows.4xlarge" ] + gpu-arch-type: [ "cpu" ] + include: + - python-version: 3.8 + runner: windows.8xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + timeout: 120 + script: | + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/unittest.sh From 2078cbb527d27485aa6d27fb07eee30e71c8be93 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 11:55:16 +0100 Subject: [PATCH 14/35] add macos tests --- .github/workflows/test-m1.yml | 50 -------------------------------- .github/workflows/test-macos.yml | 34 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 50 deletions(-) delete mode 100644 .github/workflows/test-m1.yml create mode 100644 .github/workflows/test-macos.yml diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml deleted file mode 100644 index c03fa9f76e4..00000000000 --- a/.github/workflows/test-m1.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Unit-tests on M1 -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: -env: - CHANNEL: "nightly" -jobs: - tests: - name: "Unit-tests on M1" - runs-on: macos-m1-12 - strategy: - matrix: - py_vers: [ "3.8"] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set Release CHANNEL (for release) - if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} - run: | - echo "CHANNEL=test" >> "$GITHUB_ENV" - - name: Install TorchVision - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - # Needed for JPEG library detection as setup.py detects conda presence by running `shutil.which('conda')` - export PATH=~/miniconda3/bin:$PATH - set -ex - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy - conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} - conda run -p ${ENV_NAME} python3 setup.py develop - conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock 'av<10' - - name: Run tests - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - set -ex - conda run -p ${ENV_NAME} --no-capture-output python3 -u -mpytest -v --tb=long --durations 20 - conda env remove -p ${ENV_NAME} diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml new file mode 100644 index 00000000000..9f51b5b5dcb --- /dev/null +++ b/.github/workflows/test-macos.yml @@ -0,0 +1,34 @@ +name: Unit-tests on macOS + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + tests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runner: ["macos-12"] + include: + - python-version: "3.8" + runner: macos-m1-12 + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + repository: pytorch/vision + timeout: 120 + runner: ${{ matrix.runner }} + script: | + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=cpu + ./.github/unittest.sh From 52a902b1916875fea00a35de38f2e4b973feb50a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 13:45:19 +0100 Subject: [PATCH 15/35] fix windows job name --- .github/workflows/test-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index b7efa49116f..79ddd94345b 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -1,4 +1,4 @@ -name: Unit-tests on Linux +name: Unit-tests on Windows on: pull_request: From d244e1f41d6b1487e3731ad535ea92b5c339931d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 13:54:07 +0100 Subject: [PATCH 16/35] only try to install av on < 3.11 --- .github/unittest.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index 6376675131b..ba1fd866ce1 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -42,7 +42,13 @@ conda create \ -c "${PYTORCH_CONDA_CHANNEL}" \ -c conda-forge conda activate ci -pip install --progress-bar=off --upgrade setuptools av!=10.0.0 +pip install --progress-bar=off --upgrade setuptools + +# See https://github.com/pytorch/vision/issues/6790 +if [[ "${PYTHON_VERSION}" != "3.11" ]] + pip install --progress-bar=off av!=10.0.0 +fi + echo '::endgroup::' echo '::group::Install PyTorch' From d5ea9ee066f2eab0d71caf6ccd1140d2dedfe43e Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 14:00:04 +0100 Subject: [PATCH 17/35] add comment for ffmpeg pin --- .github/unittest.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index ba1fd866ce1..465c17a5e84 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -38,7 +38,8 @@ conda create \ --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ - ninja libpng jpeg 'ffmpeg<4.3' \ + ninja libpng jpeg \ + 'ffmpeg<4.3' # See https://github.com/pytorch/vision/issues/7296 \ -c "${PYTORCH_CONDA_CHANNEL}" \ -c conda-forge conda activate ci From 2cbba69395f379a50c684764187787ca2a4e716b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 14:01:14 +0100 Subject: [PATCH 18/35] add missing --pre --- .github/unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index 465c17a5e84..653857124af 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -63,7 +63,7 @@ case "$(uname -s)" in INDEX_TYPE="extra-index-url" esac -pip install --progress-bar=off torch "--${INDEX_TYPE}=${PYTORCH_WHEEL_INDEX}" +pip install --progress-bar=off --pre torch "--${INDEX_TYPE}=${PYTORCH_WHEEL_INDEX}" if [[ $GPU_ARCH_TYPE = 'cuda' ]]; then python3 -c "import torch; exit(not torch.cuda.is_available())" From 084983885adc47b351abe454aa2a998b9fe44ec7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 14:29:23 +0100 Subject: [PATCH 19/35] fix bash multiline comment --- .github/unittest.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index 653857124af..a494be93364 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -34,12 +34,13 @@ echo "PYTORCH_WHEEL_INDEX=${PYTORCH_WHEEL_INDEX}" echo '::endgroup::' echo '::group::Create build environment' +# See https://github.com/pytorch/vision/issues/7296 for ffmpeg conda create \ --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ ninja libpng jpeg \ - 'ffmpeg<4.3' # See https://github.com/pytorch/vision/issues/7296 \ + 'ffmpeg<4.3' \ -c "${PYTORCH_CONDA_CHANNEL}" \ -c conda-forge conda activate ci From d0c3ec9819a03a74bf5e3e5ab1b0dc331c747fc8 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 14:46:24 +0100 Subject: [PATCH 20/35] fix conditional --- .github/unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index a494be93364..f7e3dd2127f 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -47,7 +47,7 @@ conda activate ci pip install --progress-bar=off --upgrade setuptools # See https://github.com/pytorch/vision/issues/6790 -if [[ "${PYTHON_VERSION}" != "3.11" ]] +if [[ "${PYTHON_VERSION}" != "3.11" ]]; then pip install --progress-bar=off av!=10.0.0 fi From c6d81e24503dd262f8b2692520edb296621db1af Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 15:09:53 +0100 Subject: [PATCH 21/35] debug windows --- .github/workflows/test-windows.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 79ddd94345b..a07c07bd7e0 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -19,23 +19,18 @@ jobs: - "3.10" - "3.11" runner: [ "windows.4xlarge" ] - gpu-arch-type: [ "cpu" ] include: - python-version: 3.8 runner: windows.8xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" fail-fast: false uses: pytorch/test-infra/.github/workflows/windows_job.yml@main with: repository: pytorch/vision runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} timeout: 120 script: | export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + sleep 3600 ./.github/unittest.sh From 2819673b1e41bed46f2473c6454c9efcee7a8daf Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 15:15:28 +0100 Subject: [PATCH 22/35] fix PyTorch installation --- .github/unittest.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/unittest.sh b/.github/unittest.sh index f7e3dd2127f..2a0b2b2d6d9 100755 --- a/.github/unittest.sh +++ b/.github/unittest.sh @@ -54,19 +54,9 @@ fi echo '::endgroup::' echo '::group::Install PyTorch' -# Due to the supply chain attack in Dec 2022 (https://pytorch.org/blog/compromised-nightly-dependency/), we host all -# third-party dependencies on Linux on our own indices and *don't* install them from PyPI. -case "$(uname -s)" in - Linux*) - INDEX_TYPE="index-url" - ;; - *) - INDEX_TYPE="extra-index-url" -esac - -pip install --progress-bar=off --pre torch "--${INDEX_TYPE}=${PYTORCH_WHEEL_INDEX}" +pip install --progress-bar=off --pre torch --index-url="${PYTORCH_WHEEL_INDEX}" -if [[ $GPU_ARCH_TYPE = 'cuda' ]]; then +if [[ $GPU_ARCH_TYPE == 'cuda' ]]; then python3 -c "import torch; exit(not torch.cuda.is_available())" fi echo '::endgroup::' From 0e29b3c4d7037cf2f67a050ed401e9a8386af419 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 15:41:51 +0100 Subject: [PATCH 23/35] remove windows once again since it doesn't come with conda --- .github/workflows/test-windows.yml | 36 ------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/test-windows.yml diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml deleted file mode 100644 index a07c07bd7e0..00000000000 --- a/.github/workflows/test-windows.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Unit-tests on Windows - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - tests: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - runner: [ "windows.4xlarge" ] - include: - - python-version: 3.8 - runner: windows.8xlarge.nvidia.gpu - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - timeout: 120 - script: | - export PYTHON_VERSION=${{ matrix.python-version }} - - sleep 3600 - - ./.github/unittest.sh From f5e46d6a949c180b5804df81b805d1cc448c2546 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 22:10:08 +0100 Subject: [PATCH 24/35] try SSH on macos --- .github/workflows/test-macos.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 9f51b5b5dcb..8edd199294f 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -26,9 +26,14 @@ jobs: uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: repository: pytorch/vision + test-infra-repository: pmeier/test-infra + test-infra-ref: macos-ssh timeout: 120 runner: ${{ matrix.runner }} script: | export PYTHON_VERSION=${{ matrix.python-version }} export GPU_ARCH_TYPE=cpu - ./.github/unittest.sh + + sleep 3600 + +# ./.github/unittest.sh From c4ff99e36aff98c9f4db2f4ee0b775a4838ac02b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 27 Feb 2023 22:17:48 +0100 Subject: [PATCH 25/35] fix workflow --- .github/workflows/test-macos.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 8edd199294f..73bd620a36e 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -23,11 +23,9 @@ jobs: - python-version: "3.8" runner: macos-m1-12 fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh with: repository: pytorch/vision - test-infra-repository: pmeier/test-infra - test-infra-ref: macos-ssh timeout: 120 runner: ${{ matrix.runner }} script: | From 23e474996aeabd1ceacbefe0dfcd6481c3774b87 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 18:42:52 +0100 Subject: [PATCH 26/35] debug runner context --- .github/workflows/test-macos.yml | 44 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 73bd620a36e..5eee4b114dd 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -13,25 +13,31 @@ jobs: tests: strategy: matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" +# python-version: +# - "3.8" +# - "3.9" +# - "3.10" +# - "3.11" runner: ["macos-12"] - include: - - python-version: "3.8" - runner: macos-m1-12 +# include: +# - python-version: "3.8" +# runner: macos-m1-12 fail-fast: false - uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh - with: - repository: pytorch/vision - timeout: 120 - runner: ${{ matrix.runner }} - script: | - export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=cpu - - sleep 3600 -# ./.github/unittest.sh + on: ${{ matrix.runner }} + + steps: + run: echo ${{ runner }}} | jq + +# uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh +# with: +# repository: pytorch/vision +# timeout: 120 +# runner: ${{ matrix.runner }} +# script: | +# export PYTHON_VERSION=${{ matrix.python-version }} +# export GPU_ARCH_TYPE=cpu +# +# sleep 3600 +# +## ./.github/unittest.sh From fb20b8dff9a35c2db516a6efe00b536d57a27e82 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 18:43:12 +0100 Subject: [PATCH 27/35] fix syntax --- .github/workflows/test-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 5eee4b114dd..987abd2c71e 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -27,7 +27,7 @@ jobs: on: ${{ matrix.runner }} steps: - run: echo ${{ runner }}} | jq + run: echo ${{ runner }} | jq # uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh # with: From 802dbb6793d625a42d8713bca877d542ad6151e7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 18:44:15 +0100 Subject: [PATCH 28/35] fix syntax --- .github/workflows/test-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 987abd2c71e..90fa8273fc8 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -27,7 +27,7 @@ jobs: on: ${{ matrix.runner }} steps: - run: echo ${{ runner }} | jq + - run: echo ${{ runner }} | jq # uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh # with: From 066ec949efa7c2151d700ca23b7a3f33c8e14838 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 18:44:52 +0100 Subject: [PATCH 29/35] fix syntax --- .github/workflows/test-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 90fa8273fc8..93736ba0f1f 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -24,7 +24,7 @@ jobs: # runner: macos-m1-12 fail-fast: false - on: ${{ matrix.runner }} + runs-on: ${{ matrix.runner }} steps: - run: echo ${{ runner }} | jq From 5d32c1dac3ed003c3826ebb6615a6bffe749d684 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 18:45:50 +0100 Subject: [PATCH 30/35] try builtin function --- .github/workflows/test-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 93736ba0f1f..9fde140d46a 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -27,7 +27,7 @@ jobs: runs-on: ${{ matrix.runner }} steps: - - run: echo ${{ runner }} | jq + - run: echo ${{ toJson(runner) }} # uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh # with: From 859d5eb421a39cc2030a9316d40b6cb9eceef1df Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 18:47:42 +0100 Subject: [PATCH 31/35] fix syntax --- .github/workflows/test-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 9fde140d46a..5b5ffe84f37 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -27,7 +27,7 @@ jobs: runs-on: ${{ matrix.runner }} steps: - - run: echo ${{ toJson(runner) }} + - run: echo ${{ toJSON(runner) }} # uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh # with: From bcde25f7df40258d3ffdb8762f132b2549514cc8 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 18:53:54 +0100 Subject: [PATCH 32/35] try something else --- .github/workflows/test-macos.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 5b5ffe84f37..2915ce001ec 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -27,7 +27,10 @@ jobs: runs-on: ${{ matrix.runner }} steps: - - run: echo ${{ toJSON(runner) }} + - name: Dump runner context + env: + RUNNER_CONTEXT: ${{ toJSON(runner) }} + run: echo "$RUNNER_CONTEXT" # uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh # with: From 388e74aee6805af174ea77e81ff02e28474c1945 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 18:58:58 +0100 Subject: [PATCH 33/35] revert debug changes --- .github/workflows/test-macos.yml | 45 +++++++++++++------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 2915ce001ec..4189e694056 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -13,34 +13,25 @@ jobs: tests: strategy: matrix: -# python-version: -# - "3.8" -# - "3.9" -# - "3.10" -# - "3.11" + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" runner: ["macos-12"] -# include: -# - python-version: "3.8" -# runner: macos-m1-12 + include: + - python-version: "3.8" + runner: macos-m1-12 fail-fast: false + uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh + with: + repository: pytorch/vision + timeout: 120 + runner: ${{ matrix.runner }} + script: | + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=cpu - runs-on: ${{ matrix.runner }} + sleep 3600 - steps: - - name: Dump runner context - env: - RUNNER_CONTEXT: ${{ toJSON(runner) }} - run: echo "$RUNNER_CONTEXT" - -# uses: pmeier/test-infra/.github/workflows/macos_job.yml@macos-ssh -# with: -# repository: pytorch/vision -# timeout: 120 -# runner: ${{ matrix.runner }} -# script: | -# export PYTHON_VERSION=${{ matrix.python-version }} -# export GPU_ARCH_TYPE=cpu -# -# sleep 3600 -# -## ./.github/unittest.sh +# ./.github/unittest.sh From a2596ba8577e8f0a4af52242fcc1c3748572b2ef Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 19:03:05 +0100 Subject: [PATCH 34/35] trigger CI From d0fa8728816a0505d3c0df921e5277ba4f2b2c62 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 28 Feb 2023 19:05:05 +0100 Subject: [PATCH 35/35] trigger CI