diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bc482f8c75..0463007af46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -124,144 +124,6 @@ jobs: python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - binary_win_conda: - <<: *binary_common - executor: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Build conda packages - no_output_timeout: 30m - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda install -yq conda-build "conda-package-handling!=1.5.0" - packaging/build_conda.sh - rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2 - - store_artifacts: - path: C:/tools/miniconda3/conda-bld/win-64 - - persist_to_workspace: - root: C:/tools/miniconda3/conda-bld/win-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_win_wheel: - <<: *binary_common - executor: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Build wheel packages - no_output_timeout: 30m - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_macos_wheel: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - # Cannot easily deduplicate this as source'ing activate - # will set environment variables which we need to propagate - # to build_wheel.sh - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_macos_conda: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build - packaging/build_conda.sh - - store_artifacts: - path: /Users/distiller/miniconda3/conda-bld/osx-64 - - persist_to_workspace: - root: /Users/distiller/miniconda3/conda-bld/osx-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - # Requires org-member context - binary_conda_upload: - docker: - - image: continuumio/miniconda - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - command: | - # Prevent credential from leaking - conda install -yq anaconda-client - set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force - - # Requires org-member context - binary_wheel_upload: - parameters: - subfolder: - description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)" - type: string - docker: - - image: cimg/python:3.8 - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - checkout - - pip_install: - args: awscli - - run: - command: | - export PATH="$HOME/.local/bin:$PATH" - # Prevent credential from leaking - set +x - export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" - export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" - set -x - for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read - done - smoke_test_docker_image_build: machine: image: ubuntu-2004:202104-01 @@ -372,10 +234,6 @@ workflows: jobs: - circleci_consistency - build: - jobs: - [] - cmake: jobs: - cmake_linux_cpu: @@ -400,9 +258,6 @@ workflows: name: cmake_macos_cpu python_version: '3.8' - nightly: - jobs: - [] docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 2358b0a9599..f3a88f2d987 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -124,144 +124,6 @@ jobs: python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - binary_win_conda: - <<: *binary_common - executor: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Build conda packages - no_output_timeout: 30m - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda install -yq conda-build "conda-package-handling!=1.5.0" - packaging/build_conda.sh - rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2 - - store_artifacts: - path: C:/tools/miniconda3/conda-bld/win-64 - - persist_to_workspace: - root: C:/tools/miniconda3/conda-bld/win-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_win_wheel: - <<: *binary_common - executor: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Build wheel packages - no_output_timeout: 30m - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_macos_wheel: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - # Cannot easily deduplicate this as source'ing activate - # will set environment variables which we need to propagate - # to build_wheel.sh - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_macos_conda: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build - packaging/build_conda.sh - - store_artifacts: - path: /Users/distiller/miniconda3/conda-bld/osx-64 - - persist_to_workspace: - root: /Users/distiller/miniconda3/conda-bld/osx-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - # Requires org-member context - binary_conda_upload: - docker: - - image: continuumio/miniconda - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - command: | - # Prevent credential from leaking - conda install -yq anaconda-client - set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force - - # Requires org-member context - binary_wheel_upload: - parameters: - subfolder: - description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)" - type: string - docker: - - image: cimg/python:3.8 - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - checkout - - pip_install: - args: awscli - - run: - command: | - export PATH="$HOME/.local/bin:$PATH" - # Prevent credential from leaking - set +x - export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" - export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" - set -x - for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read - done - smoke_test_docker_image_build: machine: image: ubuntu-2004:202104-01 @@ -372,17 +234,10 @@ workflows: jobs: - circleci_consistency - build: - jobs: - {{ build_workflows(windows_latest_only=True) }} - cmake: jobs: {{ cmake_workflows() }} - nightly: - jobs: - {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }} docker_build: triggers: - schedule: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 7278c4827a6..e7b8db3d56e 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -21,202 +21,6 @@ from jinja2 import select_autoescape -PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"] - -RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" - - -def build_workflows(prefix="", filter_branch=None, upload=False, indentation=6, windows_latest_only=False): - w = [] - - # Don't generate anything for build workflow - return indent(indentation, w) - - for btype in ["wheel", "conda"]: - for os_type in ["linux", "macos", "win"]: - python_versions = PYTHON_VERSIONS - cu_versions_dict = { - "linux": ["cpu", "cu117", "cu118", "cu121", "rocm5.2", "rocm5.3"], - "win": ["cpu", "cu117", "cu118", "cu121"], - "macos": ["cpu"], - } - cu_versions = cu_versions_dict[os_type] - for python_version in python_versions: - for cu_version in cu_versions: - # ROCm conda packages not yet supported - if cu_version.startswith("rocm") and btype == "conda": - continue - for unicode in [False]: - fb = filter_branch - if ( - windows_latest_only - and os_type == "win" - and filter_branch is None - and ( - python_version != python_versions[-1] - or (cu_version not in [cu_versions[0], cu_versions[-1]]) - ) - ): - fb = "main" - if not fb and ( - os_type == "linux" and cu_version == "cpu" and btype == "wheel" and python_version == "3.8" - ): - # the fields must match the build_docs "requires" dependency - fb = "/.*/" - - # Disable all Linux Wheels Workflows from CircleCI - if os_type == "linux" and btype == "wheel": - continue - - # Disable all Macos Wheels Workflows from CircleCI. - if os_type == "macos" and btype == "wheel": - continue - - # Disable all non-Windows Conda workflows - if os_type != "win" and btype == "conda": - continue - - w += workflow_pair( - btype, os_type, python_version, cu_version, unicode, prefix, upload, filter_branch=fb - ) - - return indent(indentation, w) - - -def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix="", upload=False, *, filter_branch=None): - - w = [] - unicode_suffix = "u" if unicode else "" - base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}" - - w.append( - generate_base_workflow( - base_workflow_name, python_version, cu_version, unicode, os_type, btype, filter_branch=filter_branch - ) - ) - - # For the remaining py3.8 Linux Wheels job left around for the docs build, - # we'll disable uploads. - if os_type == "linux" and btype == "wheel": - upload = False - - if upload: - w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch)) - # disable smoke tests, they are broken and needs to be fixed - # if filter_branch == "nightly" and os_type in ["linux", "win"]: - # pydistro = "pip" if btype == "wheel" else "conda" - # w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type)) - - return w - - -manylinux_images = { - "cu117": "pytorch/manylinux-cuda117", - "cu118": "pytorch/manylinux-cuda118", - "cu121": "pytorch/manylinux-cuda121", -} - - -def get_manylinux_image(cu_version): - if cu_version == "cpu": - return "pytorch/manylinux-cpu" - elif cu_version.startswith("cu"): - cu_suffix = cu_version[len("cu") :] - return f"pytorch/manylinux-cuda{cu_suffix}" - elif cu_version.startswith("rocm"): - rocm_suffix = cu_version[len("rocm") :] - return f"pytorch/manylinux-rocm:{rocm_suffix}" - - -def get_conda_image(cu_version): - if cu_version == "cpu": - return "pytorch/conda-builder:cpu" - elif cu_version.startswith("cu"): - cu_suffix = cu_version[len("cu") :] - return f"pytorch/conda-builder:cuda{cu_suffix}" - - -def generate_base_workflow( - base_workflow_name, python_version, cu_version, unicode, os_type, btype, *, filter_branch=None -): - - d = { - "name": base_workflow_name, - "python_version": python_version, - "cu_version": cu_version, - } - - if os_type != "win" and unicode: - d["unicode_abi"] = "1" - - if os_type != "win": - d["wheel_docker_image"] = get_manylinux_image(cu_version) - # ROCm conda packages not yet supported - if "rocm" not in cu_version: - d["conda_docker_image"] = get_conda_image(cu_version) - - if filter_branch is not None: - d["filters"] = { - "branches": {"only": filter_branch}, - "tags": { - # Using a raw string here to avoid having to escape - # anything - "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" - }, - } - - w = f"binary_{os_type}_{btype}" - return {w: d} - - -def gen_filter_branch_tree(*branches, tags_list=None): - filter_dict = {"branches": {"only": [b for b in branches]}} - if tags_list is not None: - filter_dict["tags"] = {"only": tags_list} - return filter_dict - - -def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None): - d = { - "name": f"{base_workflow_name}_upload", - "context": "org-member", - "requires": [base_workflow_name], - } - - if btype == "wheel": - d["subfolder"] = "" if os_type == "macos" else cu_version + "/" - - if filter_branch is not None: - d["filters"] = { - "branches": {"only": filter_branch}, - "tags": { - # Using a raw string here to avoid having to escape - # anything - "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" - }, - } - - return {f"binary_{btype}_upload": d} - - -def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type): - - required_build_suffix = "_upload" - required_build_name = base_workflow_name + required_build_suffix - - smoke_suffix = f"smoke_test_{pydistro}" - d = { - "name": f"{base_workflow_name}_{smoke_suffix}", - "requires": [required_build_name], - "python_version": python_version, - } - - if filter_branch: - d["filters"] = gen_filter_branch_tree(filter_branch) - - return {f"smoke_test_{os_type}_{pydistro}": d} - - def indent(indentation, data_list): return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines()) @@ -249,7 +53,6 @@ def cmake_workflows(indentation=6): with open(os.path.join(d, "config.yml"), "w") as f: f.write( env.get_template("config.yml.in").render( - build_workflows=build_workflows, cmake_workflows=cmake_workflows, ) )