config.yml

# This file uses YAML anchors and aliases to prevent repetition of blocks of config:
# https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/
#
# Two primary anchors are checkout and setup_env, called as the first step of almost all jobs:
# - checkout: A custom checkout step to reduce the amount of data downloaded to improve speed.
# - setup_env: Sets up the common environment used by all build steps.
#
# Two CCI executors are used:
# - docker (small): Used only to launch external EC2 instances for big workloads. It's the cheapest option.
# - machine (large): Used for building in CCI itself. 4cpus, 15GB has the optimal power/cost ratio.
#
# The docker executor uses a custom image build in `build_image`. It's specifically streamlined for fast download
# with just enough tools to execute the build system, and launch EC2 instances etc.
#
# There are some `join` steps that are just no-ops. They are just used to produce cleaner graph rendering in CCI.

version: 2.1

setup: true # have a dynamic config step
orbs:
  continuation: circleci/continuation@1.0.0
  slack: circleci/slack@4.12.5

parameters:
  workflow:
    type: string
    default: "system"

# This build step checks out the code from the repository. It has a hardcoded readonly key to allow the checkout.
# Initially it just fetches the repo metadata for the current commit hash to a depth of 50 commits.
# We need historical commit hashes to calculate diffs between previous and current commits.
# It then checks out the fetched head to actually download the data.
checkout: &checkout
  run:
    name: "Checkout code"
    command: |
      function retry() {
        # Retries up to 3 times with 10 second intervals
        for i in $(seq 1 3); do
          "$@" && return || sleep 10
        done
        echo "$@ failed after 3 attempts"
        exit 1
      }
      cd $HOME
      mkdir -p .ssh
      chmod 0700 .ssh
      retry ssh-keyscan -t rsa github.com >> .ssh/known_hosts

      # A read only key for cloning the repository.
      echo $GIT_CHECKOUT_KEY | base64 -d > .ssh/id_rsa

      chmod 0600 .ssh/id_rsa
      # IF YOU'RE CHANGING THIS, YOU ALSO WANT TO CHANGE: build-system/scripts/remote_runner
      # Shallow checkout this commit.
      mkdir -p project
      cd project
      git init
      git remote add origin $CIRCLE_REPOSITORY_URL

      # Only download metadata when fetching.
      retry git fetch --depth 1 --filter=blob:none origin $CIRCLE_SHA1
      retry git checkout FETCH_HEAD

# Called setup_env to setup a bunch of global variables used throughout the rest of the build process.
# It takes the required CCI environment variables as inputs, and gives them normalized names for the rest of
# the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds.
setup_env: &setup_env
  run:
    name: "Setup environment"
    command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST"

defaults_e2e_test: &defaults_e2e_test
  docker:
    - image: aztecprotocol/alpine-build-image
  resource_class: small

jobs:
  # Dynamically filter our code, quickly figuring out which jobs we can skip.
  generate-config:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: large
    steps:
      - *checkout
      - *setup_env
      - run:
          name: Generate Pipeline generated_config.yml file
          command: |
            # filter our circleci config to the minimal pipeline
            build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml
      - continuation/continue:
          configuration_path: .circleci/generated_config.yml

  # Barretenberg
  barretenberg-wasm-linux-clang:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build barretenberg-wasm-linux-clang 128
          aztec_manifest_key: barretenberg-wasm-linux-clang

  barretenberg-x86_64-linux-clang:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build barretenberg-x86_64-linux-clang 128
          aztec_manifest_key: barretenberg-x86_64-linux-clang

  # to be reenabled with https://github.com/AztecProtocol/aztec-packages/issues/6682
  # barretenberg-x86_64-linux-clang-fuzzing:
  #   docker:
  #     - image: aztecprotocol/alpine-build-image
  #   resource_class: small
  #   steps:
  #     - *checkout
  #     - *setup_env
  #     - run:
  #         name: "Build"
  #         command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 128
  #         aztec_manifest_key: barretenberg-x86_64-linux-clang-fuzzing

  barretenberg-x86_64-linux-clang-assert:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build barretenberg-x86_64-linux-clang-assert 128
          aztec_manifest_key: barretenberg-x86_64-linux-clang-assert

  barretenberg-x86_64-linux-clang-sol:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build barretenberg-x86_64-linux-clang-sol 32
          aztec_manifest_key: barretenberg-x86_64-linux-clang-sol

  barretenberg-docs:
    machine:
      image: default
    resource_class: large
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build barretenberg docs"
          command: build barretenberg-docs
          aztec_manifest_key: barretenberg-docs
      - run:
          name: "Deploy barretenberg docs"
          command: |
            barretenberg/cpp/scripts/ci/upload_doxygen_to_s3.sh

  bb-js:
    machine:
      image: default
    resource_class: large
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build and test"
          command: build bb.js
          aztec_manifest_key: bb.js

  # Noir
  noir-x86_64:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build noir 32
          aztec_manifest_key: noir

  noir-arm64:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build noir 32 arm64
          aztec_manifest_key: noir

  noir-ecr-manifest:
    machine:
      image: default
    resource_class: medium
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Create ECR manifest"
          command: create_ecr_manifest noir x86_64,arm64
          aztec_manifest_key: noir

  noir-packages:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build noir-packages 32
          aztec_manifest_key: noir-packages

  avm-transpiler:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build avm-transpiler 32
          aztec_manifest_key: avm-transpiler

  aztec-nargo:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build aztec-nargo 32
          aztec_manifest_key: aztec-nargo

  l1-contracts:
    machine:
      image: default
    resource_class: large
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build and test"
          command: build l1-contracts
          aztec_manifest_key: l1-contracts

  noir-projects:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build and test"
          command: cond_spot_run_build noir-projects 32
          aztec_manifest_key: noir-projects

  yarn-project-pre-join:
    docker:
      - image: cimg/base:2023.09
    resource_class: small
    steps:
      - run:
          name: "Noop"
          command: echo Noop

  yarn-project-x86_64:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build yarn-project 64
          aztec_manifest_key: yarn-project

  yarn-project-arm64:
    docker:
      - image: aztecprotocol/alpine-build-image
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: cond_spot_run_build yarn-project 64 arm64
          aztec_manifest_key: yarn-project

  yarn-project-ecr-manifest:
    machine:
      image: default
    resource_class: medium
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Create ECR manifest"
          command: create_ecr_manifest yarn-project x86_64,arm64
          aztec_manifest_key: yarn-project

  aztec-package:
    machine:
      image: default
    resource_class: large
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build and test"
          command: build aztec
          aztec_manifest_key: aztec

  aztec-builder:
    machine:
      image: default
    resource_class: small
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build image"
          command: build aztec-builder
          aztec_manifest_key: aztec-builder

  end-to-end:
    machine:
      image: default
    resource_class: large
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Build"
          command: build end-to-end
          aztec_manifest_key: end-to-end

  # For old e2e tests see yarn-project/end-to-end/Earthfile
  # Semantics are similar to Dockerfile

  e2e-join:
    docker:
      - image: cimg/base:2023.09
    resource_class: small
    steps:
      - run:
          name: "Noop"
          command: echo Noop

  end:
    docker:
      - image: cimg/base:2023.09
    resource_class: small
    steps:
      - run:
          name: "Noop"
          command: echo Noop

  # Deploy & release jobs.
  deploy-and-release:
    machine:
      image: default
    resource_class: medium
    steps:
      - *checkout
      - *setup_env
      - run:
          name: "Release to dockerhub"
          command: |
            should_release || exit 0
            deploy_dockerhub aztec-nargo
            deploy_dockerhub aztec
            deploy_dockerhub aztec-builder
      - run:
          name: "Release canary to NPM: bb.js"
          command: |
            should_release || exit 0
            deploy_npm bb.js canary
      - run:
          name: "Release canary to NPM: yarn-project"
          command: |
            should_release || exit 0
            yarn-project/deploy_npm.sh canary
      - run:
          name: "Release latest to NPM: bb.js"
          command: |
            should_release || exit 0
            deploy_npm bb.js latest
      - run:
          name: "Release latest to NPM: yarn-project"
          command: |
            should_release || exit 0
            yarn-project/deploy_npm.sh latest
      - run:
          name: "Release canary to NPM: l1-contracts"
          command: |
            should_release || exit 0
            deploy_npm l1-contracts canary
      - run:
          name: "Release latest to NPM: l1-contracts"
          command: |
            should_release || exit 0
            deploy_npm l1-contracts latest
      - run:
          name: "Update aztec-up"
          command: |
            should_release || exit 0
            deploy_terraform "" aztec-up/terraform

# Repeatable config for defining the workflow below.
defaults: &defaults
  filters:
    tags:
      only: /^aztec-packages-v.*/
    # We would like to ignore release-please branches,
    # but doing so breaks github status checks:
    # the PR spins as it waits for the status check to complete,
    # which never happens because the branch is ignored.
    # Long term solution would require disabling the status checks on `master`
    # when the *source branch* on the PR release-please.
    # branches:
    #   ignore:
    #     - /^release-please--.*/
  context:
    - build
    - slack
  post-steps:
    - slack/notify:
        event: fail
        branch_pattern: "master"

defaults_yarn_project_pre_join: &defaults_yarn_project_pre_join
  requires:
    - yarn-project-pre-join
  <<: *defaults

defaults_yarn_project: &defaults_yarn_project
  requires:
    - yarn-project-ecr-manifest
  <<: *defaults

defaults_deploy: &defaults_deploy
  requires:
    - end
  <<: *defaults

e2e_test: &e2e_test
  requires:
    - e2e-join
  <<: *defaults

bb_test: &bb_test
  requires:
    - barretenberg-x86_64-linux-clang-assert
  <<: *defaults

# Workflows.
workflows:
  # setup-workflow:
  #   jobs:
  #     - generate-config: *defaults
  system:
    # when:
    #   # Used to generate a dynamic 'system' workflow
    #   # This is rewritten to 'system' on the real workflow (otherwise this is ignored by circleci)
    #   equal: [NEVER, << pipeline.parameters.workflow >>]
    when:
      # Run if branch is master or branch name contains 'circle-ci'
      or:
        - equal: [ master, << pipeline.git.branch >> ]
        - matches: { pattern: ".*circle-ci.*", value: << pipeline.git.branch >> }
        - matches: { pattern: ".+", value: << pipeline.git.tag >> }
    jobs:
      # Noir
      - noir-x86_64: *defaults
      - noir-arm64: *defaults
      - noir-ecr-manifest:
          requires:
            - noir-x86_64
            - noir-arm64
          <<: *defaults
      - noir-packages:
          requires:
            - bb-js
          <<: *defaults

      # Transpiler
      - avm-transpiler:
          requires:
            - noir-ecr-manifest
          <<: *defaults

      # aztec-nargo (nargo & transpiler)
      - aztec-nargo:
          requires:
            - avm-transpiler
          <<: *defaults

      # Barretenberg
      - barretenberg-x86_64-linux-clang: *defaults
      - barretenberg-x86_64-linux-clang-assert: *defaults
      # - barretenberg-x86_64-linux-clang-fuzzing: *defaults
      - barretenberg-wasm-linux-clang: *defaults
      - barretenberg-docs: *defaults
      - bb-js:
          requires:
            - barretenberg-wasm-linux-clang
          <<: *defaults

      - l1-contracts: *defaults

      - noir-projects:
          requires:
            - avm-transpiler
            - noir-ecr-manifest
          <<: *defaults

      # Yarn Project
      - yarn-project-pre-join:
          requires:
            - bb-js
            - noir-packages
            - l1-contracts
            - noir-projects
          <<: *defaults
      - end-to-end: *defaults_yarn_project
      - yarn-project-x86_64: *defaults_yarn_project_pre_join
      - yarn-project-arm64: *defaults_yarn_project_pre_join
      - yarn-project-ecr-manifest:
          requires:
            - yarn-project-x86_64
            - yarn-project-arm64
          <<: *defaults

      # Artifacts
      - aztec-package: *defaults_yarn_project
      - aztec-builder: *defaults_yarn_project

      # End to end tests.
      - e2e-join:
          requires:
            - end-to-end
            - aztec-package
          <<: *defaults

      # Everything that must complete before deployment.
      - end:
          requires:
            - barretenberg-x86_64-linux-clang
            - barretenberg-x86_64-linux-clang-assert
            # - barretenberg-x86_64-linux-clang-fuzzing
            - barretenberg-wasm-linux-clang
            - barretenberg-docs
            - e2e-join
            - aztec-builder
          <<: *defaults

      # Production releases.
      - deploy-and-release: *defaults_deploy