diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 313694835..b0e2b3d95 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,56 +1,5 @@ dag: true -x-if-changed: - base-windows: &if_changed_base_windows - - "packer/windows/base/**" - - base-linux: &if_changed_base_linux - - "packer/linux/base/**" - - stack-windows: &if_changed_stack_windows - - "packer/windows/stack/**" - - "packer/windows/base/**" - - "plugins/**" - - stack-linux: &if_changed_stack_linux - - "packer/linux/stack/**" - - "packer/linux/shared/**" - - "packer/linux/base/**" - - "plugins/**" - - "build/**" - - test-windows: &if_changed_test_windows - - "packer/windows/stack/**" - - "packer/windows/base/**" - - "plugins/**" - - "templates/**" - - test-linux: &if_changed_test_linux - - "packer/linux/stack/**" - - "packer/linux/shared/**" - - "packer/linux/base/**" - - "plugins/**" - - "build/**" - - "templates/**" - - ami-all: &if_changed_ami_all - - "packer/windows/stack/**" - - "packer/windows/base/**" - - "packer/linux/stack/**" - - "packer/linux/shared/**" - - "packer/linux/base/**" - - "plugins/**" - - "build/**" - - publish-all: &if_changed_publish_all - - "packer/windows/stack/**" - - "packer/windows/base/**" - - "packer/linux/stack/**" - - "packer/linux/shared/**" - - "packer/linux/base/**" - - "plugins/**" - - "build/**" - steps: - group: ":lint-roller: Linting" key: linting @@ -86,10 +35,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: - - "internal/fixperms/**" - - "go.mod" - - "go.sum" plugins: - docker-compose#v5.12.0: run: fixperms-tests @@ -101,7 +46,6 @@ steps: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" command: .buildkite/steps/deploy-service-role-stack.sh if: build.source != "schedule" - if_changed: *if_changed_publish_all depends_on: - "linting" - "fixperms-tests" @@ -122,7 +66,7 @@ steps: AMI_PUBLIC: false agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" - if_changed: *if_changed_base_windows + if_changed: "packer/windows/base/**" depends_on: - "linting" - "fixperms-tests" @@ -139,7 +83,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: *if_changed_stack_windows depends_on: - "packer-base-windows-amd64" - "linting" @@ -147,20 +90,6 @@ steps: plugins: - *aws_role_plugin - - id: "ensure-ami-windows-amd64" - name: ":mag: Ensure Windows AMD64 AMI" - command: .buildkite/steps/ensure_ami_metadata.py windows amd64 - agents: - queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" - if: build.source != "schedule" - if_changed: *if_changed_test_windows - depends_on: - - "packer-windows-amd64" - - "linting" - - "fixperms-tests" - plugins: - - *aws_role_plugin - - id: "launch-windows-amd64" name: ":cloudformation: :windows: AMD64 Launch" command: .buildkite/steps/launch.sh windows amd64 @@ -168,9 +97,8 @@ steps: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" artifact_paths: "build/aws-stack.yml" if: build.source != "schedule" - if_changed: *if_changed_test_windows depends_on: - - "ensure-ami-windows-amd64" + - "packer-windows-amd64" - "deploy-service-role-stack" plugins: - *aws_role_plugin @@ -188,7 +116,6 @@ steps: stack: "buildkite-aws-stack-test-windows-amd64-${BUILDKITE_BUILD_NUMBER}" queue: "testqueue-windows-amd64-${BUILDKITE_BUILD_NUMBER}" if: build.source != "schedule" - if_changed: *if_changed_test_windows depends_on: - "launch-windows-amd64" @@ -199,7 +126,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: *if_changed_test_windows depends_on: - "test-windows-amd64" plugins: @@ -213,7 +139,7 @@ steps: AMI_PUBLIC: false agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" - if_changed: *if_changed_base_linux + if_changed: "packer/linux/base/**" depends_on: - "linting" - "fixperms-tests" @@ -230,7 +156,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: *if_changed_stack_linux depends_on: - "packer-base-linux-amd64" - "linting" @@ -238,20 +163,6 @@ steps: plugins: - *aws_role_plugin - - id: "ensure-ami-linux-amd64" - name: ":mag: Ensure Linux AMD64 AMI" - command: .buildkite/steps/ensure_ami_metadata.py linux amd64 - agents: - queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" - if: build.source != "schedule" - if_changed: *if_changed_test_linux - depends_on: - - "packer-linux-amd64" - - "linting" - - "fixperms-tests" - plugins: - - *aws_role_plugin - - id: "launch-linux-amd64" name: ":cloudformation: :linux: AMD64 Launch" command: .buildkite/steps/launch.sh linux @@ -260,9 +171,8 @@ steps: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" artifact_paths: "build/aws-stack.yml" if: build.source != "schedule" - if_changed: *if_changed_test_linux depends_on: - - "ensure-ami-linux-amd64" + - "packer-linux-amd64" - "deploy-service-role-stack" plugins: - *aws_role_plugin @@ -280,7 +190,6 @@ steps: stack: "buildkite-aws-stack-test-linux-amd64-${BUILDKITE_BUILD_NUMBER}" queue: "testqueue-linux-amd64-${BUILDKITE_BUILD_NUMBER}" if: build.source != "schedule" - if_changed: *if_changed_test_linux depends_on: - "launch-linux-amd64" @@ -291,7 +200,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: *if_changed_test_linux depends_on: - "test-linux-amd64" plugins: @@ -305,7 +213,7 @@ steps: AMI_PUBLIC: false agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" - if_changed: *if_changed_base_linux + if_changed: "packer/linux/base/**" depends_on: - "linting" - "fixperms-tests" @@ -322,7 +230,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: *if_changed_stack_linux depends_on: - "packer-base-linux-arm64" - "linting" @@ -330,20 +237,6 @@ steps: plugins: - *aws_role_plugin - - id: "ensure-ami-linux-arm64" - name: ":mag: Ensure Linux ARM64 AMI" - command: .buildkite/steps/ensure_ami_metadata.py linux arm64 - agents: - queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" - if: build.source != "schedule" - if_changed: *if_changed_test_linux - depends_on: - - "packer-linux-arm64" - - "linting" - - "fixperms-tests" - plugins: - - *aws_role_plugin - - id: "launch-linux-arm64" name: ":cloudformation: :linux: ARM64 Launch" command: .buildkite/steps/launch.sh linux arm64 @@ -351,9 +244,8 @@ steps: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" artifact_paths: "build/aws-stack.yml" if: build.source != "schedule" - if_changed: *if_changed_test_linux depends_on: - - "ensure-ami-linux-arm64" + - "packer-linux-arm64" - "deploy-service-role-stack" plugins: - *aws_role_plugin @@ -371,7 +263,6 @@ steps: stack: "buildkite-aws-stack-test-linux-arm64-${BUILDKITE_BUILD_NUMBER}" queue: "testqueue-linux-arm64-${BUILDKITE_BUILD_NUMBER}" if: build.source != "schedule" - if_changed: *if_changed_test_linux depends_on: - "launch-linux-arm64" @@ -382,7 +273,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: *if_changed_test_linux depends_on: - "test-linux-arm64" plugins: @@ -394,7 +284,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: *if_changed_publish_all depends_on: - "delete-windows-amd64" - "delete-linux-amd64" @@ -409,7 +298,6 @@ steps: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" artifact_paths: "build/mappings.yml" if: build.source != "schedule" - if_changed: *if_changed_ami_all depends_on: - "test-linux-amd64" - "test-linux-arm64" @@ -427,7 +315,6 @@ steps: concurrency_method: eager artifact_paths: "build/*.yml" if: build.source != "schedule" - if_changed: *if_changed_publish_all depends_on: "copy-ami" plugins: - *aws_role_plugin @@ -438,7 +325,6 @@ steps: agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" if: build.source != "schedule" - if_changed: *if_changed_publish_all depends_on: "publish" plugins: - *aws_role_plugin diff --git a/.buildkite/steps/ensure_ami_metadata.py b/.buildkite/steps/ensure_ami_metadata.py deleted file mode 100755 index 0d9190e85..000000000 --- a/.buildkite/steps/ensure_ami_metadata.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python3 -""" -Ensure AMI metadata is set for Stack AMI builds. - -This script checks if the packer build step set AMI metadata. If not, -it fetches the AMI ID from the main branch CloudFormation template, -which happens when the build was skipped due to if_changed conditions. -""" - -import os -import re -import subprocess -import sys -import urllib.request -from typing import Optional - - -def get_metadata(key: str) -> Optional[str]: - """Get metadata from Buildkite agent, return None if not found.""" - try: - result = subprocess.run( - ["buildkite-agent", "meta-data", "get", key], - capture_output=True, - text=True, - check=False, - ) - if result.returncode == 0 and result.stdout.strip(): - return result.stdout.strip() - except FileNotFoundError: - print("Warning: buildkite-agent not found", file=sys.stderr) - return None - - -def set_metadata(key: str, value: str) -> None: - """Set metadata in Buildkite agent.""" - try: - subprocess.run( - ["buildkite-agent", "meta-data", "set", key, value], - check=True, - capture_output=True, - text=True, - ) - except subprocess.CalledProcessError as e: - print(f"ERROR: Failed to set metadata: {e.stderr}", file=sys.stderr) - raise - except FileNotFoundError: - print("ERROR: buildkite-agent not found", file=sys.stderr) - raise - - -def fetch_ami_from_template(os_type: str, arch: str, region: str) -> str: - """ - Fetch AMI ID from the main branch CloudFormation template. - - Args: - os_type: Operating system (linux or windows) - arch: Architecture (amd64 or arm64) - region: AWS region - - Returns: - AMI ID string - - Raises: - RuntimeError: If AMI cannot be found - """ - template_url = "https://s3.amazonaws.com/buildkite-aws-stack/main/aws-stack.yml" - - print(f"--- Fetching AMI ID from main branch template for {os_type}/{arch}") - - try: - with urllib.request.urlopen(template_url) as response: - template_content = response.read().decode("utf-8") - except Exception as e: - raise RuntimeError( - f"Failed to download main branch template from {template_url}: {e}" - ) from e - - if os_type == "windows": - key_name = "windows" - elif arch == "arm64": - key_name = "linuxarm64" - else: - key_name = "linuxamd64" - - # Template format: " us-east-1: { linuxamd64: ami-xxx, linuxarm64: ami-yyy, windows: ami-zzz }" - pattern = rf"^\s+{re.escape(region)}\s*:.*{key_name}:\s*(ami-[a-z0-9]+)" - - for line in template_content.split("\n"): - match = re.search(pattern, line) - if match: - ami_id = match.group(1) - print(f"Found AMI ID: {ami_id}") - return ami_id - - raise RuntimeError( - f"Could not find AMI ID for region {region}, os {os_type}, arch {arch} in main template" - ) - - -def ensure_ami_metadata(os_type: str, arch: str) -> None: - """ - Ensure AMI metadata is set, fetching from template if necessary. - - Args: - os_type: Operating system (linux or windows) - arch: Architecture (amd64 or arm64) - """ - metadata_key = f"{os_type}_{arch}_image_id" - - existing_ami = get_metadata(metadata_key) - if existing_ami: - print(f"AMI metadata already set: {existing_ami}") - return - - region = os.environ.get("AWS_REGION") - if not region: - raise RuntimeError("AWS_REGION environment variable not set") - - print("AMI metadata not found, fetching from main branch template...") - ami_id = fetch_ami_from_template(os_type, arch, region) - - set_metadata(metadata_key, ami_id) - print(f"Set AMI metadata: {metadata_key}={ami_id}") - - -def main() -> int: - """Main entry point.""" - if len(sys.argv) != 3: - print(f"Usage: {sys.argv[0]} ", file=sys.stderr) - print(" os: linux or windows", file=sys.stderr) - print(" arch: amd64 or arm64", file=sys.stderr) - return 1 - - os_type = sys.argv[1] - arch = sys.argv[2] - - if os_type not in ("linux", "windows"): - print(f"ERROR: Invalid OS '{os_type}', must be 'linux' or 'windows'", file=sys.stderr) - return 1 - - if arch not in ("amd64", "arm64"): - print(f"ERROR: Invalid arch '{arch}', must be 'amd64' or 'arm64'", file=sys.stderr) - return 1 - - try: - ensure_ami_metadata(os_type, arch) - return 0 - except Exception as e: - print(f"ERROR: {e}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/packer/linux/stack/buildkite-ami.pkr.hcl b/packer/linux/stack/buildkite-ami.pkr.hcl index 3c406dbe2..69451b676 100644 --- a/packer/linux/stack/buildkite-ami.pkr.hcl +++ b/packer/linux/stack/buildkite-ami.pkr.hcl @@ -150,4 +150,3 @@ build { script = "../shared/scripts/cleanup.sh" } } - diff --git a/packer/windows/stack/buildkite-ami.pkr.hcl b/packer/windows/stack/buildkite-ami.pkr.hcl index e3a1832c9..9e2908cb8 100644 --- a/packer/windows/stack/buildkite-ami.pkr.hcl +++ b/packer/windows/stack/buildkite-ami.pkr.hcl @@ -139,4 +139,3 @@ build { ] } } -