Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .buildkite/pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ steps:
AMI_PUBLIC: true
agents:
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
if_changed: "packer/windows/base/**"
depends_on:
- "linting"
- "fixperms-tests"
Expand Down Expand Up @@ -138,6 +139,7 @@ steps:
AMI_PUBLIC: true
agents:
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
if_changed: "packer/linux/base/**"
depends_on:
- "linting"
- "fixperms-tests"
Expand Down Expand Up @@ -211,6 +213,7 @@ steps:
AMI_PUBLIC: true
agents:
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
if_changed: "packer/linux/base/**"
depends_on:
- "linting"
- "fixperms-tests"
Expand Down
11 changes: 0 additions & 11 deletions .buildkite/scripts/trigger-base-rebuild.sh

This file was deleted.

17 changes: 4 additions & 13 deletions .buildkite/steps/packer.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,7 @@ fi

os="${1:-linux}"
arch="${2:-amd64}"
variant="${3:-full}" # "full" (default) or "base"
agent_binary="buildkite-agent-${os}-${arch}"

if [[ "$os" == "windows" ]]; then
agent_binary+=".exe"
fi
variant="${3:-stack}" # "stack" (default) or "base"

mkdir -p "build/"

Expand All @@ -22,15 +17,9 @@ timestamp=$(date -u +"%Y%m%d-%H%M%S")
if [[ "${variant}" == "base" ]]; then
packer_file="packer-base-${os}-${arch}-${timestamp}.output"
local_output="packer-base-${os}-${arch}.output"
else
packer_file="packer-${os}-${arch}-${timestamp}.output"
local_output="packer-${os}-${arch}.output"
fi

if [[ "${variant}" == "base" ]]; then
make "packer-base-${os}-${arch}.output"
else
# Try metadata first, then S3 fallback for base AMI ID
# Get base AMI ID from metadata (set by base build step) or S3 fallback
base_ami_id="$(buildkite-agent meta-data get "${os}-base-${arch}-ami" || true)"

if [[ -z "$base_ami_id" ]]; then
Expand All @@ -50,6 +39,8 @@ else
exit 1
fi

packer_file="packer-${os}-${arch}-${timestamp}.output"
local_output="packer-${os}-${arch}.output"
make "packer-${os}-${arch}.output" BASE_AMI_ID="$base_ami_id"
fi

Expand Down
34 changes: 18 additions & 16 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ VERSION = $(shell git describe --tags --candidates=1)
SHELL = /bin/bash -o pipefail

PACKER_VERSION ?= 1.11.2
PACKER_LINUX_FILES = $(exec find packer/linux)
PACKER_WINDOWS_FILES = $(exec find packer/windows)
PACKER_LINUX_BASE_FILES = $(exec find packer/linux/base)
PACKER_LINUX_STACK_FILES = $(exec find packer/linux/stack)
PACKER_WINDOWS_BASE_FILES = $(exec find packer/windows/base)
PACKER_WINDOWS_STACK_FILES = $(exec find packer/windows/stack)

# Allow passing an existing golden base AMI into packer via `BASE_AMI_ID` env var
override BASE_AMI_ID ?=
Expand Down Expand Up @@ -105,7 +107,7 @@ build/linux-amd64-ami.txt: packer-linux-amd64.output env-AWS_REGION
grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@

# Build linux packer image
packer-linux-amd64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-amd64
packer-linux-amd64.output: $(PACKER_LINUX_STACK_FILES) build/fix-perms-linux-amd64
docker run \
-e AWS_DEFAULT_REGION \
-e AWS_PROFILE \
Expand All @@ -116,7 +118,7 @@ packer-linux-amd64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-amd64
-v ${HOME}/.aws:/root/.aws \
-v "$(PWD):/src" \
--rm \
-w /src/packer/linux \
-w /src/packer/linux/stack \
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
-var 'region=$(AWS_REGION)' \
-var 'arch=x86_64' \
Expand All @@ -133,15 +135,15 @@ build/linux-arm64-ami.txt: packer-linux-arm64.output env-AWS_REGION
grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@

# NOTE: make removes the $ escapes, everything else is passed to the shell
CURRENT_AGENT_VERSION_LINUX ?= $(shell $(SED) -En 's/^AGENT_VERSION="?(.+?)"?$$/\1/p' packer/linux/scripts/install-buildkite-agent.sh)
CURRENT_AGENT_VERSION_WINDOWS ?= $(shell $(SED) -En 's/^\$$AGENT_VERSION = "(.+?)"$$/\1/p' packer/windows/scripts/install-buildkite-agent.ps1)
CURRENT_AGENT_VERSION_LINUX ?= $(shell $(SED) -En 's/^AGENT_VERSION="?(.+?)"?$$/\1/p' packer/linux/stack/scripts/install-buildkite-agent.sh)
CURRENT_AGENT_VERSION_WINDOWS ?= $(shell $(SED) -En 's/^\$$AGENT_VERSION = "(.+?)"$$/\1/p' packer/windows/stack/scripts/install-buildkite-agent.ps1)

print-agent-versions:
@echo Linux: $(CURRENT_AGENT_VERSION_LINUX)
@echo Windows: $(CURRENT_AGENT_VERSION_WINDOWS)

# Build linuxarm64 packer image
packer-linux-arm64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-arm64
packer-linux-arm64.output: $(PACKER_LINUX_STACK_FILES) build/fix-perms-linux-arm64
@echo Agent Version: $(CURRENT_AGENT_VERSION_LINUX)
docker run \
-e AWS_DEFAULT_REGION \
Expand All @@ -153,7 +155,7 @@ packer-linux-arm64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-arm64
-v ${HOME}/.aws:/root/.aws \
-v "$(PWD):/src" \
--rm \
-w /src/packer/linux \
-w /src/packer/linux/stack \
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
-var 'region=$(AWS_REGION)' \
-var 'arch=arm64' \
Expand All @@ -171,7 +173,7 @@ build/windows-amd64-ami.txt: packer-windows-amd64.output env-AWS_REGION
grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@

# Build windows packer image
packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
packer-windows-amd64.output: $(PACKER_WINDOWS_STACK_FILES)
@echo Agent Version: $(CURRENT_AGENT_VERSION_WINDOWS)
docker run \
-e AWS_DEFAULT_REGION \
Expand All @@ -183,7 +185,7 @@ packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
-v ${HOME}/.aws:/root/.aws \
-v "$(PWD):/src" \
--rm \
-w /src/packer/windows \
-w /src/packer/windows/stack \
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
-var 'region=$(AWS_REGION)' \
-var 'arch=x86_64' \
Expand All @@ -200,7 +202,7 @@ packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
# Base AMI creation

# Build base AMI for linux amd64
packer-base-linux-amd64.output: $(PACKER_LINUX_FILES)
packer-base-linux-amd64.output: $(PACKER_LINUX_BASE_FILES)
docker run \
-e AWS_DEFAULT_REGION \
-e AWS_PROFILE \
Expand All @@ -211,7 +213,7 @@ packer-base-linux-amd64.output: $(PACKER_LINUX_FILES)
-v ${HOME}/.aws:/root/.aws \
-v "$(PWD):/src" \
--rm \
-w /src/packer/linux \
-w /src/packer/linux/base \
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
-var 'region=$(AWS_REGION)' \
-var 'arch=x86_64' \
Expand All @@ -221,7 +223,7 @@ packer-base-linux-amd64.output: $(PACKER_LINUX_FILES)
base.pkr.hcl | tee $@

# Build base AMI for linux arm64
packer-base-linux-arm64.output: $(PACKER_LINUX_FILES)
packer-base-linux-arm64.output: $(PACKER_LINUX_BASE_FILES)
docker run \
-e AWS_DEFAULT_REGION \
-e AWS_PROFILE \
Expand All @@ -232,7 +234,7 @@ packer-base-linux-arm64.output: $(PACKER_LINUX_FILES)
-v ${HOME}/.aws:/root/.aws \
-v "$(PWD):/src" \
--rm \
-w /src/packer/linux \
-w /src/packer/linux/base \
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
-var 'region=$(AWS_REGION)' \
-var 'arch=arm64' \
Expand All @@ -242,7 +244,7 @@ packer-base-linux-arm64.output: $(PACKER_LINUX_FILES)
base.pkr.hcl | tee $@

# Build base AMI for windows amd64
packer-base-windows-amd64.output: $(PACKER_WINDOWS_FILES)
packer-base-windows-amd64.output: $(PACKER_WINDOWS_BASE_FILES)
docker run \
-e AWS_DEFAULT_REGION \
-e AWS_PROFILE \
Expand All @@ -253,7 +255,7 @@ packer-base-windows-amd64.output: $(PACKER_WINDOWS_FILES)
-v ${HOME}/.aws:/root/.aws \
-v "$(PWD):/src" \
--rm \
-w /src/packer/windows \
-w /src/packer/windows/base \
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
-var 'region=$(AWS_REGION)' \
-var 'arch=x86_64' \
Expand Down
1 change: 0 additions & 1 deletion packer/linux/.trigger-base-build

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ build {
source = "conf"
}

provisioner "file" {
destination = "/tmp/conf"
source = "../shared/conf/"
}

# Essential utilities & updates
provisioner "shell" {
Expand All @@ -109,6 +113,6 @@ build {

# Clean up
provisioner "shell" {
script = "scripts/cleanup.sh"
script = "../shared/scripts/cleanup.sh"
}
}
7 changes: 7 additions & 0 deletions packer/linux/base/scripts/install-cloudwatch-agent.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail

echo "Installing cloudwatch agent..."
sudo dnf install -yq amazon-cloudwatch-agent

echo "CloudWatch agent installed. Configuration will be done in stack layer."
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,6 @@ sudo curl -L "https://github.com/goss-org/goss/releases/download/${GOSS_VERSION}
sudo chmod +rx /usr/local/bin/goss
sudo curl -L "https://github.com/goss-org/goss/releases/download/${GOSS_VERSION}/dgoss" -o /usr/local/bin/dgoss
sudo chmod +rx /usr/local/bin/dgoss

echo "Adding authorized keys systemd units..."
sudo cp /tmp/conf/ssh/systemd/* /etc/systemd/system
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -114,17 +114,26 @@ build {
source = "conf"
}

provisioner "file" {
destination = "/tmp/conf"
source = "../shared/conf/"
}

provisioner "file" {
destination = "/tmp/plugins"
source = "../../plugins"
source = "../../../plugins"
}

provisioner "file" {
destination = "/tmp/build"
source = "../../build"
source = "../../../build"
}


provisioner "shell" {
script = "scripts/configure-cloudwatch-agent.sh"
}

provisioner "shell" {
script = "scripts/install-buildkite-agent.sh"
}
Expand All @@ -134,6 +143,6 @@ build {
}

provisioner "shell" {
script = "scripts/cleanup.sh"
script = "../shared/scripts/cleanup.sh"
}
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
#!/usr/bin/env bash
set -euo pipefail

echo "Installing cloudwatch agent..."
sudo dnf install -yq amazon-cloudwatch-agent
echo "Configuring cloudwatch agent..."

echo "Adding amazon-cloudwatch-agent config..."
sudo cp /tmp/conf/cloudwatch-agent/config.json /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
sudo cp /tmp/conf/cloudwatch-agent/amazon-cloudwatch-agent.json /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json

echo "Configuring amazon-cloudwatch-agent to start at boot"
sudo systemctl enable amazon-cloudwatch-agent
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,3 @@ sudo curl -Lf -o /usr/bin/lifecycled \
sudo chmod +x /usr/bin/lifecycled
sudo curl -Lf -o /etc/systemd/system/lifecycled.service \
https://raw.githubusercontent.com/buildkite/lifecycled/${LIFECYCLED_VERSION}/init/systemd/lifecycled.unit

echo "Adding authorized keys systemd units..."
sudo cp /tmp/conf/ssh/systemd/* /etc/systemd/system
1 change: 0 additions & 1 deletion packer/windows/.trigger-base-build

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ source "amazon-ebs" "buildkite-base" {
instance_type = var.instance_type
region = var.region
source_ami = data.amazon-ami.windows-server-2022.id
user_data_file = "scripts/ec2-userdata.ps1"
user_data_file = "../shared/scripts/ec2-userdata.ps1"
winrm_insecure = true
winrm_use_ssl = true
winrm_port = 5986
Expand All @@ -78,11 +78,6 @@ source "amazon-ebs" "buildkite-base" {
build {
sources = ["source.amazon-ebs.buildkite-base"]

provisioner "file" {
destination = "C:/packer-temp"
source = "conf"
}

provisioner "file" {
destination = "C:/packer-temp"
source = "scripts"
Expand Down
15 changes: 15 additions & 0 deletions packer/windows/base/scripts/install-cloudwatch-agent.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Stop script execution when a non-terminating error occurs
$ErrorActionPreference = "Stop"

Write-Output "Downloading amazon cloudwatch agent..."
Invoke-WebRequest -OutFile C:\packer-temp\amazon-cloudwatch-agent.msi -Uri "https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi"

Write-Output "Installing amazon cloudwatch agent..."
$process = Start-Process msiexec -ArgumentList "/i", "C:\packer-temp\amazon-cloudwatch-agent.msi", "/quiet", "/norestart" -Wait -PassThru
If ($process.ExitCode -ne 0) { Exit $process.ExitCode }

Write-Output "Setting amazon cloudwatch agent start type to manual (will be configured in stack layer)..."
sc.exe config AmazonCloudWatchAgent start= demand
If ($lastexitcode -ne 0) { Exit $lastexitcode }

Write-Output "CloudWatch agent installed. Configuration will be done in stack layer."
18 changes: 0 additions & 18 deletions packer/windows/scripts/install-cloudwatch-agent.ps1

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ source "amazon-ebs" "elastic-ci-stack" {
region = var.region
# Allow golden-base override
source_ami = var.base_ami_id
user_data_file = "scripts/ec2-userdata.ps1"
user_data_file = "../shared/scripts/ec2-userdata.ps1"
winrm_insecure = true
winrm_use_ssl = true
winrm_port = 5986
Expand Down Expand Up @@ -115,11 +115,12 @@ build {

provisioner "file" {
destination = "C:/packer-temp"
source = "../../plugins"
source = "../../../plugins"
}

provisioner "powershell" {
scripts = [
"scripts/configure-cloudwatch-agent.ps1",
"scripts/install-buildkite-agent.ps1",
"scripts/install-s3secrets-helper.ps1"
]
Expand Down
10 changes: 10 additions & 0 deletions packer/windows/stack/scripts/configure-cloudwatch-agent.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Stop script execution when a non-terminating error occurs
$ErrorActionPreference = "Stop"

Write-Output "Configuring CloudWatch agent..."

Write-Output "Copying amazon cloudwatch agent config..."
Copy-Item -Path C:\packer-temp\conf\cloudwatch-agent\amazon-cloudwatch-agent.json -Destination C:\ProgramData\Amazon\AmazonCloudWatchAgent

Write-Output "Starting amazon cloudwatch agent..."
& 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -c file:C:\ProgramData\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent.json -s