From 15a25f8aecb0e096d03a18ed813708713bbd0175 Mon Sep 17 00:00:00 2001 From: Lucas Pierrat Date: Mon, 15 Sep 2025 13:57:06 +0200 Subject: [PATCH] add investigation pipelines --- .../flooding-one-worker/README.md | 15 ++ .../flooding-one-worker/docker-compose.yml | 150 ++++++++++++++ .../flooding-one-worker/pipeline.yml | 18 ++ .../flooding-one-worker/prometheus.yml | 40 ++++ .../flooding-one-worker/script.sh | 11 ++ .../two-workers-with-flock/README.md | 27 +++ .../two-workers-with-flock/docker-compose.yml | 184 ++++++++++++++++++ .../two-workers-with-flock/pipeline.yml | 71 +++++++ .../two-workers-with-flock/prometheus.yml | 40 ++++ 9 files changed, 556 insertions(+) create mode 100644 pci-4464-investigation/flooding-one-worker/README.md create mode 100644 pci-4464-investigation/flooding-one-worker/docker-compose.yml create mode 100644 pci-4464-investigation/flooding-one-worker/pipeline.yml create mode 100644 pci-4464-investigation/flooding-one-worker/prometheus.yml create mode 100644 pci-4464-investigation/flooding-one-worker/script.sh create mode 100644 pci-4464-investigation/two-workers-with-flock/README.md create mode 100644 pci-4464-investigation/two-workers-with-flock/docker-compose.yml create mode 100644 pci-4464-investigation/two-workers-with-flock/pipeline.yml create mode 100644 pci-4464-investigation/two-workers-with-flock/prometheus.yml diff --git a/pci-4464-investigation/flooding-one-worker/README.md b/pci-4464-investigation/flooding-one-worker/README.md new file mode 100644 index 0000000..e8babc7 --- /dev/null +++ b/pci-4464-investigation/flooding-one-worker/README.md @@ -0,0 +1,15 @@ +# How To + +1. Deploy the stack: +``` +docker compose -f docker-compose.yml -f prometheus.yml up +``` + +2. Run the script +``` +./script.sh +``` + +# Results + +If two (or more) tasks lands on the same worker at the same time, it is a big. diff --git a/pci-4464-investigation/flooding-one-worker/docker-compose.yml b/pci-4464-investigation/flooding-one-worker/docker-compose.yml new file mode 100644 index 0000000..f728c2f --- /dev/null +++ b/pci-4464-investigation/flooding-one-worker/docker-compose.yml @@ -0,0 +1,150 @@ +# NOTES +# - Some environment variables are actually set in the file `.env` in the same directory. +# This allows, if you wish, to change some values and not having to modify this file. +# See https://docs.docker.com/compose/environment-variables/ for details. +# - The funky notation &foo and *foo are YAML anchors and references, used to keep the +# configuration DRYer. See for example +# https://docs.ansible.com/ansible/latest/user_guide/playbooks_advanced_syntax.html + + +services: + minio: + image: minio/minio + command: server /minio-storage --console-address ":9001" + ports: + - 9000:9000 + - 9001:9001 + environment: + MINIO_REGION_NAME: &minio-region ${MINIO_REGION_NAME} + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + + # This container will stop as soon as the script has run. + minio-setup: + image: minio/mc + depends_on: + - minio + volumes: + - "./scripts:/scripts" + environment: + MINIO_ADDR: &minio-addr http://minio:9000 + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + entrypoint: /scripts/minio-setup.sh + + vault: + image: vault:1.13.1 + command: server -dev + ports: + - 8200:8200 + environment: + VAULT_DEV_ROOT_TOKEN_ID: ${VAULT_DEV_ROOT_TOKEN_ID} + cap_add: + - IPC_LOCK + + # This container will stop as soon as the script has run. + vault-setup: + image: vault:1.13.1 + depends_on: + - vault + volumes: + - "./scripts:/scripts" + environment: + VAULT_ADDR: &vault-addr http://vault:8200 + # NOTE In production you would NOT pass the root token! + VAULT_DEV_ROOT_TOKEN_ID: ${VAULT_DEV_ROOT_TOKEN_ID} + MINIO_ADDR : *minio-addr + MINIO_REGION_NAME: *minio-region + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + + entrypoint: /scripts/vault-setup.sh + + db: + image: postgres:14.11 + shm_size: 1gb + ports: [6543:5432] + environment: + POSTGRES_DB: &postgres-db concourse + POSTGRES_USER: &postgres-user dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + + web: + image: &concourse-image concourse/concourse:7.13.2 + command: web + depends_on: + - db + - minio-setup + - vault-setup + ports: [8080:8080] + volumes: + - "./concourse-keys:/concourse-keys" + environment: + CONCOURSE_SESSION_SIGNING_KEY: /concourse-keys/session_signing_key + CONCOURSE_TSA_AUTHORIZED_KEYS: /concourse-keys/authorized_worker_keys + CONCOURSE_TSA_HOST_KEY: /concourse-keys/tsa_host_key + + CONCOURSE_LOG_LEVEL: error + CONCOURSE_TSA_LOG_LEVEL: error + CONCOURSE_POSTGRES_HOST: db + CONCOURSE_POSTGRES_USER: *postgres-user + CONCOURSE_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + CONCOURSE_POSTGRES_DATABASE: *postgres-db + CONCOURSE_EXTERNAL_URL: http://localhost:8080 + CONCOURSE_ADD_LOCAL_USER: ${CONCOURSE_MAIN_USER}:${CONCOURSE_MAIN_PASSWORD},${CONCOURSE_NORMAL_USER}:${CONCOURSE_NORMAL_PASSWORD},${CONCOURSE_READONLY_USER}:${CONCOURSE_READONLY_PASSWORD} + CONCOURSE_MAIN_TEAM_LOCAL_USER: ${CONCOURSE_MAIN_USER} + CONCOURSE_CLUSTER_NAME: dev + CONCOURSE_ENABLE_PIPELINE_INSTANCES: "true" + CONCOURSE_ENABLE_ACROSS_STEP: "true" + # NOTE this connection is unencrypted! + CONCOURSE_VAULT_URL: *vault-addr + # NOTE In production you would NOT pass the root token! + CONCOURSE_VAULT_CLIENT_TOKEN: ${VAULT_DEV_ROOT_TOKEN_ID} + CONCOURSE_ENABLE_CACHE_STREAMED_VOLUMES: "true" + CONCOURSE_ENABLE_RESOURCE_CAUSALITY: "true" + CONCOURSE_CONTAINER_PLACEMENT_STRATEGY: limit-active-tasks,volume-locality + CONCOURSE_MAX_ACTIVE_TASKS_PER_WORKER: 1 + + # This container will stop as soon as the script has run. + web-setup: + image: alpine/curl + depends_on: + - web + volumes: + - "./scripts:/scripts" + environment: + CONCOURSE_ADDR: web:8080 + + entrypoint: /scripts/concourse-setup.sh + + worker: + image: *concourse-image + command: worker + privileged: true + depends_on: [web] + ports: + - 7777:7777 + - 7788:7788 + volumes: + - "./concourse-keys:/concourse-keys" + stop_signal: SIGUSR2 + environment: + CONCOURSE_RUNTIME: containerd + CONCOURSE_NAME: worker-1 + + CONCOURSE_TSA_PUBLIC_KEY: /concourse-keys/tsa_host_key.pub + CONCOURSE_TSA_WORKER_PRIVATE_KEY: /concourse-keys/worker_key + + CONCOURSE_LOG_LEVEL: error + CONCOURSE_TSA_HOST: web:2222 + + CONCOURSE_BAGGAGECLAIM_LOG_LEVEL: error + CONCOURSE_CONTAINERD_LOG_LEVEL: error + CONCOURSE_BIND_IP: 0.0.0.0 + CONCOURSE_BAGGAGECLAIM_BIND_IP: 0.0.0.0 + + # avoid using loopbacks + CONCOURSE_BAGGAGECLAIM_DRIVER: overlay + + # work with docker-compose's dns + CONCOURSE_CONTAINERD_DNS_PROXY_ENABLE: "true" diff --git a/pci-4464-investigation/flooding-one-worker/pipeline.yml b/pci-4464-investigation/flooding-one-worker/pipeline.yml new file mode 100644 index 0000000..51de7e2 --- /dev/null +++ b/pci-4464-investigation/flooding-one-worker/pipeline.yml @@ -0,0 +1,18 @@ +--- +jobs: +- name: loop + plan: + - task: loop + config: + platform: linux + image_resource: + type: registry-image + source: + repository: busybox + run: + path: sh + args: + - -c + - | + echo "Executing on worker: `hostname`" + for i in `seq 1 30`; do sleep 1; echo "Slept for ${i} seconds."; done diff --git a/pci-4464-investigation/flooding-one-worker/prometheus.yml b/pci-4464-investigation/flooding-one-worker/prometheus.yml new file mode 100644 index 0000000..8bd2933 --- /dev/null +++ b/pci-4464-investigation/flooding-one-worker/prometheus.yml @@ -0,0 +1,40 @@ +# prometheus.yml - a docker-compose override that adds 'prometheus' to the stack +# +# once running, head to `localhost:9090` to get access to the Prometheus console. +# +# ref: https://prometheus.io/ +# ref: https://docs.docker.com/compose/extends/ +# +version: '3' + +services: + web: + environment: + CONCOURSE_PROMETHEUS_BIND_IP: "0.0.0.0" + CONCOURSE_PROMETHEUS_BIND_PORT: "9100" + ports: + - '9100:9100' + + prometheus: + image: prom/prometheus + entrypoint: + - /bin/sh + - -c + - | + echo " + global: + scrape_interval: '1s' + evaluation_interval: '1s' + + scrape_configs: + - job_name: 'concourse' + static_configs: + - targets: + - 'web:9100' + " > config.yml + + exec prometheus \ + --config.file=config.yml \ + --storage.tsdb.path=/prometheus + ports: + - '9090:9090' diff --git a/pci-4464-investigation/flooding-one-worker/script.sh b/pci-4464-investigation/flooding-one-worker/script.sh new file mode 100644 index 0000000..65e10bd --- /dev/null +++ b/pci-4464-investigation/flooding-one-worker/script.sh @@ -0,0 +1,11 @@ +#!/bin/bash -ex + +for i in `seq 1 60`; do fly -t ci sp -p parallel${i}-second-linux -c pipeline.yml -n && fly -t ci up -p parallel${i}-second-linux; done + +task(){ + fly -t ci tj -j parallel${i}-second-linux/loop; +} + +for i in `seq 1 60`; do + task "$i" & +done diff --git a/pci-4464-investigation/two-workers-with-flock/README.md b/pci-4464-investigation/two-workers-with-flock/README.md new file mode 100644 index 0000000..acd9d94 --- /dev/null +++ b/pci-4464-investigation/two-workers-with-flock/README.md @@ -0,0 +1,27 @@ +# How To + +1. Deploy the stack: +``` +docker compose -f docker-compose.yml -f prometheus.yml up +``` + +2. Generate a big file +`dd bs=1024 count=1048576 bigfile-1` + +3. Upload the file to minio + - connect to minio http://localhost:9000 with user minio and password in .env + - upload bigfile-1 to concourse bucket + +4. Set the pipeline and trigger job1 +``` +$ fly --target=ci login --concourse-url=http://localhost:8080 --open-browser +$ fly -t ci sp -p two-workers-with-flock -c pipeline.yml +$ fly -t ci up -p two-workers-with-flock +$ fly -t ci tj -j two-workers-with-flock/job1 +``` + +5. Look at the pipeline, when job1/use-big-file is streaming the file, trigger job2 + +# Results + +If two (or more) tasks lands on the same worker at the same time, it is a big. diff --git a/pci-4464-investigation/two-workers-with-flock/docker-compose.yml b/pci-4464-investigation/two-workers-with-flock/docker-compose.yml new file mode 100644 index 0000000..d4b2ed0 --- /dev/null +++ b/pci-4464-investigation/two-workers-with-flock/docker-compose.yml @@ -0,0 +1,184 @@ +# NOTES +# - Some environment variables are actually set in the file `.env` in the same directory. +# This allows, if you wish, to change some values and not having to modify this file. +# See https://docs.docker.com/compose/environment-variables/ for details. +# - The funky notation &foo and *foo are YAML anchors and references, used to keep the +# configuration DRYer. See for example +# https://docs.ansible.com/ansible/latest/user_guide/playbooks_advanced_syntax.html + + +services: + minio: + image: minio/minio + command: server /minio-storage --console-address ":9001" + ports: + - 9000:9000 + - 9001:9001 + environment: + MINIO_REGION_NAME: &minio-region ${MINIO_REGION_NAME} + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + + # This container will stop as soon as the script has run. + minio-setup: + image: minio/mc + depends_on: + - minio + volumes: + - "./scripts:/scripts" + environment: + MINIO_ADDR: &minio-addr http://minio:9000 + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + entrypoint: /scripts/minio-setup.sh + + vault: + image: vault:1.13.1 + command: server -dev + ports: + - 8200:8200 + environment: + VAULT_DEV_ROOT_TOKEN_ID: ${VAULT_DEV_ROOT_TOKEN_ID} + cap_add: + - IPC_LOCK + + # This container will stop as soon as the script has run. + vault-setup: + image: vault:1.13.1 + depends_on: + - vault + volumes: + - "./scripts:/scripts" + environment: + VAULT_ADDR: &vault-addr http://vault:8200 + # NOTE In production you would NOT pass the root token! + VAULT_DEV_ROOT_TOKEN_ID: ${VAULT_DEV_ROOT_TOKEN_ID} + MINIO_ADDR : *minio-addr + MINIO_REGION_NAME: *minio-region + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + + entrypoint: /scripts/vault-setup.sh + + db: + image: postgres:14.11 + shm_size: 1gb + ports: [6543:5432] + environment: + POSTGRES_DB: &postgres-db concourse + POSTGRES_USER: &postgres-user dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + + web: + image: &concourse-image concourse/concourse:7.13.2 + command: web + depends_on: + - db + - minio-setup + - vault-setup + ports: [8080:8080] + volumes: + - "./concourse-keys:/concourse-keys" + environment: + CONCOURSE_SESSION_SIGNING_KEY: /concourse-keys/session_signing_key + CONCOURSE_TSA_AUTHORIZED_KEYS: /concourse-keys/authorized_worker_keys + CONCOURSE_TSA_HOST_KEY: /concourse-keys/tsa_host_key + + CONCOURSE_LOG_LEVEL: error + CONCOURSE_TSA_LOG_LEVEL: error + CONCOURSE_POSTGRES_HOST: db + CONCOURSE_POSTGRES_USER: *postgres-user + CONCOURSE_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + CONCOURSE_POSTGRES_DATABASE: *postgres-db + CONCOURSE_EXTERNAL_URL: http://localhost:8080 + CONCOURSE_ADD_LOCAL_USER: ${CONCOURSE_MAIN_USER}:${CONCOURSE_MAIN_PASSWORD},${CONCOURSE_NORMAL_USER}:${CONCOURSE_NORMAL_PASSWORD},${CONCOURSE_READONLY_USER}:${CONCOURSE_READONLY_PASSWORD} + CONCOURSE_MAIN_TEAM_LOCAL_USER: ${CONCOURSE_MAIN_USER} + CONCOURSE_CLUSTER_NAME: dev + CONCOURSE_ENABLE_PIPELINE_INSTANCES: "true" + CONCOURSE_ENABLE_ACROSS_STEP: "true" + # NOTE this connection is unencrypted! + CONCOURSE_VAULT_URL: *vault-addr + # NOTE In production you would NOT pass the root token! + CONCOURSE_VAULT_CLIENT_TOKEN: ${VAULT_DEV_ROOT_TOKEN_ID} + CONCOURSE_ENABLE_CACHE_STREAMED_VOLUMES: "true" + CONCOURSE_ENABLE_RESOURCE_CAUSALITY: "true" + CONCOURSE_CONTAINER_PLACEMENT_STRATEGY: limit-active-tasks,volume-locality + CONCOURSE_MAX_ACTIVE_TASKS_PER_WORKER: 1 + + # This container will stop as soon as the script has run. + web-setup: + image: alpine/curl + depends_on: + - web + volumes: + - "./scripts:/scripts" + environment: + CONCOURSE_ADDR: web:8080 + + entrypoint: /scripts/concourse-setup.sh + + worker1: + image: *concourse-image + command: worker + privileged: true + depends_on: [web] + ports: + - 7777:7777 + - 7788:7788 + volumes: + - "./concourse-keys:/concourse-keys" + stop_signal: SIGUSR2 + environment: + CONCOURSE_RUNTIME: containerd + CONCOURSE_NAME: worker-1 + CONCOURSE_TAG: worker-1 + + CONCOURSE_TSA_PUBLIC_KEY: /concourse-keys/tsa_host_key.pub + CONCOURSE_TSA_WORKER_PRIVATE_KEY: /concourse-keys/worker_key + + CONCOURSE_LOG_LEVEL: error + CONCOURSE_TSA_HOST: web:2222 + + CONCOURSE_BAGGAGECLAIM_LOG_LEVEL: error + CONCOURSE_CONTAINERD_LOG_LEVEL: error + CONCOURSE_BIND_IP: 0.0.0.0 + CONCOURSE_BAGGAGECLAIM_BIND_IP: 0.0.0.0 + + # avoid using loopbacks + CONCOURSE_BAGGAGECLAIM_DRIVER: overlay + + # work with docker-compose's dns + CONCOURSE_CONTAINERD_DNS_PROXY_ENABLE: "true" + + worker2: + image: *concourse-image + command: worker + privileged: true + depends_on: [web] + ports: + - 7778:7777 + - 7789:7788 + volumes: + - "./concourse-keys:/concourse-keys" + stop_signal: SIGUSR2 + environment: + CONCOURSE_RUNTIME: containerd + CONCOURSE_NAME: worker-2 + CONCOURSE_TAG: worker-2 + + CONCOURSE_TSA_PUBLIC_KEY: /concourse-keys/tsa_host_key.pub + CONCOURSE_TSA_WORKER_PRIVATE_KEY: /concourse-keys/worker_key + + CONCOURSE_LOG_LEVEL: error + CONCOURSE_TSA_HOST: web:2222 + + CONCOURSE_BAGGAGECLAIM_LOG_LEVEL: error + CONCOURSE_CONTAINERD_LOG_LEVEL: error + CONCOURSE_BIND_IP: 0.0.0.0 + CONCOURSE_BAGGAGECLAIM_BIND_IP: 0.0.0.0 + + # avoid using loopbacks + CONCOURSE_BAGGAGECLAIM_DRIVER: overlay + + # work with docker-compose's dns + CONCOURSE_CONTAINERD_DNS_PROXY_ENABLE: "true" diff --git a/pci-4464-investigation/two-workers-with-flock/pipeline.yml b/pci-4464-investigation/two-workers-with-flock/pipeline.yml new file mode 100644 index 0000000..5f890d3 --- /dev/null +++ b/pci-4464-investigation/two-workers-with-flock/pipeline.yml @@ -0,0 +1,71 @@ +# bigfile-1 is generated with +# dd bs=1024 count=1048576 bigfile-1 +# then manually updated to minio in concourse bucket + +resources: + - name: artifacts + type: s3 + icon: gift + tags: [worker-1] + source: + bucket: concourse + regexp: bigfile-(.*) + endpoint: ((s3-endpoint)) + region_name: ((s3-region)) + access_key_id: ((s3-access-key)) + secret_access_key: ((s3-secret-key)) + +jobs: + - name: job1 + max_in_flight: 1 + plan: + - get: artifacts + tags: [worker-1] + - task: use-big-file + tags: [worker-2] # force streaming artifact from worker-1 to worker-2 + config: + platform: linux + caches: + - path: ../../../tmp/cache # should be /tmp/cache on worker-2 + image_resource: + type: registry-image + source: { repository: busybox } + inputs: + - name: artifacts + run: + path: sh + args: + - -c + - | + set -e + if [ ! -e "../../../tmp/cache/lock" ]; then + echo "creating file" + touch "../../../tmp/cache/lock" + fi + flock ../../../tmp/cache/lock -c 'echo "[worker-2] job1 is taking lock and doing hard work..." && sleep 60' + echo "releasing lock" + + - name: job2 + max_in_flight: 1 + plan: + - task: random-task + tags: [worker-2] + config: + platform: linux + caches: + - path: ../../../tmp/cache # should be /tmp/cache on worker-2 + image_resource: + type: registry-image + source: { repository: busybox } + run: + path: sh + args: + - -c + - | + set -e + if [ ! -e "../../../tmp/cache/lock" ]; then + echo "creating file" + touch "../../../tmp/cache/lock" + fi + flock ../../../tmp/cache/lock -c 'echo "[worker-2] job2 is taking lock and doing hard work..." && sleep 60' + echo "releasing lock" diff --git a/pci-4464-investigation/two-workers-with-flock/prometheus.yml b/pci-4464-investigation/two-workers-with-flock/prometheus.yml new file mode 100644 index 0000000..8bd2933 --- /dev/null +++ b/pci-4464-investigation/two-workers-with-flock/prometheus.yml @@ -0,0 +1,40 @@ +# prometheus.yml - a docker-compose override that adds 'prometheus' to the stack +# +# once running, head to `localhost:9090` to get access to the Prometheus console. +# +# ref: https://prometheus.io/ +# ref: https://docs.docker.com/compose/extends/ +# +version: '3' + +services: + web: + environment: + CONCOURSE_PROMETHEUS_BIND_IP: "0.0.0.0" + CONCOURSE_PROMETHEUS_BIND_PORT: "9100" + ports: + - '9100:9100' + + prometheus: + image: prom/prometheus + entrypoint: + - /bin/sh + - -c + - | + echo " + global: + scrape_interval: '1s' + evaluation_interval: '1s' + + scrape_configs: + - job_name: 'concourse' + static_configs: + - targets: + - 'web:9100' + " > config.yml + + exec prometheus \ + --config.file=config.yml \ + --storage.tsdb.path=/prometheus + ports: + - '9090:9090'