diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index ce6e049679d..e5d3a021f6d 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -4,6 +4,7 @@ on: branches: - master - release/** + - test/nightly-distroless jobs: build-multiplatform: @@ -81,6 +82,7 @@ jobs: ghcr: true tag_nightly: false tag_latest: false + publish_on_pr: ${{ github.head_ref == 'test/nightly-distroless' }} # Debug distroless image — with busybox for troubleshooting build-debug-multiplatform: @@ -136,7 +138,7 @@ jobs: assemble-distroless: needs: [build-distroless-multiplatform] - if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} + if: ${{ needs.build-distroless-multiplatform.result == 'success' && github.repository_owner == 'getsentry' && (github.ref_name == 'master' || startsWith(github.ref_name, 'release/') || github.ref_name == 'test/nightly-distroless' || github.head_ref == 'test/nightly-distroless') }} runs-on: ubuntu-latest permissions: contents: read @@ -151,12 +153,14 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Create distroless multiplatform manifests + env: + IMAGE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: | docker buildx imagetools create \ - --tag ghcr.io/getsentry/snuba:${{ github.sha }}-distroless \ + --tag ghcr.io/getsentry/snuba:${IMAGE_SHA}-distroless \ --tag ghcr.io/getsentry/snuba:nightly-distroless \ - ghcr.io/getsentry/snuba:${{ github.sha }}-distroless-amd64 \ - ghcr.io/getsentry/snuba:${{ github.sha }}-distroless-arm64 + ghcr.io/getsentry/snuba:${IMAGE_SHA}-distroless-amd64 \ + ghcr.io/getsentry/snuba:${IMAGE_SHA}-distroless-arm64 assemble-debug: needs: [build-debug-multiplatform] diff --git a/devservices/config.yml b/devservices/config.yml index 16aa137158b..1ddbd3131a2 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -75,7 +75,7 @@ services: restart: unless-stopped snuba: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless ports: - 127.0.0.1:1218:1218 - 127.0.0.1:1219:1219 @@ -83,7 +83,7 @@ services: - devserver - --${SNUBA_NO_WORKERS:+no-workers} healthcheck: - test: curl -f http://localhost:1218/health_envoy + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:1218/health_envoy')"] interval: 5s timeout: 5s retries: 3 @@ -114,7 +114,7 @@ services: - orchestrator=devservices restart: unless-stopped profiles-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless command: [ rust-consumer, --storage=profiles, @@ -143,7 +143,7 @@ services: - orchestrator=devservices restart: unless-stopped profile-chunks-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless command: [ rust-consumer, --storage=profile_chunks, @@ -172,7 +172,7 @@ services: - orchestrator=devservices restart: unless-stopped functions-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless command: [ rust-consumer, --storage=functions_raw, @@ -201,7 +201,7 @@ services: - orchestrator=devservices restart: unless-stopped metrics-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless command: [ rust-consumer, --storage=metrics_raw, @@ -230,7 +230,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-distributions-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless command: [ rust-consumer, --storage=generic_metrics_distributions_raw, @@ -259,7 +259,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-sets-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless command: [ rust-consumer, --storage=generic_metrics_sets_raw, @@ -288,7 +288,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-counters-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless command: [ rust-consumer, --storage=generic_metrics_counters_raw, @@ -317,7 +317,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-gauges-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:nightly-distroless command: [ rust-consumer, --storage=generic_metrics_gauges_raw, diff --git a/granian_entrypoint.py b/granian_entrypoint.py new file mode 100644 index 00000000000..ceaf9cc2a36 --- /dev/null +++ b/granian_entrypoint.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +""" +Python replacement for granian_entrypoint.sh — compatible with distroless images. +""" + +import os +import time +import urllib.error +import urllib.request + + +def wait_for_envoy() -> None: + port = os.environ.get("ENVOY_ADMIN_PORT") + if not port: + print("ENVOY_ADMIN_PORT env var is unset") + print("Fall through to snuba start without check") + return + + url = f"http://localhost:{port}/ready" + print(f"Check envoy readiness on {url}") + while True: + time.sleep(1) + try: + with urllib.request.urlopen(url, timeout=2) as resp: + if resp.read().decode().strip() == "LIVE": + print("Envoy is ready") + return + except Exception: + pass + print("Envoy not ready, looping..") + + +def main() -> None: + wait_for_envoy() + + wsgi_target = os.environ.get("SNUBA_WSGI_TARGET", "snuba.web.wsgi:application") + args = ["granian", "--interface", "wsgi", "--host", "0.0.0.0", "--http", "auto", wsgi_target] + os.execvp(args[0], args) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 42c04c9f530..69bab7335a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,6 @@ snuba = "snuba.cli:main" dev = [ "devservices>=1.2.1", "freezegun>=1.5.5", - "honcho>=1.1.0", "mypy>=1.1.1", "pre-commit>=4.2.0", "pytest>=8.3.3", @@ -136,7 +135,6 @@ module = [ "fastjsonschema", "fastjsonschema.exceptions", "granian", - "honcho.manager", "jsonschema", "jsonschema.exceptions", "jsonschema2md", diff --git a/snuba/cli/devserver.py b/snuba/cli/devserver.py index f7e1128828e..4fb34243d5b 100644 --- a/snuba/cli/devserver.py +++ b/snuba/cli/devserver.py @@ -1,6 +1,9 @@ import os +import signal +import subprocess import sys -from subprocess import call, list2cmdline +import threading +from subprocess import call import click @@ -21,8 +24,6 @@ def devserver(*, bootstrap: bool, workers: bool, log_level: str) -> None: "Starts all Snuba processes for local development." - from honcho.manager import Manager - os.environ["PYTHONUNBUFFERED"] = "1" if bootstrap: @@ -518,13 +519,44 @@ def devserver(*, bootstrap: bool, workers: bool, log_level: str) -> None: ), ] - manager = Manager() + sys.exit(_run_daemons(daemons)) + + +def _run_daemons(daemons: list[tuple[str, list[str]]]) -> int: + procs: dict[str, subprocess.Popen[bytes]] = {} + first_failure: list[int] = [] + done = threading.Event() + + def stream(name: str, proc: subprocess.Popen[bytes]) -> None: + assert proc.stdout is not None + for line in proc.stdout: + sys.stdout.write(f"{name} | {line.decode(errors='replace')}") + sys.stdout.flush() + rc = proc.wait() + if rc != 0 and not first_failure: + first_failure.append(rc) + done.set() + for name, cmd in daemons: - manager.add_process( - name, - list2cmdline(cmd), - quiet=False, - ) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + procs[name] = proc + threading.Thread(target=stream, args=(name, proc), daemon=True).start() + + def shutdown(signum: int, frame: object) -> None: + for proc in procs.values(): + if proc.poll() is None: + proc.terminate() + + signal.signal(signal.SIGINT, shutdown) + signal.signal(signal.SIGTERM, shutdown) + + done.wait() + if first_failure: + for proc in procs.values(): + if proc.poll() is None: + proc.terminate() + + for proc in procs.values(): + proc.wait() - manager.loop() - sys.exit(manager.returncode) + return first_failure[0] if first_failure else 0 diff --git a/uv.lock b/uv.lock index 9e33136ccda..9190a0043c0 100644 --- a/uv.lock +++ b/uv.lock @@ -416,14 +416,6 @@ wheels = [ { url = "https://pypi.devinfra.sentry.io/wheels/grpcio-1.73.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ab860d5bfa788c5a021fba264802e2593688cd965d1374d31d2b1a34cacd854" }, ] -[[package]] -name = "honcho" -version = "1.1.0" -source = { registry = "https://pypi.devinfra.sentry.io/simple" } -wheels = [ - { url = "https://pypi.devinfra.sentry.io/wheels/honcho-1.1.0-py2.py3-none-any.whl", hash = "sha256:a4d6e3a88a7b51b66351ecfc6e9d79d8f4b87351db9ad7e923f5632cc498122f" }, -] - [[package]] name = "httplib2" version = "0.22.0" @@ -1104,7 +1096,6 @@ dependencies = [ dev = [ { name = "devservices", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "freezegun", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "honcho", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "mypy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pre-commit", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -1175,7 +1166,6 @@ requires-dist = [ dev = [ { name = "devservices", specifier = ">=1.2.1" }, { name = "freezegun", specifier = ">=1.5.5" }, - { name = "honcho", specifier = ">=1.1.0" }, { name = "mypy", specifier = ">=1.1.1" }, { name = "pre-commit", specifier = ">=4.2.0" }, { name = "pytest", specifier = ">=8.3.3" },