From d174862eb20168b69169265a6dafabddb2f7e0cb Mon Sep 17 00:00:00 2001 From: MyroTk Date: Wed, 19 Mar 2025 17:27:03 -0400 Subject: [PATCH 01/18] get docker credentials from secrets --- tests/ci/docker_images_helper.py | 5 ++--- tests/ci/env_helper.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py index 9faefef12c24..219fdc00e20a 100644 --- a/tests/ci/docker_images_helper.py +++ b/tests/ci/docker_images_helper.py @@ -6,8 +6,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from env_helper import ROOT_DIR, DOCKER_TAG -from get_robot_token import get_parameter_from_ssm +from env_helper import ROOT_DIR, DOCKER_TAG, DOCKER_PASSWORD from ci_utils import Shell IMAGES_FILE_PATH = Path("docker/images.json") @@ -22,7 +21,7 @@ def docker_login(relogin: bool = True) -> None: Shell.check( # pylint: disable=unexpected-keyword-arg "docker login --username 'altinityinfra' --password-stdin", strict=True, - stdin_str=get_parameter_from_ssm("dockerhub-password"), + stdin_str=DOCKER_PASSWORD, encoding="utf-8", ) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index cad781a17445..3240c6a49756 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -16,6 +16,7 @@ REPORT_PATH = f"{TEMP_PATH}/reports" # FIXME: latest should not be used in CI, set temporary for transition to "docker with digest as a tag" DOCKER_TAG = os.getenv("DOCKER_TAG", "latest") +DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") From 581666df4aebca5ec69b7745ccb1952640eab4ff Mon Sep 17 00:00:00 2001 From: MyroTk Date: Wed, 19 Mar 2025 17:29:19 -0400 Subject: [PATCH 02/18] add docker password to env --- .github/workflows/release_branches.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e70fd154996a..f6df8b0e4123 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -7,6 +7,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} on: # yamllint disable-line rule:truthy pull_request: From a31e6b4b23fec95fbd0a05377c024249f05a7ade Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 21 Mar 2025 11:11:40 -0400 Subject: [PATCH 03/18] add secrets to env and prevent aws leaks --- .github/create_combined_ci_report.py | 2 +- .github/workflows/release_branches.yml | 4 ++++ .github/workflows/reusable_sign.yml | 10 +++++---- .github/workflows/reusable_test.yml | 9 ++++++-- docker/packager/packager | 30 ++++++++++++++++++-------- tests/ci/clickhouse_helper.py | 7 +++--- tests/ci/env_helper.py | 7 +++++- tests/ci/get_robot_token.py | 5 +---- 8 files changed, 50 insertions(+), 24 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 2607760e4abb..2a55f20a6dba 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -13,7 +13,7 @@ DATABASE_HOST_VAR = "CHECKS_DATABASE_HOST" DATABASE_USER_VAR = "CHECKS_DATABASE_USER" DATABASE_PASSWORD_VAR = "CHECKS_DATABASE_PASSWORD" -S3_BUCKET = "altinity-build-artifacts" +S3_BUCKET = "altinity-test-new-credentials" def get_checks_fails(client: Client, job_url: str): diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index f6df8b0e4123..c02206a14ca3 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -7,7 +7,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} on: # yamllint disable-line rule:truthy pull_request: diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml index 2bd8ae430fa5..fee2f9131469 100644 --- a/.github/workflows/reusable_sign.yml +++ b/.github/workflows/reusable_sign.yml @@ -1,7 +1,4 @@ -### For the pure soul wishes to move it to another place -# https://github.com/orgs/community/discussions/9050 - -name: Testing workflow +name: Sigining workflow 'on': workflow_call: inputs: @@ -63,6 +60,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: runner_labels_setup: diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 4867cab03480..2c293a9e1eb7 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -44,10 +44,10 @@ name: Testing workflow description: if given, it's passed to the environments required: false AWS_SECRET_ACCESS_KEY: - description: the access key to the aws param store. + description: the access key to the aws s3 bucket. required: true AWS_ACCESS_KEY_ID: - description: the access key id to the aws param store. + description: the access key id to the aws s3 bucket. required: true env: @@ -57,6 +57,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: runner_labels_setup: diff --git a/docker/packager/packager b/docker/packager/packager index 7975b6a03aae..2c28bb891b64 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -6,7 +6,9 @@ import os import subprocess import sys from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional + +from env_helper import TEMP_PATH SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" @@ -82,9 +84,22 @@ def run_docker_image_with_env( ch_root: Path, cargo_cache_dir: Path, ccache_dir: Optional[Path], + aws_secrets : Optional[Dict[str:str]] ) -> None: output_dir.mkdir(parents=True, exist_ok=True) cargo_cache_dir.mkdir(parents=True, exist_ok=True) + extra_parts = "" + + if aws_secrets: + # Pass AWS credentials via file rather than via env to avoid leaking secrets + env_part["AWS_CONFIG_FILE"] = "/aws_config" + host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' + with open(host_aws_config_file_path, 'wt') as f: + f.write("[default]") + for key, value in aws_secrets.items(): + f.write(f"{key}={value}") + + extra_parts = f" --volume={host_aws_config_file_path}:{env_part["AWS_CONFIG_FILE"]} " env_part = " -e ".join(env_variables) if env_part: @@ -107,6 +122,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount} " f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " + f" {extra_parts} " f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}" ) @@ -130,11 +146,9 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, - s3_access_key_id: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, - s3_secret_access_key: str, clang_tidy: bool, version: str, official: bool, @@ -323,10 +337,6 @@ def parse_env_variables( result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}") if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") - if s3_access_key_id: - result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}") - if s3_secret_access_key: - result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}") if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. @@ -544,11 +554,9 @@ def main() -> None: args.sanitizer, args.package_type, args.cache, - args.s3_access_key_id, args.s3_bucket, args.s3_directory, args.s3_rw_access, - args.s3_secret_access_key, args.clang_tidy, args.version, args.official, @@ -567,6 +575,10 @@ def main() -> None: ch_root, args.cargo_cache_dir, args.ccache_dir, + { + "aws_access_key_id" : args.s3_access_key_id, + "aws_secret_access_key" : args.s3_secret_access_key + } ) logging.info("Output placed into %s", args.output_dir) diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 422e1738701a..e49e2f2d0b7e 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -9,6 +9,7 @@ import requests +from env_helper import CLICKHOUSE_TEST_STAT_URL, CLICKHOUSE_TEST_STAT_PASSWORD, CLICKHOUSE_TEST_STAT_LOGIN from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import TestResults @@ -27,12 +28,12 @@ def __init__( self, url: Optional[str] = None, auth: Optional[Dict[str, str]] = None ): if url is None: - url = get_parameter_from_ssm("clickhouse-test-stat-url") + url = CLICKHOUSE_TEST_STAT_URL self.url = url self.auth = auth or { - "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"), - "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"), + "X-ClickHouse-User": CLICKHOUSE_TEST_STAT_LOGIN, + "X-ClickHouse-Key": CLICKHOUSE_TEST_STAT_PASSWORD, } @staticmethod diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 3240c6a49756..11eae74689d2 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -16,7 +16,6 @@ REPORT_PATH = f"{TEMP_PATH}/reports" # FIXME: latest should not be used in CI, set temporary for transition to "docker with digest as a tag" DOCKER_TAG = os.getenv("DOCKER_TAG", "latest") -DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") @@ -47,6 +46,12 @@ ) CI_CONFIG_PATH = f"{TEMP_PATH}/ci_config.json" +CLICKHOUSE_TEST_STAT_LOGIN = os.getenv("CLICKHOUSE_TEST_STAT_LOGIN") +CLICKHOUSE_TEST_STAT_PASSWORD = os.getenv("CLICKHOUSE_TEST_STAT_PASSWORD") +CLICKHOUSE_TEST_STAT_URL = os.getenv("CLICKHOUSE_TEST_STAT_URL") +DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") +ROBOT_TOKEN = os.getenv("ROBOT_TOKEN") + # These parameters are set only on demand, and only once _GITHUB_JOB_ID = "" _GITHUB_JOB_URL = "" diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index a218202fd76e..68a35259ac2e 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -10,6 +10,7 @@ from github.GithubException import BadCredentialsException from github.NamedUser import NamedUser +from env_helper import ROBOT_TOKEN @dataclass class Token: @@ -66,10 +67,6 @@ def get_parameters_from_ssm( def get_best_robot_token(token_prefix_env_name="github_robot_token"): # Re-use already fetched token (same as in get_best_robot_token_original) # except here we assume it is always a string (since we use only one token and don't do token rotation) - global ROBOT_TOKEN - if ROBOT_TOKEN is not None: - return ROBOT_TOKEN - ROBOT_TOKEN = get_parameter_from_ssm(token_prefix_env_name) return ROBOT_TOKEN def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: From a6d14f930864e332a82801bae292e33971ac7509 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 21 Mar 2025 11:19:07 -0400 Subject: [PATCH 04/18] use temp test bucket --- tests/ci/env_helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 11eae74689d2..eb4103af3107 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -33,10 +33,10 @@ RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) S3_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") -S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-build-artifacts") -S3_BUILDS_BUCKET_PUBLIC = "altinity-build-artifacts" +S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-test-new-credentials") +S3_BUILDS_BUCKET_PUBLIC = "altinity-test-new-credentials" S3_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") -S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") +S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-test-new-credentials") S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) From 0377def587785a67a62a9bdf7e1f787da309d85b Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 21 Mar 2025 11:38:25 -0400 Subject: [PATCH 05/18] fix syntax issue --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 2c28bb891b64..b6a4fb49e520 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -99,7 +99,7 @@ def run_docker_image_with_env( for key, value in aws_secrets.items(): f.write(f"{key}={value}") - extra_parts = f" --volume={host_aws_config_file_path}:{env_part["AWS_CONFIG_FILE"]} " + extra_parts = f" --volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']} " env_part = " -e ".join(env_variables) if env_part: From af1aeef8673c55bde35ad342d9b04c5743710e59 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 21 Mar 2025 11:52:34 -0400 Subject: [PATCH 06/18] avoid using env_helper module in packager --- docker/packager/packager | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index b6a4fb49e520..2e855ebd6478 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -5,15 +5,16 @@ import logging import os import subprocess import sys +from os import path as p from pathlib import Path from typing import Dict, List, Optional -from env_helper import TEMP_PATH +module_dir = p.abspath(p.dirname(__file__)) SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" - +TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) class BuildException(Exception): pass From a1955aebb8c207e295184e98e145185e4d32b8e7 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 21 Mar 2025 12:37:31 -0400 Subject: [PATCH 07/18] fix dict issue --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 2e855ebd6478..2ab5cfa9cea8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -85,7 +85,7 @@ def run_docker_image_with_env( ch_root: Path, cargo_cache_dir: Path, ccache_dir: Optional[Path], - aws_secrets : Optional[Dict[str:str]] + aws_secrets : Optional[Dict[str,str]] ) -> None: output_dir.mkdir(parents=True, exist_ok=True) cargo_cache_dir.mkdir(parents=True, exist_ok=True) From 93a391354805d91b32b919a8696c2fe4a85d2b53 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 21 Mar 2025 14:31:33 -0400 Subject: [PATCH 08/18] fix dict --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 2ab5cfa9cea8..57cb7e6e6362 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -93,7 +93,7 @@ def run_docker_image_with_env( if aws_secrets: # Pass AWS credentials via file rather than via env to avoid leaking secrets - env_part["AWS_CONFIG_FILE"] = "/aws_config" + env_part = {"AWS_CONFIG_FILE": "/aws_config"} host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' with open(host_aws_config_file_path, 'wt') as f: f.write("[default]") From f06682cd33693e7107661a2a10fba535da803dea Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 21 Mar 2025 15:42:26 -0400 Subject: [PATCH 09/18] update envs in build script --- .github/workflows/release_branches.yml | 2 +- .github/workflows/reusable_build.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index c02206a14ca3..1b4ce0995e15 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -21,7 +21,7 @@ on: # yamllint disable-line rule:truthy - opened branches: # Anything/24.8 (e.g customizations/24.8.x) - - '**/24.8*' + - '**24.8*' release: types: - published diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index c64d0aaec500..1cd3f410f996 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -7,6 +7,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} name: Build ClickHouse 'on': From 0f5c6c51be000f7408998fb481f98230291d9ebd Mon Sep 17 00:00:00 2001 From: MyroTk Date: Mon, 24 Mar 2025 10:13:25 -0400 Subject: [PATCH 10/18] aws config file fix --- docker/packager/packager | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 57cb7e6e6362..57a76106eb25 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -98,9 +98,9 @@ def run_docker_image_with_env( with open(host_aws_config_file_path, 'wt') as f: f.write("[default]") for key, value in aws_secrets.items(): - f.write(f"{key}={value}") + f.write(f"\n{key}={value}") - extra_parts = f" --volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']} " + extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']}" env_part = " -e ".join(env_variables) if env_part: From 288a32fce611b5ce202760164a9507b27ed0eaf6 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Mon, 24 Mar 2025 10:44:50 -0400 Subject: [PATCH 11/18] update aws config path inside docker --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 57a76106eb25..df69ca3058ac 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -93,7 +93,7 @@ def run_docker_image_with_env( if aws_secrets: # Pass AWS credentials via file rather than via env to avoid leaking secrets - env_part = {"AWS_CONFIG_FILE": "/aws_config"} + env_part = {"AWS_CONFIG_FILE": "/root/.aws/config"} host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' with open(host_aws_config_file_path, 'wt') as f: f.write("[default]") From eff4415937c6ef3728db85cd700c8e5909a9ec50 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Mon, 24 Mar 2025 10:59:55 -0400 Subject: [PATCH 12/18] aws credentials path inside docker --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index df69ca3058ac..6c421ee8f99e 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -93,7 +93,7 @@ def run_docker_image_with_env( if aws_secrets: # Pass AWS credentials via file rather than via env to avoid leaking secrets - env_part = {"AWS_CONFIG_FILE": "/root/.aws/config"} + env_part = {"AWS_CONFIG_FILE": "/root/.aws/credentials"} host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' with open(host_aws_config_file_path, 'wt') as f: f.write("[default]") From 9ac681fbbbb64a75aeb876859ad9ee41fe2344c6 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Mon, 24 Mar 2025 12:50:10 -0400 Subject: [PATCH 13/18] move aws credentials to the correct user inside container --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 6c421ee8f99e..0b6bb4e739f8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -93,7 +93,7 @@ def run_docker_image_with_env( if aws_secrets: # Pass AWS credentials via file rather than via env to avoid leaking secrets - env_part = {"AWS_CONFIG_FILE": "/root/.aws/credentials"} + env_part = {"AWS_CONFIG_FILE": "/clickhouse/.aws/credentials"} host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' with open(host_aws_config_file_path, 'wt') as f: f.write("[default]") From abf28f578a7ab11111c414b981e1add69799a45c Mon Sep 17 00:00:00 2001 From: MyroTk Date: Mon, 24 Mar 2025 13:36:14 -0400 Subject: [PATCH 14/18] change aws credentials directory. again. --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 0b6bb4e739f8..d7cc2ac807bc 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -93,7 +93,7 @@ def run_docker_image_with_env( if aws_secrets: # Pass AWS credentials via file rather than via env to avoid leaking secrets - env_part = {"AWS_CONFIG_FILE": "/clickhouse/.aws/credentials"} + env_part = {"AWS_CONFIG_FILE": "/home/clickhouse/.aws/credentials"} host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' with open(host_aws_config_file_path, 'wt') as f: f.write("[default]") From 95644cbc9c08cbd5cffa4ce7991886a32cac83a9 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Mon, 24 Mar 2025 19:49:26 -0400 Subject: [PATCH 15/18] switch back to altinitybuild0artifacts bucket --- .github/create_combined_ci_report.py | 2 +- tests/ci/env_helper.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 2a55f20a6dba..2607760e4abb 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -13,7 +13,7 @@ DATABASE_HOST_VAR = "CHECKS_DATABASE_HOST" DATABASE_USER_VAR = "CHECKS_DATABASE_USER" DATABASE_PASSWORD_VAR = "CHECKS_DATABASE_PASSWORD" -S3_BUCKET = "altinity-test-new-credentials" +S3_BUCKET = "altinity-build-artifacts" def get_checks_fails(client: Client, job_url: str): diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index eb4103af3107..11eae74689d2 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -33,10 +33,10 @@ RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) S3_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") -S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-test-new-credentials") -S3_BUILDS_BUCKET_PUBLIC = "altinity-test-new-credentials" +S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-build-artifacts") +S3_BUILDS_BUCKET_PUBLIC = "altinity-build-artifacts" S3_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") -S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-test-new-credentials") +S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) From 530a662f43200bed05e46ad70999e32ddd1fece2 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Tue, 25 Mar 2025 13:07:05 -0400 Subject: [PATCH 16/18] Update secrets definition in workflows and temp path in packager --- .github/workflows/reusable_test.yml | 15 +++++++++++++++ docker/packager/packager | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 2c293a9e1eb7..4cb00cc9e83f 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -49,6 +49,21 @@ name: Testing workflow AWS_ACCESS_KEY_ID: description: the access key id to the aws s3 bucket. required: true + CLICKHOUSE_TEST_STAT_LOGIN: + description: username for ci db. + required: true + CLICKHOUSE_TEST_STAT_PASSWORD: + description: password for ci db. + required: true + CLICKHOUSE_TEST_STAT_URL: + description: url for ci db. + required: true + DOCKER_PASSWORD: + description: token to upload docker images. + required: true + ROBOT_TOKEN: + description: token to update ci status. + required: true env: # Force the stdout and stderr streams to be unbuffered diff --git a/docker/packager/packager b/docker/packager/packager index d7cc2ac807bc..37ecaf6b6b44 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -14,7 +14,8 @@ module_dir = p.abspath(p.dirname(__file__)) SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" -TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) +DEFAULT_TMP_PATH = SCRIPT_PATH.parent.absolute() / 'tmp' +TEMP_PATH = Path(os.getenv("TEMP_PATH", DEFAULT_TMP_PATH)) class BuildException(Exception): pass From 781e811c26cce8b95093ef93c106557f6ee9107e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 25 Mar 2025 19:28:24 +0100 Subject: [PATCH 17/18] remved unused module_dir --- docker/packager/packager | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 37ecaf6b6b44..c21bf41de4e7 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -5,12 +5,9 @@ import logging import os import subprocess import sys -from os import path as p from pathlib import Path from typing import Dict, List, Optional -module_dir = p.abspath(p.dirname(__file__)) - SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" From 926c396ba3b5ba0e3ccb842768b25089a50474b0 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Tue, 25 Mar 2025 16:48:27 -0400 Subject: [PATCH 18/18] fix regression aws secrets and robot token function --- .github/workflows/regression.yml | 6 +++--- tests/ci/get_robot_token.py | 7 +------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 576f68300b29..ac63da1695d0 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -88,9 +88,9 @@ name: Regression test workflow - Release env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }} diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 68a35259ac2e..f9b30beb178f 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -57,16 +57,11 @@ def get_parameters_from_ssm( return results - -ROBOT_TOKEN = None # type: Optional[Token] - # NOTE(Arthur Passos): Original CI code uses the "_original" version of this method. Each robot token is rate limited # and the original implementation selects the "best one". To make it simpler and iterate faster, # we are using only one robot and keeping the method signature. In the future we might reconsider # having multiple robot tokens -def get_best_robot_token(token_prefix_env_name="github_robot_token"): - # Re-use already fetched token (same as in get_best_robot_token_original) - # except here we assume it is always a string (since we use only one token and don't do token rotation) +def get_best_robot_token(): return ROBOT_TOKEN def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: