diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 576f68300b29..ac63da1695d0 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -88,9 +88,9 @@ name: Regression test workflow - Release env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }} diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e70fd154996a..1b4ce0995e15 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -7,6 +7,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} on: # yamllint disable-line rule:truthy pull_request: @@ -16,7 +21,7 @@ on: # yamllint disable-line rule:truthy - opened branches: # Anything/24.8 (e.g customizations/24.8.x) - - '**/24.8*' + - '**24.8*' release: types: - published diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index c64d0aaec500..1cd3f410f996 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -7,6 +7,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} name: Build ClickHouse 'on': diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml index 2bd8ae430fa5..fee2f9131469 100644 --- a/.github/workflows/reusable_sign.yml +++ b/.github/workflows/reusable_sign.yml @@ -1,7 +1,4 @@ -### For the pure soul wishes to move it to another place -# https://github.com/orgs/community/discussions/9050 - -name: Testing workflow +name: Sigining workflow 'on': workflow_call: inputs: @@ -63,6 +60,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: runner_labels_setup: diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 4867cab03480..4cb00cc9e83f 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -44,10 +44,25 @@ name: Testing workflow description: if given, it's passed to the environments required: false AWS_SECRET_ACCESS_KEY: - description: the access key to the aws param store. + description: the access key to the aws s3 bucket. required: true AWS_ACCESS_KEY_ID: - description: the access key id to the aws param store. + description: the access key id to the aws s3 bucket. + required: true + CLICKHOUSE_TEST_STAT_LOGIN: + description: username for ci db. + required: true + CLICKHOUSE_TEST_STAT_PASSWORD: + description: password for ci db. + required: true + CLICKHOUSE_TEST_STAT_URL: + description: url for ci db. + required: true + DOCKER_PASSWORD: + description: token to upload docker images. + required: true + ROBOT_TOKEN: + description: token to update ci status. required: true env: @@ -57,6 +72,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: runner_labels_setup: diff --git a/docker/packager/packager b/docker/packager/packager index 7975b6a03aae..c21bf41de4e7 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -6,12 +6,13 @@ import os import subprocess import sys from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" - +DEFAULT_TMP_PATH = SCRIPT_PATH.parent.absolute() / 'tmp' +TEMP_PATH = Path(os.getenv("TEMP_PATH", DEFAULT_TMP_PATH)) class BuildException(Exception): pass @@ -82,9 +83,22 @@ def run_docker_image_with_env( ch_root: Path, cargo_cache_dir: Path, ccache_dir: Optional[Path], + aws_secrets : Optional[Dict[str,str]] ) -> None: output_dir.mkdir(parents=True, exist_ok=True) cargo_cache_dir.mkdir(parents=True, exist_ok=True) + extra_parts = "" + + if aws_secrets: + # Pass AWS credentials via file rather than via env to avoid leaking secrets + env_part = {"AWS_CONFIG_FILE": "/home/clickhouse/.aws/credentials"} + host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' + with open(host_aws_config_file_path, 'wt') as f: + f.write("[default]") + for key, value in aws_secrets.items(): + f.write(f"\n{key}={value}") + + extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']}" env_part = " -e ".join(env_variables) if env_part: @@ -107,6 +121,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount} " f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " + f" {extra_parts} " f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}" ) @@ -130,11 +145,9 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, - s3_access_key_id: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, - s3_secret_access_key: str, clang_tidy: bool, version: str, official: bool, @@ -323,10 +336,6 @@ def parse_env_variables( result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}") if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") - if s3_access_key_id: - result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}") - if s3_secret_access_key: - result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}") if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. @@ -544,11 +553,9 @@ def main() -> None: args.sanitizer, args.package_type, args.cache, - args.s3_access_key_id, args.s3_bucket, args.s3_directory, args.s3_rw_access, - args.s3_secret_access_key, args.clang_tidy, args.version, args.official, @@ -567,6 +574,10 @@ def main() -> None: ch_root, args.cargo_cache_dir, args.ccache_dir, + { + "aws_access_key_id" : args.s3_access_key_id, + "aws_secret_access_key" : args.s3_secret_access_key + } ) logging.info("Output placed into %s", args.output_dir) diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 422e1738701a..e49e2f2d0b7e 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -9,6 +9,7 @@ import requests +from env_helper import CLICKHOUSE_TEST_STAT_URL, CLICKHOUSE_TEST_STAT_PASSWORD, CLICKHOUSE_TEST_STAT_LOGIN from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import TestResults @@ -27,12 +28,12 @@ def __init__( self, url: Optional[str] = None, auth: Optional[Dict[str, str]] = None ): if url is None: - url = get_parameter_from_ssm("clickhouse-test-stat-url") + url = CLICKHOUSE_TEST_STAT_URL self.url = url self.auth = auth or { - "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"), - "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"), + "X-ClickHouse-User": CLICKHOUSE_TEST_STAT_LOGIN, + "X-ClickHouse-Key": CLICKHOUSE_TEST_STAT_PASSWORD, } @staticmethod diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py index 9faefef12c24..219fdc00e20a 100644 --- a/tests/ci/docker_images_helper.py +++ b/tests/ci/docker_images_helper.py @@ -6,8 +6,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from env_helper import ROOT_DIR, DOCKER_TAG -from get_robot_token import get_parameter_from_ssm +from env_helper import ROOT_DIR, DOCKER_TAG, DOCKER_PASSWORD from ci_utils import Shell IMAGES_FILE_PATH = Path("docker/images.json") @@ -22,7 +21,7 @@ def docker_login(relogin: bool = True) -> None: Shell.check( # pylint: disable=unexpected-keyword-arg "docker login --username 'altinityinfra' --password-stdin", strict=True, - stdin_str=get_parameter_from_ssm("dockerhub-password"), + stdin_str=DOCKER_PASSWORD, encoding="utf-8", ) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index cad781a17445..11eae74689d2 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -46,6 +46,12 @@ ) CI_CONFIG_PATH = f"{TEMP_PATH}/ci_config.json" +CLICKHOUSE_TEST_STAT_LOGIN = os.getenv("CLICKHOUSE_TEST_STAT_LOGIN") +CLICKHOUSE_TEST_STAT_PASSWORD = os.getenv("CLICKHOUSE_TEST_STAT_PASSWORD") +CLICKHOUSE_TEST_STAT_URL = os.getenv("CLICKHOUSE_TEST_STAT_URL") +DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") +ROBOT_TOKEN = os.getenv("ROBOT_TOKEN") + # These parameters are set only on demand, and only once _GITHUB_JOB_ID = "" _GITHUB_JOB_URL = "" diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index a218202fd76e..f9b30beb178f 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -10,6 +10,7 @@ from github.GithubException import BadCredentialsException from github.NamedUser import NamedUser +from env_helper import ROBOT_TOKEN @dataclass class Token: @@ -56,20 +57,11 @@ def get_parameters_from_ssm( return results - -ROBOT_TOKEN = None # type: Optional[Token] - # NOTE(Arthur Passos): Original CI code uses the "_original" version of this method. Each robot token is rate limited # and the original implementation selects the "best one". To make it simpler and iterate faster, # we are using only one robot and keeping the method signature. In the future we might reconsider # having multiple robot tokens -def get_best_robot_token(token_prefix_env_name="github_robot_token"): - # Re-use already fetched token (same as in get_best_robot_token_original) - # except here we assume it is always a string (since we use only one token and don't do token rotation) - global ROBOT_TOKEN - if ROBOT_TOKEN is not None: - return ROBOT_TOKEN - ROBOT_TOKEN = get_parameter_from_ssm(token_prefix_env_name) +def get_best_robot_token(): return ROBOT_TOKEN def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: