From 39eb896b353f51c67cc4ec7f502a07daa2cfab65 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 12:51:30 -0400 Subject: [PATCH 01/21] cherry-pick #693 --- tests/ci/docker_images_helper.py | 7 ++++--- tests/ci/env_helper.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py index fe7b80a3b669..c67d50832e29 100644 --- a/tests/ci/docker_images_helper.py +++ b/tests/ci/docker_images_helper.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from env_helper import ROOT_DIR, DOCKER_TAG -from get_robot_token import get_parameter_from_ssm +from env_helper import ROOT_DIR, DOCKER_TAG, DOCKER_PASSWORD +from ci_utils import Shell IMAGES_FILE_PATH = Path("docker/images.json") @@ -28,7 +28,8 @@ def docker_login(relogin: bool = True) -> None: logging.info('Doing docker login') subprocess.check_output( # pylint: disable=unexpected-keyword-arg "docker login --username 'altinityinfra' --password-stdin", - input=get_parameter_from_ssm("dockerhub-password"), + strict=True, + stdin_str=DOCKER_PASSWORD, encoding="utf-8", shell=True, ) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index a8a00b02882b..a55cfd705c1d 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -15,6 +15,7 @@ REPORT_PATH = f"{TEMP_PATH}/reports" # FIXME: latest should not be used in CI, set temporary for transition to "docker with digest as a tag" DOCKER_TAG = os.getenv("DOCKER_TAG", "latest") +DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") From 94af4c2c571ec2e2df05724e5ebd2d59707a924a Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 09:26:56 -0400 Subject: [PATCH 02/21] Scan files for secrets in _upload_file_to_s3 --- tests/ci/s3_helper.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index e073430df953..0353a92210a6 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -6,6 +6,7 @@ from multiprocessing.dummy import Pool from pathlib import Path from typing import Any, List, Union +import os import boto3 # type: ignore import botocore # type: ignore @@ -19,6 +20,42 @@ S3_URL, ) +sensitive_var_pattern = re.compile( + r"\b[A-Z_]*(? str: + logging.debug("Checking %s for sensitive values", file_path) + try: + file_content = file_path.read_text(encoding="utf-8") + except UnicodeDecodeError: + logging.warning("Failed to read file %s, unknown encoding", file_path) + else: + scan_file_for_sensitive_data(file_content, file_path.name) + logging.debug( "Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path ) From 0126ef6a33e547d23745a89d52ca862cce437a29 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 13:09:05 -0400 Subject: [PATCH 03/21] fix --- tests/ci/docker_images_helper.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py index c67d50832e29..cc9671ec773e 100644 --- a/tests/ci/docker_images_helper.py +++ b/tests/ci/docker_images_helper.py @@ -8,7 +8,6 @@ from typing import Any, Dict, List, Optional from env_helper import ROOT_DIR, DOCKER_TAG, DOCKER_PASSWORD -from ci_utils import Shell IMAGES_FILE_PATH = Path("docker/images.json") @@ -28,8 +27,7 @@ def docker_login(relogin: bool = True) -> None: logging.info('Doing docker login') subprocess.check_output( # pylint: disable=unexpected-keyword-arg "docker login --username 'altinityinfra' --password-stdin", - strict=True, - stdin_str=DOCKER_PASSWORD, + input=DOCKER_PASSWORD, encoding="utf-8", shell=True, ) @@ -57,7 +55,7 @@ def pull_image(image: DockerImage) -> DockerImage: ) logging.info("Pulling image %s - done", image) except Exception as ex: - logging.info("Got execption pulling docker %s", ex) + logging.info("Got exception pulling docker %s", ex) raise ex return image From c9b29df726558ea931965d10899e3b2fba75d46b Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 13:35:00 -0400 Subject: [PATCH 04/21] update workflow secrets --- .github/workflows/regression.yml | 6 +++--- .github/workflows/release_branches.yml | 2 ++ .github/workflows/reusable_build.yml | 5 +++++ .github/workflows/reusable_sign.yml | 5 +++++ .github/workflows/reusable_test.yml | 24 ++++++++++++++++++++++-- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 3413cccdc764..f4ae11d7cf84 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -88,9 +88,9 @@ name: Regression test workflow - Release env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }} diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 7cc9cd849895..0e03ab3859fd 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -7,6 +7,8 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} on: # yamllint disable-line rule:truthy pull_request: diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index 4cff1c198c3b..637b9e94aaf6 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -8,6 +8,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} name: Build ClickHouse 'on': diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml index 5682f202e63b..ad9eab8d1692 100644 --- a/.github/workflows/reusable_sign.yml +++ b/.github/workflows/reusable_sign.yml @@ -63,6 +63,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: runner_labels_setup: diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 99aad0a3e36a..38a6794f555f 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -44,10 +44,25 @@ name: Testing workflow description: if given, it's passed to the environments required: false AWS_SECRET_ACCESS_KEY: - description: the access key to the aws param store. + description: the access key to the aws s3 bucket. required: true AWS_ACCESS_KEY_ID: - description: the access key id to the aws param store. + description: the access key id to the aws s3 bucket. + required: true + CLICKHOUSE_TEST_STAT_LOGIN: + description: username for ci db. + required: true + CLICKHOUSE_TEST_STAT_PASSWORD: + description: password for ci db. + required: true + CLICKHOUSE_TEST_STAT_URL: + description: url for ci db. + required: true + DOCKER_PASSWORD: + description: token to upload docker images. + required: true + ROBOT_TOKEN: + description: token to update ci status. required: true env: @@ -57,6 +72,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: runner_labels_setup: From a17e8b1041a6cbdd43dd466beff335630f1265c0 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 13:40:06 -0400 Subject: [PATCH 05/21] update get_robot_token.py --- tests/ci/env_helper.py | 5 +++++ tests/ci/get_robot_token.py | 40 +++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index a55cfd705c1d..c1946eb5aa2a 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -38,6 +38,11 @@ f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/" "{pr_or_release}/{commit}/{build_name}/{artifact}" ) +CLICKHOUSE_TEST_STAT_LOGIN = os.getenv("CLICKHOUSE_TEST_STAT_LOGIN") +CLICKHOUSE_TEST_STAT_PASSWORD = os.getenv("CLICKHOUSE_TEST_STAT_PASSWORD") +CLICKHOUSE_TEST_STAT_URL = os.getenv("CLICKHOUSE_TEST_STAT_URL") +DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") +ROBOT_TOKEN = os.getenv("ROBOT_TOKEN") # These parameters are set only on demand, and only once _GITHUB_JOB_ID = "" diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 5639b72fa3bc..f9b30beb178f 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import logging +import random from dataclasses import dataclass from typing import Any, Dict, List, Optional, Union @@ -9,6 +10,7 @@ from github.GithubException import BadCredentialsException from github.NamedUser import NamedUser +from env_helper import ROBOT_TOKEN @dataclass class Token: @@ -17,6 +19,9 @@ class Token: rest: int +SAFE_REQUESTS_LIMIT = 1000 + + def get_parameter_from_ssm( name: str, decrypt: bool = True, client: Optional[Any] = None ) -> str: @@ -52,20 +57,11 @@ def get_parameters_from_ssm( return results - -ROBOT_TOKEN = None # type: Optional[Token] - # NOTE(Arthur Passos): Original CI code uses the "_original" version of this method. Each robot token is rate limited # and the original implementation selects the "best one". To make it simpler and iterate faster, # we are using only one robot and keeping the method signature. In the future we might reconsider # having multiple robot tokens -def get_best_robot_token(token_prefix_env_name="github_robot_token"): - # Re-use already fetched token (same as in get_best_robot_token_original) - # except here we assume it is always a string (since we use only one token and don't do token rotation) - global ROBOT_TOKEN - if ROBOT_TOKEN is not None: - return ROBOT_TOKEN - ROBOT_TOKEN = get_parameter_from_ssm(token_prefix_env_name) +def get_best_robot_token(): return ROBOT_TOKEN def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: @@ -81,10 +77,15 @@ def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: } assert tokens - for name, value in tokens.items(): + token_items = list(tokens.items()) + random.shuffle(token_items) + + best_token: Optional[Token] = None + + for name, value in token_items: gh = Github(value, per_page=100) try: - # Do not spend additional request to API by accessin user.login unless + # Do not spend additional request to API by accessing user.login unless # the token is chosen by the remaining requests number user = gh.get_user() rest, _ = gh.rate_limiting @@ -96,13 +97,14 @@ def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: ) continue logging.info("Get token with %s remaining requests", rest) - if ROBOT_TOKEN is None: - ROBOT_TOKEN = Token(user, value, rest) - continue - if ROBOT_TOKEN.rest < rest: - ROBOT_TOKEN.user, ROBOT_TOKEN.value, ROBOT_TOKEN.rest = user, value, rest - - assert ROBOT_TOKEN + if best_token is None: + best_token = Token(user, value, rest) + elif best_token.rest < rest: + best_token = Token(user, value, rest) + if best_token.rest > SAFE_REQUESTS_LIMIT: + break + assert best_token + ROBOT_TOKEN = best_token logging.info( "User %s with %s remaining requests is used", ROBOT_TOKEN.user.login, From 7753dde28b4f3c04ee0a8f00a534c00a52ba5ffb Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 13:49:47 -0400 Subject: [PATCH 06/21] don't import from env_helper, circular import? --- tests/ci/get_robot_token.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index f9b30beb178f..abadb30adce1 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import logging +import os import random from dataclasses import dataclass from typing import Any, Dict, List, Optional, Union @@ -10,7 +11,7 @@ from github.GithubException import BadCredentialsException from github.NamedUser import NamedUser -from env_helper import ROBOT_TOKEN +ROBOT_TOKEN = os.getenv("ROBOT_TOKEN") @dataclass class Token: From a8fca35fa9264b742f4cd18170b60e3094f26a84 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 14:37:53 -0400 Subject: [PATCH 07/21] update clickhouse_helper.py --- tests/ci/clickhouse_helper.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 43fe915b765f..287fd30a7a44 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -4,9 +4,11 @@ import logging import time from pathlib import Path -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional import requests + +from env_helper import CLICKHOUSE_TEST_STAT_URL, CLICKHOUSE_TEST_STAT_PASSWORD, CLICKHOUSE_TEST_STAT_LOGIN from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import TestResults @@ -25,12 +27,12 @@ def __init__( self, url: Optional[str] = None, auth: Optional[Dict[str, str]] = None ): if url is None: - url = get_parameter_from_ssm("clickhouse-test-stat-url") + url = CLICKHOUSE_TEST_STAT_URL self.url = url self.auth = auth or { - "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"), - "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"), + "X-ClickHouse-User": CLICKHOUSE_TEST_STAT_LOGIN, + "X-ClickHouse-Key": CLICKHOUSE_TEST_STAT_PASSWORD, } @staticmethod @@ -40,6 +42,7 @@ def insert_file( query: str, file: Path, additional_options: Optional[Dict[str, str]] = None, + **kwargs: Any, ) -> None: params = { "query": query, @@ -52,7 +55,7 @@ def insert_file( with open(file, "rb") as data_fd: ClickHouseHelper._insert_post( - url, params=params, data=data_fd, headers=auth + url, params=params, data=data_fd, headers=auth, **kwargs ) @staticmethod From b14694fa49dd3d7db3c60171c563e5cba2849c7c Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:14:51 -0400 Subject: [PATCH 08/21] update packager --- docker/packager/packager | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index aeb413f567d4..8c015255539d 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -6,12 +6,13 @@ import os import subprocess import sys from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" - +DEFAULT_TMP_PATH = SCRIPT_PATH.parent.absolute() / 'tmp' +TEMP_PATH = Path(os.getenv("TEMP_PATH", DEFAULT_TMP_PATH)) class BuildException(Exception): pass @@ -82,9 +83,22 @@ def run_docker_image_with_env( ch_root: Path, cargo_cache_dir: Path, ccache_dir: Optional[Path], + aws_secrets : Optional[Dict[str,str]] ) -> None: output_dir.mkdir(parents=True, exist_ok=True) cargo_cache_dir.mkdir(parents=True, exist_ok=True) + extra_parts = "" + + if aws_secrets: + # Pass AWS credentials via file rather than via env to avoid leaking secrets + env_part = {"AWS_CONFIG_FILE": "/home/clickhouse/.aws/credentials"} + host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' + with open(host_aws_config_file_path, 'wt') as f: + f.write("[default]") + for key, value in aws_secrets.items(): + f.write(f"\n{key}={value}") + + extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']}" env_part = " -e ".join(env_variables) if env_part: @@ -107,6 +121,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount} " f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " + f" {extra_parts} " f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}" ) @@ -130,11 +145,9 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, - s3_access_key_id: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, - s3_secret_access_key: str, clang_tidy: bool, version: str, official: bool, @@ -319,10 +332,6 @@ def parse_env_variables( result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}") if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") - if s3_access_key_id: - result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}") - if s3_secret_access_key: - result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}") if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. @@ -539,11 +548,9 @@ def main() -> None: args.sanitizer, args.package_type, args.cache, - args.s3_access_key_id, args.s3_bucket, args.s3_directory, args.s3_rw_access, - args.s3_secret_access_key, args.clang_tidy, args.version, args.official, @@ -562,6 +569,10 @@ def main() -> None: ch_root, args.cargo_cache_dir, args.ccache_dir, + { + "aws_access_key_id" : args.s3_access_key_id, + "aws_secret_access_key" : args.s3_secret_access_key + } ) logging.info("Output placed into %s", args.output_dir) From faaaafedd364068372d5460fb4075ed9e144d686 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 17:38:27 -0400 Subject: [PATCH 09/21] update fasttest/Dockerfile --- docker/test/fasttest/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 62d7dec60531..36739e05ffe6 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -30,7 +30,9 @@ RUN apt-get update \ zstd \ --yes --no-install-recommends \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \ + && groupadd --system --gid 1000 clickhouse \ + && useradd --system --gid 1000 --uid 1000 -m clickhouse RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3 From 3245aee2bbee2db0349d6e9d67a85f528768d566 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 17:49:23 -0400 Subject: [PATCH 10/21] fix builddockers env --- .github/workflows/reusable_docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable_docker.yml b/.github/workflows/reusable_docker.yml index 5a0178039ddc..6834bd65483e 100644 --- a/.github/workflows/reusable_docker.yml +++ b/.github/workflows/reusable_docker.yml @@ -26,6 +26,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} jobs: DockerBuildAarch64: From dcd062599ee8c21d6246af64f17b6ead1ed123cc Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 18:14:13 -0400 Subject: [PATCH 11/21] update util/Dockerfile --- .github/workflows/reusable_docker.yml | 1 + docker/test/util/Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_docker.yml b/.github/workflows/reusable_docker.yml index 6834bd65483e..f0c7c1a24756 100644 --- a/.github/workflows/reusable_docker.yml +++ b/.github/workflows/reusable_docker.yml @@ -27,6 +27,7 @@ env: AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: DockerBuildAarch64: diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index 91316346069d..499357f82f08 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -33,9 +33,9 @@ RUN apt-get update \ # Install cmake 3.20+ for Rust support # Used https://askubuntu.com/a/1157132 as reference -RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | \ - gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg && \ - echo "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" >> /etc/apt/sources.list +RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg && \ + echo "deb [signed-by=/etc/apt/trusted.gpg.d/kitware.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" >> /etc/apt/sources.list.d/kitware.list + # initial packages RUN apt-get update \ From 244c976f8e907a8c9dd866c25ee7b2dbb159b656 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 18:43:39 -0400 Subject: [PATCH 12/21] pin cmake version --- docker/test/fasttest/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 36739e05ffe6..e41bb5d88626 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -8,7 +8,8 @@ RUN apt-get update \ brotli \ clang-${LLVM_VERSION} \ clang-tidy-${LLVM_VERSION} \ - cmake \ + cmake=3.22.1-1ubuntu1 \ + cmake-data=3.22.1-1ubuntu1 \ expect \ file \ libclang-${LLVM_VERSION}-dev \ From 4ec3f0c23cf7994b9922d80d0aeab8f7f9406477 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Apr 2025 19:07:00 -0400 Subject: [PATCH 13/21] need robot token for builddockers --- .github/workflows/reusable_docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable_docker.yml b/.github/workflows/reusable_docker.yml index f0c7c1a24756..7614257e4ae5 100644 --- a/.github/workflows/reusable_docker.yml +++ b/.github/workflows/reusable_docker.yml @@ -28,6 +28,7 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: DockerBuildAarch64: From 6f0713d323f423cf48f490bf96dcf17dce4e5f54 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 15 Apr 2025 09:20:06 -0400 Subject: [PATCH 14/21] pin binary-builder version --- tests/ci/build_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index e71e461df69a..908cac3d28a0 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -216,9 +216,9 @@ def main(): ) cargo_cache.download() - docker_image = docker_images_helper.pull_image( - docker_images_helper.get_docker_image(IMAGE_NAME) - ) + docker_image = docker_images_helper.get_docker_image(IMAGE_NAME) + docker_image.version = "e0a138049b31" + docker_image = docker_images_helper.pull_image(docker_image) packager_cmd = get_packager_cmd( build_config, From 10cba76e50f520a91fed3fccc6a7a9a7bcecfabf Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 15 Apr 2025 10:32:11 -0400 Subject: [PATCH 15/21] fix passing of aws credentials to builder --- docker/packager/packager | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 8c015255539d..92b85f7344d8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -91,14 +91,14 @@ def run_docker_image_with_env( if aws_secrets: # Pass AWS credentials via file rather than via env to avoid leaking secrets - env_part = {"AWS_CONFIG_FILE": "/home/clickhouse/.aws/credentials"} + env_part = {"AWS_SHARED_CREDENTIALS_FILE": "/aws_credentials"} host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' with open(host_aws_config_file_path, 'wt') as f: f.write("[default]") for key, value in aws_secrets.items(): f.write(f"\n{key}={value}") - extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']}" + extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_SHARED_CREDENTIALS_FILE']} -e AWS_SHARED_CREDENTIALS_FILE={env_part['AWS_SHARED_CREDENTIALS_FILE']}" env_part = " -e ".join(env_variables) if env_part: From 83a0b2ab5a36e5d9d60d46d7ab1fea523dd587b2 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 16 Apr 2025 07:39:01 -0400 Subject: [PATCH 16/21] update regression hash --- .github/workflows/release_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 0e03ab3859fd..e4f74f3ba99f 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -474,7 +474,7 @@ jobs: secrets: inherit with: runner_type: altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: 53d73ed32155a8a17ee0d0cdb15aee96c98010a2 + commit: a170f32119a5c872e5ff209b8f39e13acc2d6626 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 300 @@ -485,7 +485,7 @@ jobs: secrets: inherit with: runner_type: altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: 53d73ed32155a8a17ee0d0cdb15aee96c98010a2 + commit: a170f32119a5c872e5ff209b8f39e13acc2d6626 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 300 From 281e0d86c97a155203b2adde283a9c0bab510e72 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 16 Apr 2025 07:41:09 -0400 Subject: [PATCH 17/21] add regression job.retry attr --- .github/workflows/regression.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index f4ae11d7cf84..1142a1310867 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -179,7 +179,7 @@ jobs: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -243,7 +243,7 @@ jobs: -u alter/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} --only "/alter/${{ matrix.ONLY }} partition/*" - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -314,7 +314,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -374,7 +374,7 @@ jobs: -u ${{ env.SUITE }}/regression.py --ssl --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -436,7 +436,7 @@ jobs: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -494,7 +494,7 @@ jobs: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -562,7 +562,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -633,7 +633,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -703,7 +703,7 @@ jobs: --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} --with-${{ matrix.STORAGE }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE From bc4bbe02c35ada2c83df884cd00f7d3407c84c45 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 16 Apr 2025 07:43:54 -0400 Subject: [PATCH 18/21] update scanner regex --- tests/ci/s3_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 0353a92210a6..71cebf7fc575 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -21,7 +21,7 @@ ) sensitive_var_pattern = re.compile( - r"\b[A-Z_]*(? Date: Wed, 16 Apr 2025 08:13:20 -0400 Subject: [PATCH 19/21] revert dopckerfile changes --- docker/test/fasttest/Dockerfile | 7 ++----- docker/test/util/Dockerfile | 6 +++--- tests/ci/build_check.py | 6 +++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index e41bb5d88626..62d7dec60531 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -8,8 +8,7 @@ RUN apt-get update \ brotli \ clang-${LLVM_VERSION} \ clang-tidy-${LLVM_VERSION} \ - cmake=3.22.1-1ubuntu1 \ - cmake-data=3.22.1-1ubuntu1 \ + cmake \ expect \ file \ libclang-${LLVM_VERSION}-dev \ @@ -31,9 +30,7 @@ RUN apt-get update \ zstd \ --yes --no-install-recommends \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \ - && groupadd --system --gid 1000 clickhouse \ - && useradd --system --gid 1000 --uid 1000 -m clickhouse + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3 diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index 499357f82f08..91316346069d 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -33,9 +33,9 @@ RUN apt-get update \ # Install cmake 3.20+ for Rust support # Used https://askubuntu.com/a/1157132 as reference -RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg && \ - echo "deb [signed-by=/etc/apt/trusted.gpg.d/kitware.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" >> /etc/apt/sources.list.d/kitware.list - +RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | \ + gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg && \ + echo "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" >> /etc/apt/sources.list # initial packages RUN apt-get update \ diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 908cac3d28a0..e71e461df69a 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -216,9 +216,9 @@ def main(): ) cargo_cache.download() - docker_image = docker_images_helper.get_docker_image(IMAGE_NAME) - docker_image.version = "e0a138049b31" - docker_image = docker_images_helper.pull_image(docker_image) + docker_image = docker_images_helper.pull_image( + docker_images_helper.get_docker_image(IMAGE_NAME) + ) packager_cmd = get_packager_cmd( build_config, From 582b9f93e1138801f08bbf4b0ffe277dd099a99f Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 16 Apr 2025 10:35:33 -0400 Subject: [PATCH 20/21] update sensitive values regex --- tests/ci/s3_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 71cebf7fc575..a48f3a828917 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -21,7 +21,7 @@ ) sensitive_var_pattern = re.compile( - r"\b[A-Z_]*(? Date: Wed, 16 Apr 2025 14:53:35 -0400 Subject: [PATCH 21/21] update sensitive values regex --- tests/ci/s3_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index a48f3a828917..a473a108acd6 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -21,7 +21,7 @@ ) sensitive_var_pattern = re.compile( - r"\b[A-Z_]*(?