diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 3413cccdc764..1142a1310867 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -88,9 +88,9 @@ name: Regression test workflow - Release env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }} @@ -179,7 +179,7 @@ jobs: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -243,7 +243,7 @@ jobs: -u alter/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} --only "/alter/${{ matrix.ONLY }} partition/*" - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -314,7 +314,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -374,7 +374,7 @@ jobs: -u ${{ env.SUITE }}/regression.py --ssl --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -436,7 +436,7 @@ jobs: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -494,7 +494,7 @@ jobs: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -562,7 +562,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -633,7 +633,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE @@ -703,7 +703,7 @@ jobs: --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} --with-${{ matrix.STORAGE }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} || EXITCODE=$?; .github/add_link_to_logs.sh; exit $EXITCODE diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 7cc9cd849895..e4f74f3ba99f 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -7,6 +7,8 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} on: # yamllint disable-line rule:truthy pull_request: @@ -472,7 +474,7 @@ jobs: secrets: inherit with: runner_type: altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression - commit: 53d73ed32155a8a17ee0d0cdb15aee96c98010a2 + commit: a170f32119a5c872e5ff209b8f39e13acc2d6626 arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 300 @@ -483,7 +485,7 @@ jobs: secrets: inherit with: runner_type: altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression - commit: 53d73ed32155a8a17ee0d0cdb15aee96c98010a2 + commit: a170f32119a5c872e5ff209b8f39e13acc2d6626 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 300 diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index 4cff1c198c3b..637b9e94aaf6 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -8,6 +8,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} name: Build ClickHouse 'on': diff --git a/.github/workflows/reusable_docker.yml b/.github/workflows/reusable_docker.yml index 5a0178039ddc..7614257e4ae5 100644 --- a/.github/workflows/reusable_docker.yml +++ b/.github/workflows/reusable_docker.yml @@ -26,6 +26,9 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: DockerBuildAarch64: diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml index 5682f202e63b..ad9eab8d1692 100644 --- a/.github/workflows/reusable_sign.yml +++ b/.github/workflows/reusable_sign.yml @@ -63,6 +63,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: runner_labels_setup: diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 99aad0a3e36a..38a6794f555f 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -44,10 +44,25 @@ name: Testing workflow description: if given, it's passed to the environments required: false AWS_SECRET_ACCESS_KEY: - description: the access key to the aws param store. + description: the access key to the aws s3 bucket. required: true AWS_ACCESS_KEY_ID: - description: the access key id to the aws param store. + description: the access key id to the aws s3 bucket. + required: true + CLICKHOUSE_TEST_STAT_LOGIN: + description: username for ci db. + required: true + CLICKHOUSE_TEST_STAT_PASSWORD: + description: password for ci db. + required: true + CLICKHOUSE_TEST_STAT_URL: + description: url for ci db. + required: true + DOCKER_PASSWORD: + description: token to upload docker images. + required: true + ROBOT_TOKEN: + description: token to update ci status. required: true env: @@ -57,6 +72,11 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} jobs: runner_labels_setup: diff --git a/docker/packager/packager b/docker/packager/packager index aeb413f567d4..92b85f7344d8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -6,12 +6,13 @@ import os import subprocess import sys from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" - +DEFAULT_TMP_PATH = SCRIPT_PATH.parent.absolute() / 'tmp' +TEMP_PATH = Path(os.getenv("TEMP_PATH", DEFAULT_TMP_PATH)) class BuildException(Exception): pass @@ -82,9 +83,22 @@ def run_docker_image_with_env( ch_root: Path, cargo_cache_dir: Path, ccache_dir: Optional[Path], + aws_secrets : Optional[Dict[str,str]] ) -> None: output_dir.mkdir(parents=True, exist_ok=True) cargo_cache_dir.mkdir(parents=True, exist_ok=True) + extra_parts = "" + + if aws_secrets: + # Pass AWS credentials via file rather than via env to avoid leaking secrets + env_part = {"AWS_SHARED_CREDENTIALS_FILE": "/aws_credentials"} + host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config' + with open(host_aws_config_file_path, 'wt') as f: + f.write("[default]") + for key, value in aws_secrets.items(): + f.write(f"\n{key}={value}") + + extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_SHARED_CREDENTIALS_FILE']} -e AWS_SHARED_CREDENTIALS_FILE={env_part['AWS_SHARED_CREDENTIALS_FILE']}" env_part = " -e ".join(env_variables) if env_part: @@ -107,6 +121,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount} " f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " + f" {extra_parts} " f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}" ) @@ -130,11 +145,9 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, - s3_access_key_id: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, - s3_secret_access_key: str, clang_tidy: bool, version: str, official: bool, @@ -319,10 +332,6 @@ def parse_env_variables( result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}") if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") - if s3_access_key_id: - result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}") - if s3_secret_access_key: - result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}") if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. @@ -539,11 +548,9 @@ def main() -> None: args.sanitizer, args.package_type, args.cache, - args.s3_access_key_id, args.s3_bucket, args.s3_directory, args.s3_rw_access, - args.s3_secret_access_key, args.clang_tidy, args.version, args.official, @@ -562,6 +569,10 @@ def main() -> None: ch_root, args.cargo_cache_dir, args.ccache_dir, + { + "aws_access_key_id" : args.s3_access_key_id, + "aws_secret_access_key" : args.s3_secret_access_key + } ) logging.info("Output placed into %s", args.output_dir) diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 43fe915b765f..287fd30a7a44 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -4,9 +4,11 @@ import logging import time from pathlib import Path -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional import requests + +from env_helper import CLICKHOUSE_TEST_STAT_URL, CLICKHOUSE_TEST_STAT_PASSWORD, CLICKHOUSE_TEST_STAT_LOGIN from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import TestResults @@ -25,12 +27,12 @@ def __init__( self, url: Optional[str] = None, auth: Optional[Dict[str, str]] = None ): if url is None: - url = get_parameter_from_ssm("clickhouse-test-stat-url") + url = CLICKHOUSE_TEST_STAT_URL self.url = url self.auth = auth or { - "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"), - "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"), + "X-ClickHouse-User": CLICKHOUSE_TEST_STAT_LOGIN, + "X-ClickHouse-Key": CLICKHOUSE_TEST_STAT_PASSWORD, } @staticmethod @@ -40,6 +42,7 @@ def insert_file( query: str, file: Path, additional_options: Optional[Dict[str, str]] = None, + **kwargs: Any, ) -> None: params = { "query": query, @@ -52,7 +55,7 @@ def insert_file( with open(file, "rb") as data_fd: ClickHouseHelper._insert_post( - url, params=params, data=data_fd, headers=auth + url, params=params, data=data_fd, headers=auth, **kwargs ) @staticmethod diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py index fe7b80a3b669..cc9671ec773e 100644 --- a/tests/ci/docker_images_helper.py +++ b/tests/ci/docker_images_helper.py @@ -7,8 +7,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from env_helper import ROOT_DIR, DOCKER_TAG -from get_robot_token import get_parameter_from_ssm +from env_helper import ROOT_DIR, DOCKER_TAG, DOCKER_PASSWORD IMAGES_FILE_PATH = Path("docker/images.json") @@ -28,7 +27,7 @@ def docker_login(relogin: bool = True) -> None: logging.info('Doing docker login') subprocess.check_output( # pylint: disable=unexpected-keyword-arg "docker login --username 'altinityinfra' --password-stdin", - input=get_parameter_from_ssm("dockerhub-password"), + input=DOCKER_PASSWORD, encoding="utf-8", shell=True, ) @@ -56,7 +55,7 @@ def pull_image(image: DockerImage) -> DockerImage: ) logging.info("Pulling image %s - done", image) except Exception as ex: - logging.info("Got execption pulling docker %s", ex) + logging.info("Got exception pulling docker %s", ex) raise ex return image diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index a8a00b02882b..c1946eb5aa2a 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -15,6 +15,7 @@ REPORT_PATH = f"{TEMP_PATH}/reports" # FIXME: latest should not be used in CI, set temporary for transition to "docker with digest as a tag" DOCKER_TAG = os.getenv("DOCKER_TAG", "latest") +DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") @@ -37,6 +38,11 @@ f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/" "{pr_or_release}/{commit}/{build_name}/{artifact}" ) +CLICKHOUSE_TEST_STAT_LOGIN = os.getenv("CLICKHOUSE_TEST_STAT_LOGIN") +CLICKHOUSE_TEST_STAT_PASSWORD = os.getenv("CLICKHOUSE_TEST_STAT_PASSWORD") +CLICKHOUSE_TEST_STAT_URL = os.getenv("CLICKHOUSE_TEST_STAT_URL") +DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") +ROBOT_TOKEN = os.getenv("ROBOT_TOKEN") # These parameters are set only on demand, and only once _GITHUB_JOB_ID = "" diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 5639b72fa3bc..abadb30adce1 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 import logging +import os +import random from dataclasses import dataclass from typing import Any, Dict, List, Optional, Union @@ -9,6 +11,7 @@ from github.GithubException import BadCredentialsException from github.NamedUser import NamedUser +ROBOT_TOKEN = os.getenv("ROBOT_TOKEN") @dataclass class Token: @@ -17,6 +20,9 @@ class Token: rest: int +SAFE_REQUESTS_LIMIT = 1000 + + def get_parameter_from_ssm( name: str, decrypt: bool = True, client: Optional[Any] = None ) -> str: @@ -52,20 +58,11 @@ def get_parameters_from_ssm( return results - -ROBOT_TOKEN = None # type: Optional[Token] - # NOTE(Arthur Passos): Original CI code uses the "_original" version of this method. Each robot token is rate limited # and the original implementation selects the "best one". To make it simpler and iterate faster, # we are using only one robot and keeping the method signature. In the future we might reconsider # having multiple robot tokens -def get_best_robot_token(token_prefix_env_name="github_robot_token"): - # Re-use already fetched token (same as in get_best_robot_token_original) - # except here we assume it is always a string (since we use only one token and don't do token rotation) - global ROBOT_TOKEN - if ROBOT_TOKEN is not None: - return ROBOT_TOKEN - ROBOT_TOKEN = get_parameter_from_ssm(token_prefix_env_name) +def get_best_robot_token(): return ROBOT_TOKEN def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: @@ -81,10 +78,15 @@ def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: } assert tokens - for name, value in tokens.items(): + token_items = list(tokens.items()) + random.shuffle(token_items) + + best_token: Optional[Token] = None + + for name, value in token_items: gh = Github(value, per_page=100) try: - # Do not spend additional request to API by accessin user.login unless + # Do not spend additional request to API by accessing user.login unless # the token is chosen by the remaining requests number user = gh.get_user() rest, _ = gh.rate_limiting @@ -96,13 +98,14 @@ def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: ) continue logging.info("Get token with %s remaining requests", rest) - if ROBOT_TOKEN is None: - ROBOT_TOKEN = Token(user, value, rest) - continue - if ROBOT_TOKEN.rest < rest: - ROBOT_TOKEN.user, ROBOT_TOKEN.value, ROBOT_TOKEN.rest = user, value, rest - - assert ROBOT_TOKEN + if best_token is None: + best_token = Token(user, value, rest) + elif best_token.rest < rest: + best_token = Token(user, value, rest) + if best_token.rest > SAFE_REQUESTS_LIMIT: + break + assert best_token + ROBOT_TOKEN = best_token logging.info( "User %s with %s remaining requests is used", ROBOT_TOKEN.user.login, diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index e073430df953..a473a108acd6 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -6,6 +6,7 @@ from multiprocessing.dummy import Pool from pathlib import Path from typing import Any, List, Union +import os import boto3 # type: ignore import botocore # type: ignore @@ -19,6 +20,42 @@ S3_URL, ) +sensitive_var_pattern = re.compile( + r"\b[A-Z_]*(? str: + logging.debug("Checking %s for sensitive values", file_path) + try: + file_content = file_path.read_text(encoding="utf-8") + except UnicodeDecodeError: + logging.warning("Failed to read file %s, unknown encoding", file_path) + else: + scan_file_for_sensitive_data(file_content, file_path.name) + logging.debug( "Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path )