diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 81a6a395bf3d..d3a3690b6219 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -348,6 +348,7 @@ jobs: # - BuilderBinDarwin # - BuilderBinDarwinAarch64 - BuilderDebRelease + - SignRelease # - BuilderDebAarch64 runs-on: [self-hosted, style-checker] steps: @@ -363,7 +364,7 @@ jobs: #################################### INSTALL PACKAGES ###################################### ############################################################################################ InstallPackagesTestRelease: - needs: [BuilderDebRelease] + needs: [SignRelease] runs-on: [self-hosted, style-checker] steps: - name: Set envs @@ -428,11 +429,18 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" + + tests_start: + ## Do-nothing stage to trigger tests, makes is easier to + needs: [InstallPackagesTestRelease] + runs-on: ubuntu-latest + steps: + - run: true ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### ############################################################################################## FunctionalStatelessTestRelease: - needs: [BuilderDebRelease] + needs: [tests_start] runs-on: [self-hosted, func-tester] steps: - name: Set envs @@ -505,7 +513,7 @@ jobs: ############################ FUNCTIONAl STATEFUL TESTS ####################################### ############################################################################################## FunctionalStatefulTestRelease: - needs: [BuilderDebRelease] + needs: [tests_start] runs-on: [self-hosted, func-tester] steps: - name: Set envs @@ -578,7 +586,7 @@ jobs: ############################# INTEGRATION TESTS ############################################# ############################################################################################# IntegrationTestsRelease0: - needs: [BuilderDebRelease] + needs: [tests_start] runs-on: [self-hosted, stress-tester] steps: - name: Set envs @@ -613,7 +621,7 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: - needs: [BuilderDebRelease] + needs: [tests_start] runs-on: [self-hosted, stress-tester] steps: - name: Set envs @@ -652,7 +660,7 @@ jobs: ############################################################################################# regression_start: ## Not depending on the tests above since they can fail at any given moment. - needs: [BuilderDebRelease] + needs: [tests_start] runs-on: ubuntu-latest steps: - run: true diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 06b4be289f1e..66b95184666e 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -368,6 +368,9 @@ "SQLancer (debug)": { "required_build": "package_debug", }, + "Sign release (actions)": { + "required_build": "package_release" + } }, } # type: dict diff --git a/tests/ci/sign_release.py b/tests/ci/sign_release.py new file mode 100644 index 000000000000..701ccc29b65b --- /dev/null +++ b/tests/ci/sign_release.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import sys +import os +import logging +from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH +from s3_helper import S3Helper +from pr_info import PRInfo +from build_download_helper import download_builds_filter +import hashlib + +GPG_BINARY_SIGNING_KEY = os.getenv("GPG_BINARY_SIGNING_KEY") +GPG_BINARY_SIGNING_PASSPHRASE = os.getenv("GPG_BINARY_SIGNING_PASSPHRASE") + +CHECK_NAME = "Sign release (actions)" + +def hash_file(file_path): + BLOCK_SIZE = 65536 # The size of each read from the file + + file_hash = hashlib.sha256() # Create the hash object, can use something other than `.sha256()` if you wish + with open(file_path, 'rb') as f: # Open the file to read it's bytes + fb = f.read(BLOCK_SIZE) # Read from the file. Take in the amount declared above + while len(fb) > 0: # While there is still data being read from the file + file_hash.update(fb) # Update the hash + fb = f.read(BLOCK_SIZE) # Read the next block from the file + + hash_file_path = file_path + '.sha256' + with open(hash_file_path, 'x') as f: + digest = file_hash.hexdigest() + f.write(digest) + print(f'Hashed {file_path}: {digest}') + + return hash_file_path + +def sign_file(file_path): + priv_key_file_path = 'priv.key' + with open(priv_key_file_path, 'x') as f: + f.write(GPG_BINARY_SIGNING_KEY) + + out_file_path = f'{file_path}.gpg' + + os.system(f'echo {GPG_BINARY_SIGNING_PASSPHRASE} | gpg --batch --import {priv_key_file_path}') + os.system(f'gpg -o {out_file_path} --pinentry-mode=loopback --batch --yes --passphrase {GPG_BINARY_SIGNING_PASSPHRASE} --sign {file_path}') + print(f"Signed {file_path}") + os.remove(priv_key_file_path) + + return out_file_path + +def main(): + reports_path = REPORTS_PATH + + if not os.path.exists(TEMP_PATH): + os.makedirs(TEMP_PATH) + + pr_info = PRInfo() + + logging.info("Repo copy path %s", REPO_COPY) + + s3_helper = S3Helper() + + s3_path_prefix = f"{pr_info.number}/{pr_info.sha}/" + CHECK_NAME.lower().replace( + " ", "_" + ).replace("(", "_").replace(")", "_").replace(",", "_") + + # downloads `package_release` artifacts generated + download_builds_filter(CHECK_NAME, reports_path, TEMP_PATH) + + for f in os.listdir(TEMP_PATH): + full_path = os.path.join(TEMP_PATH, f) + hashed_file_path = hash_file(full_path) + signed_file_path = sign_file(hashed_file_path) + s3_path = f'{s3_path_prefix}/{os.path.basename(signed_file_path)}' + s3_helper.upload_build_file_to_s3(signed_file_path, s3_path) + print(f'Uploaded file {signed_file_path} to {s3_path}') + + # Signed hashes are: + # clickhouse-client_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-client-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-client_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg + # clickhouse-client-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg + # clickhouse-common-static_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-common-static-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-common-static_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper.sha512.gpg + # clickhouse-common-static-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-library-bridge.sha512.gpg + # clickhouse-common-static-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-odbc-bridge.sha512.gpg + # clickhouse-common-static-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_amd64.deb.sha512.gpg + # clickhouse-common-static-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-server-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg + # clickhouse-common-static-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-keeper_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-server-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-keeper-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse.sha512.gpg + + sys.exit(0) + +if __name__ == "__main__": + main()