SignRelease cherrypick

.github/workflows/release_branches.yml

@@ -981,7 +981,48 @@ jobs:
             ./*/_instances/*/logs/*.log
             ./*/*/_instances/*/logs/*.log
             ./*/*/_instances/*.log
-
+
+  SignRelease:
+    needs: [BuilderDebRelease]
+    runs-on: [ self-hosted ]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/signed
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          EOF
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Sign release
+        env:
+          GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }}
+          GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }}
+          REPORTS_PATH: ${{ env.REPORTS_PATH }}
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 sign_release.py
+      - name: Upload signed hashes
+        uses: actions/upload-artifact@v2
+        with:
+          name: signed-hashes
+          path: ${{ env.TEMP_PATH }}/*.gpg
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  ###########################################################################################
+  ################################ FINISH CHECK #############################################
+  ###########################################################################################
   FinishCheck:
     needs:
       - DockerHubPush
@@ -996,6 +1037,7 @@ jobs:
       - IntegrationTestsRelease0
       - IntegrationTestsRelease1
       - CompatibilityCheck
+      - SignRelease
       - regression_common
       - benchmark
       - ldap

tests/ci/ci_config.py

@@ -335,6 +335,9 @@
             "required_build": "package_aarch64",
             "test_grep_exclude_filter": "constant_column_search",
         },
+        "Sign release (actions)": {
+            "required_build": "package_release"
+        }
     },
 }  # type: dict
 

tests/ci/env_helper.py

@@ -11,6 +11,8 @@
 
 CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH)
 CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN")
+GPG_BINARY_SIGNING_KEY = os.getenv("GPG_BINARY_SIGNING_KEY")
+GPG_BINARY_SIGNING_PASSPHRASE = os.getenv("GPG_BINARY_SIGNING_PASSPHRASE")
 GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "")
 GITHUB_JOB = os.getenv("GITHUB_JOB", "local")
 GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "Altinity/ClickHouse")

tests/ci/sign_release.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+import sys
+import os
+import logging
+from env_helper import GPG_BINARY_SIGNING_KEY, GPG_BINARY_SIGNING_PASSPHRASE, TEMP_PATH, REPO_COPY, REPORTS_PATH
+from s3_helper import S3Helper
+from pr_info import PRInfo
+from build_download_helper import download_builds_filter
+import hashlib
+
+
+CHECK_NAME = "Sign release (actions)"
+
+def hash_file(file_path):
+    BLOCK_SIZE = 65536 # The size of each read from the file
+
+    file_hash = hashlib.sha256() # Create the hash object, can use something other than `.sha256()` if you wish
+    with open(file_path, 'rb') as f: # Open the file to read it's bytes
+        fb = f.read(BLOCK_SIZE) # Read from the file. Take in the amount declared above
+        while len(fb) > 0: # While there is still data being read from the file
+            file_hash.update(fb) # Update the hash
+            fb = f.read(BLOCK_SIZE) # Read the next block from the file
+
+    hash_file_path = file_path + '.sha256'
+    with open(hash_file_path, 'x') as f:
+        digest = file_hash.hexdigest()
+        f.write(digest)
+        print(f'Hashed {file_path}: {digest}')
+
+    return hash_file_path
+
+def sign_file(file_path):
+    priv_key_file_path = 'priv.key'
+    with open(priv_key_file_path, 'x') as f:
+        f.write(GPG_BINARY_SIGNING_KEY)
+
+    out_file_path = f'{file_path}.gpg'
+
+    os.system(f'echo {GPG_BINARY_SIGNING_PASSPHRASE} | gpg --batch --import {priv_key_file_path}')
+    os.system(f'gpg -o {out_file_path} --pinentry-mode=loopback --batch --yes --passphrase {GPG_BINARY_SIGNING_PASSPHRASE} --sign {file_path}')
+    print(f"Signed {file_path}")
+    os.remove(priv_key_file_path)
+
+    return out_file_path
+
+def main():
+    reports_path = REPORTS_PATH
+
+    if not os.path.exists(TEMP_PATH):
+        os.makedirs(TEMP_PATH)
+
+    pr_info = PRInfo()
+
+    logging.info("Repo copy path %s", REPO_COPY)
+
+    s3_helper = S3Helper("https://s3.amazonaws.com")
+
+    s3_path_prefix = f"{pr_info.number}/{pr_info.sha}/" + CHECK_NAME.lower().replace(
+        " ", "_"
+    ).replace("(", "_").replace(")", "_").replace(",", "_")
+
+    # downloads `package_release` artifacts generated
+    download_builds_filter(CHECK_NAME, reports_path, TEMP_PATH)
+
+    for f in os.listdir(TEMP_PATH):
+        full_path = os.path.join(TEMP_PATH, f)
+        hashed_file_path = hash_file(full_path)
+        signed_file_path = sign_file(hashed_file_path)
+        s3_path = f'{s3_path_prefix}/{os.path.basename(signed_file_path)}'
+        s3_helper.upload_build_file_to_s3(signed_file_path, s3_path)
+        print(f'Uploaded file {signed_file_path} to {s3_path}')
+
+    # Signed hashes are:
+    # clickhouse-client_22.3.15.2.altinitystable_amd64.deb.sha512.gpg              clickhouse-keeper_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg
+    # clickhouse-client-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg              clickhouse-keeper-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg
+    # clickhouse-client_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg             clickhouse-keeper-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg
+    # clickhouse-client-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg             clickhouse-keeper-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg
+    # clickhouse-common-static_22.3.15.2.altinitystable_amd64.deb.sha512.gpg       clickhouse-keeper-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg
+    # clickhouse-common-static-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg       clickhouse-keeper-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg
+    # clickhouse-common-static_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg      clickhouse-keeper.sha512.gpg
+    # clickhouse-common-static-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg      clickhouse-library-bridge.sha512.gpg
+    # clickhouse-common-static-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg   clickhouse-odbc-bridge.sha512.gpg
+    # clickhouse-common-static-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg   clickhouse-server_22.3.15.2.altinitystable_amd64.deb.sha512.gpg
+    # clickhouse-common-static-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg  clickhouse-server-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg
+    # clickhouse-common-static-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg  clickhouse-server_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg
+    # clickhouse-keeper_22.3.15.2.altinitystable_amd64.deb.sha512.gpg              clickhouse-server-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg
+    # clickhouse-keeper-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg              clickhouse.sha512.gpg
+
+    sys.exit(0)
+
+if __name__ == "__main__":
+    main()