Project antalya backport remote s3 cluster

.github/actionlint.yml

@@ -4,7 +4,10 @@ self-hosted-runner:
     - func-tester
     - func-tester-aarch64
     - fuzzer-unit-tester
-    - stress-tester
+    - altinity-on-demand
+    - altinity-type-cpx51
+    - altinity-in-ash
+    - altinity-image-x86-system-ubuntu-22.04
     - style-checker
     - style-checker-aarch64
     - release-maker

.github/actions/common_setup/action.yml

@@ -28,6 +28,16 @@ runs:
       run: |
           # to remove every leftovers
           sudo rm -fr "$TEMP_PATH" && mkdir -p "$TEMP_PATH"
+    - name: Setup zram
+      shell: bash
+      run: |
+          sudo modprobe zram
+          MemTotal=$(grep -Po "(?<=MemTotal:)\s+\d+" /proc/meminfo) # KiB
+          Percent=200
+          ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB
+          .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0
+          sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0
+          sudo sysctl vm.swappiness=200
     - name: Tune vm.mmap_rnd_bits for sanitizers
       shell: bash
       run: |

.github/actions/debug/action.yml

@@ -0,0 +1,34 @@
+name: DebugInfo
+description: Prints workflow debug info
+
+runs:
+  using: "composite"
+  steps:
+    - name: Envs, event.json and contexts
+      shell: bash
+      run: |
+          echo '::group::Environment variables'
+          env | sort
+          echo '::endgroup::'
+
+          echo '::group::event.json'
+          python3 -m json.tool "$GITHUB_EVENT_PATH"
+          echo '::endgroup::'
+
+          cat << 'EOF'
+          ::group::github context
+          ${{ toJSON(github) }}
+          ::endgroup::
+
+          ::group::env context
+          ${{ toJSON(env) }}
+          ::endgroup::
+
+          ::group::runner context
+          ${{ toJSON(runner) }}
+          ::endgroup::
+
+          ::group::job context
+          ${{ toJSON(job) }}
+          ::endgroup::
+          EOF

.github/actions/docker_setup/action.yml

@@ -0,0 +1,29 @@
+name: Docker setup
+description: Setup docker
+inputs:
+  nested_job:
+    description: the fuse for unintended use inside of the reusable callable jobs
+    default: true
+    type: boolean
+runs:
+  using: "composite"
+  steps:
+    - name: Docker IPv6 configuration
+      shell: bash
+      run: |
+        # make sure docker uses proper IPv6 config
+        sudo touch /etc/docker/daemon.json
+        sudo chown ubuntu:ubuntu /etc/docker/daemon.json
+        sudo cat <<EOT > /etc/docker/daemon.json
+        {
+          "ipv6": true,
+          "fixed-cidr-v6": "2001:3984:3989::/64"
+        }
+        EOT
+        sudo chown root:root /etc/docker/daemon.json
+        sudo systemctl restart docker
+        sudo systemctl status docker
+    - name: Docker info
+      shell: bash
+      run: |
+        docker info

.github/retry.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Execute command until exitcode is 0 or
+# maximum number of retries is reached
+# Example:
+#    ./retry <retries> <delay> <command>
+retries=$1
+delay=$2
+command="${@:3}"
+exitcode=0
+try=0
+until [ "$try" -ge $retries ]
+do
+  echo "$command"
+  eval "$command"
+  exitcode=$?
+  if [ $exitcode -eq 0 ]; then
+    break
+  fi
+  try=$((try+1))
+  sleep $2
+done
+exit $exitcode

.github/workflows/backport_branches.yml

@@ -22,11 +22,13 @@ jobs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get version
           filter: tree:0
+      - name: Debug Info
+        uses: ./.github/actions/debug
       - name: Labels check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -162,7 +164,7 @@ jobs:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
       - name: Download reports
         run: |
           python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
@@ -192,7 +194,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Install packages (aarch64)
-      runner_type: style-checker-aarch64
+      runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 install_check.py "$CHECK_NAME"
@@ -227,18 +229,26 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Stress test (tsan)
-      runner_type: stress-tester
+      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
 #############################################################################################
 ############################# INTEGRATION TESTS #############################################
 #############################################################################################
-  IntegrationTestsRelease:
-    needs: [RunConfig, BuilderDebRelease]
+  IntegrationTestsAsanOldAnalyzer:
+    needs: [RunConfig, BuilderDebAsan]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Integration tests (asan, old analyzer)
+      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 
+      data: ${{ needs.RunConfig.outputs.data }}
+  IntegrationTestsTsan:
+    needs: [RunConfig, BuilderDebTsan]
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: Integration tests (release)
-      runner_type: stress-tester
+      test_name: Integration tests (tsan)
+      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
   FinishCheck:
     if: ${{ !cancelled() }}
@@ -248,13 +258,14 @@ jobs:
       - FunctionalStatelessTestAsan
       - FunctionalStatefulTestDebug
       - StressTestTsan
-      - IntegrationTestsRelease
+      - IntegrationTestsTsan
+      - IntegrationTestsAsanOldAnalyzer
       - CompatibilityCheckX86
       - CompatibilityCheckAarch64
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true
       - name: Finish label

.github/workflows/cherry_pick.yml

@@ -28,11 +28,13 @@ jobs:
           REPO_TEAM=core
           EOF
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
+      - name: Debug Info
+        uses: ./.github/actions/debug
       - name: Cherry pick
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"

.github/workflows/create_release.yml

@@ -33,13 +33,13 @@ jobs:
       GH_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
     runs-on: [self-hosted, release-maker]
     steps:
-      - name: DebugInfo
-        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
+      - name: Debug Info
+        uses: ./.github/actions/debug
       - name: Prepare Release Info
         shell: bash
         run: |

.github/workflows/docker_test_images.yml

@@ -10,15 +10,31 @@ name: Build docker images
         description: set latest tag for resulting multiarch manifest
         required: false
         type: boolean
-        default: false
+        default: false    
+    secrets:
+      secret_envs:
+        description: if given, it's passed to the environments
+        required: false
+      AWS_SECRET_ACCESS_KEY:
+        description: the access key to the aws param store.
+        required: true
+      AWS_ACCESS_KEY_ID:
+        description: the access key id to the aws param store.
+        required: true
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
 jobs:
   DockerBuildAarch64:
-    runs-on: [self-hosted, style-checker-aarch64]
+    runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none]
     if: |
       !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_aarch64) != '[]'
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
         with:
           ref: ${{ fromJson(inputs.data).git_ref }}
       - name: Build images
@@ -28,12 +44,12 @@ jobs:
             --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
             --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_aarch64) }}'
   DockerBuildAmd64:
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
     if: |
       !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_amd64) != '[]'
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
         with:
           ref: ${{ fromJson(inputs.data).git_ref }}
       - name: Build images
@@ -44,12 +60,12 @@ jobs:
             --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_amd64) }}'
   DockerMultiArchManifest:
     needs: [DockerBuildAmd64, DockerBuildAarch64]
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
     if: |
       !failure() && !cancelled() && (toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' || inputs.set_latest)
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
         with:
           ref: ${{ fromJson(inputs.data).git_ref }}
       - name: Build images

.github/workflows/jepsen.yml

@@ -8,27 +8,28 @@ on: # yamllint disable-line rule:truthy
   schedule:
     - cron: '0 */6 * * *'
   workflow_dispatch:
+
 jobs:
   RunConfig:
     runs-on: [self-hosted, style-checker-aarch64]
     outputs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
-      - name: DebugInfo
-        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get version
           filter: tree:0
+      - name: Debug Info
+        uses: ./.github/actions/debug
       - name: PrepareRunConfig
         id: runconfig
         run: |
           echo "::group::configure CI run"
           python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --workflow "$GITHUB_WORKFLOW" --outfile ${{ runner.temp }}/ci_run_data.json
           echo "::endgroup::"
-          
+
           echo "::group::CI run configure results"
           python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
           echo "::endgroup::"
@@ -42,7 +43,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse Keeper Jepsen
-      runner_type: style-checker-aarch64
+      runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 jepsen_check.py keeper
@@ -52,7 +53,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse Server Jepsen
-      runner_type: style-checker-aarch64
+      runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 jepsen_check.py server
@@ -62,7 +63,7 @@ jobs:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
       - name: Check Workflow results
         if: ${{ !cancelled() }}
         run: |

.github/workflows/master.yml

@@ -15,14 +15,14 @@ jobs:
     outputs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
-      - name: DebugInfo
-        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get version
           filter: tree:0
+      - name: Debug Info
+        uses: ./.github/actions/debug
       - name: Merge sync PR
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -118,7 +118,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Builds
-      runner_type: style-checker-aarch64
+      runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce
       data: ${{ needs.RunConfig.outputs.data }}
 
   FinishCheck:
@@ -127,7 +127,7 @@ jobs:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"