PoA quorum and HA failover fixes (cherry-pick e1588b6)

.cargo/audit.toml

@@ -1,4 +1,4 @@
 [advisories]
 ignore = [
     "RUSTSEC-2025-0009", # https://github.com/FuelLabs/fuel-core/issues/2814
-]
+]

.changes/changed/3225.md

@@ -0,0 +1 @@
+PoA quorum and HA failover fixes: Redis leader lease adapter improvements, write_block.lua HEIGHT_EXISTS check, sub-quorum block repair, Prometheus metrics, and chaos test harness.

.changes/changed/3237.md

@@ -0,0 +1 @@
+PoA quorum and HA failover fixes: Redis leader lease adapter improvements, write_block.lua HEIGHT_EXISTS check, sub-quorum block repair, Prometheus metrics, and chaos test harness.

.github/workflows/chaos-test.yml

@@ -0,0 +1,121 @@
+name: Leader Lock Chaos Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      seeds:
+        description: 'Seed range (e.g. "0-100")'
+        required: false
+        default: '0-50'
+      duration:
+        description: 'Test duration per seed (e.g. "30m")'
+        required: false
+        default: '30m'
+      block_time:
+        description: 'Block production interval'
+        required: false
+        default: '100ms'
+      fault_interval:
+        description: 'Average fault injection interval'
+        required: false
+        default: '1s'
+      stall_threshold:
+        description: 'Max allowed production stall'
+        required: false
+        default: '20s'
+      parallelism:
+        description: 'Number of seeds per runner'
+        required: false
+        default: '5'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - id: matrix
+        run: |
+          RANGE="${{ inputs.seeds }}"
+          START="${RANGE%-*}"
+          END="${RANGE#*-}"
+          BATCH_SIZE=${{ inputs.parallelism }}
+          BATCHES="["
+          FIRST=true
+          for ((i=START; i<=END; i+=BATCH_SIZE)); do
+            BATCH_END=$((i + BATCH_SIZE - 1))
+            if [ $BATCH_END -gt $END ]; then BATCH_END=$END; fi
+            if [ "$FIRST" = true ]; then FIRST=false; else BATCHES+=","; fi
+            BATCHES+="{\"start\":$i,\"end\":$BATCH_END}"
+          done
+          BATCHES+="]"
+          echo "matrix={\"batch\":$BATCHES}" >> "$GITHUB_OUTPUT"
+
+  chaos-test:
+    needs: prepare
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install Redis
+        run: sudo apt-get update && sudo apt-get install -y redis-server
+
+      - name: Cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: chaos-test-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Build chaos test
+        run: cargo build --release -p fuel-core-chaos-test
+
+      - name: Run chaos tests (seeds ${{ matrix.batch.start }}-${{ matrix.batch.end }})
+        run: |
+          FAILED=0
+          for seed in $(seq ${{ matrix.batch.start }} ${{ matrix.batch.end }}); do
+            echo "=== Seed $seed ==="
+            LOG="chaos_seed${seed}.log"
+            cargo run --release -p fuel-core-chaos-test -- \
+              --seed $seed \
+              --duration ${{ inputs.duration }} \
+              --block-time ${{ inputs.block_time }} \
+              --fault-interval ${{ inputs.fault_interval }} \
+              --stall-threshold ${{ inputs.stall_threshold }} \
+              > "$LOG" 2>&1
+            RC=$?
+            if [ $RC -ne 0 ]; then
+              echo "SEED $seed: FAIL"
+              grep -E "FORK|RESULT" "$LOG" | tail -3
+              if grep -q "FORK" "$LOG"; then
+                echo "::error::FORK detected at seed $seed"
+              fi
+              FAILED=$((FAILED + 1))
+            else
+              echo "SEED $seed: PASS"
+            fi
+          done
+          if [ $FAILED -gt 0 ]; then
+            echo "::error::$FAILED seed(s) failed"
+            exit 1
+          fi
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: chaos-logs-${{ matrix.batch.start }}-${{ matrix.batch.end }}
+          path: chaos_seed*.log
+          retention-days: 14

.github/workflows/ci.yml

@@ -22,7 +22,7 @@
   AWS_ROLE_ARN: arn:aws:iam::024848458133:role/github_oidc_FuelLabs_fuel-core
   AWS_ECR_ORG: fuellabs
   CARGO_TERM_COLOR: always
-  RUST_VERSION: 1.93.0
+  RUST_VERSION: 1.90.0
   RUST_VERSION_FMT: nightly-2025-09-28
   RUST_VERSION_COV: nightly-2025-09-28
   RUSTFLAGS: -D warnings
@@ -225,43 +225,65 @@
         run: cargo test -p fuel-core-tests --features aws-kms -- kms
 
   leader-lock-integration-tests:
-    name: Leader Lock Integration Tests (w/Redis)
+    name: Leader Lock Tests (w/Redis)
     needs:
       - lint-toml-files
       - prevent-openssl
       - rustfmt
       - check-changelog
     runs-on: warp-ubuntu-2404-x64-4x
     timeout-minutes: 45
     env:
       RUSTFLAGS: -D warnings
+      REDIS_VERSION: 8.6.0
     steps:
       - uses: actions/checkout@v6
       - name: Install Rust
         uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.RUST_VERSION }}
       - name: Install Redis
         run: |
           sudo apt-get update
-          sudo apt-get install -y redis-server
+          sudo apt-get install -y build-essential tcl curl ca-certificates
+          curl -fsSL "https://download.redis.io/releases/redis-${REDIS_VERSION}.tar.gz" -o /tmp/redis.tar.gz
+          tar -xzf /tmp/redis.tar.gz -C /tmp
+          cd "/tmp/redis-${REDIS_VERSION}"
+          make -j"$(nproc)"
+          sudo make install
+          redis-server --version
       - uses: rui314/setup-mold@v1
       - uses: Swatinem/rust-cache@v2
         with:
           key: leader-lock-integration-tests
           cache-on-failure: true
       - name: Start Redis
         run: |
-          redis-server --daemonize yes
-          redis-cli ping
+          redis-server \
+            --port 6379 \
+            --bind 127.0.0.1 \
+            --save "" \
+            --appendonly no \
+            --daemonize yes \
+            --pidfile /tmp/redis-ci.pid \
+            --dir /tmp
+          for _ in $(seq 1 20); do
+            if redis-cli -h 127.0.0.1 -p 6379 ping | grep -q PONG; then
+              exit 0
+            fi
+            sleep 0.5
+          done
+          echo "Redis failed to start"
+          exit 1
       - name: Run leader lock integration tests
         run: |
           echo "REDIS_URL=${REDIS_URL:-redis://127.0.0.1:6379} REDIS_DB=${REDIS_DB:-0} LEADER_LOCK_KEY_PREFIX=${LEADER_LOCK_KEY_PREFIX:-<unset>}"
           env | sort | grep -Ei 'REDIS|LEADER|LOCK|FUEL' || true
           (timeout 90s redis-cli MONITOR | stdbuf -oL grep -E 'SELECT|SET|PEXPIRE|DEL|leader|lock' || true) &
+          cargo test --package fuel-core --lib service::adapters::consensus_module::poa::tests:: --features leader_lock -- --test-threads=1 --nocapture
           cargo test --package fuel-core-tests --test integration_tests leader_lock --features leader_lock -- --test-threads=1 --nocapture
 
   verifications-complete:
     needs:
       - cargo-verifications
       - publish-crates-check
@@ -450,6 +472,7 @@
           asset_path: ./${{ env.ZIP_FILE_NAME }}
           asset_name: ${{ env.ZIP_FILE_NAME }}
           asset_content_type: application/gzip
+
   cargo-audit:
     runs-on: ubuntu-latest
     continue-on-error: true

.github/workflows/docker-images.yml

@@ -20,7 +20,7 @@ env:
   AWS_ROLE_ARN: arn:aws:iam::024848458133:role/github_oidc_FuelLabs_fuel-core
   AWS_ECR_ORG: fuellabs
   CARGO_TERM_COLOR: always
-  RUST_VERSION: 1.93.0
+  RUST_VERSION: 1.90.0
   RUST_VERSION_FMT: nightly-2023-10-29
   RUST_VERSION_COV: nightly-2024-06-05
   RUSTFLAGS: -D warnings
@@ -173,6 +173,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           build-args: "DEBUG_SYMBOLS=true"
 
+
   publish-e2e-client-docker-image:
     needs:
       - publish-docker-images