Merge bitcoin#34952: Update libmultiprocess subtree in 31.x branch to fix race conditions on disconnects

2478a15 Squashed 'src/ipc/libmultiprocess/' changes from 1868a84451f..70f632bda8f (Ryan Ofsky)

Pull request description:

  This PR is a backport of bitcoin#34804 to the 31.x branch. (Both PR's point to the same source branch, they just have different target branches.) Some previous discussion about whether these changes should be merged into 31.x happened in bitcoin-core/libmultiprocess#249 (comment). The changes fix some IPC crashes that can happen with broken clients and unlucky thread timing that have only been seen in tests and antithesis runs, but the fixes are fairly simple and seem unlikely to cause new problems. The other changes in the PR are mostly CI/documentation/test changes and should also be safe.

  Includes:

  - bitcoin-core/libmultiprocess#246
  - bitcoin-core/libmultiprocess#242
  - bitcoin-core/libmultiprocess#247
  - bitcoin-core/libmultiprocess#251
  - bitcoin-core/libmultiprocess#255
  - bitcoin-core/libmultiprocess#258
  - bitcoin-core/libmultiprocess#262
  - bitcoin-core/libmultiprocess#253
  - bitcoin-core/libmultiprocess#263
  - bitcoin-core/libmultiprocess#256
  - bitcoin-core/libmultiprocess#264
  - bitcoin-core/libmultiprocess#249
  - bitcoin-core/libmultiprocess#265

  The main change is bitcoin-core/libmultiprocess#249 which fixes 3 intermittent race conditions detected in bitcoin core CI and antithesis: bitcoin#34711/bitcoin#34756, bitcoin#34777, and bitcoin#34782.

  The changes can be verified by running `test/lint/git-subtree-check.sh src/ipc/libmultiprocess` as described in [developer notes](https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#subtrees) and [lint instructions](https://github.com/bitcoin/bitcoin/tree/master/test/lint#git-subtree-checksh)

ACKs for top commit:
  Sjors:
    ACK 85c016b

Tree-SHA512: 54358428dc5a9cea84c3e816136ab828e702fc04b5af03cfd81c60522f1de491bf4867aed2e6c6791da2725dff2004b398ebbf42dd882cc22a5912bc5945cb6e

Author: fanquake

src/ipc/libmultiprocess/.github/workflows/bitcoin-core-ci.yml

@@ -0,0 +1,300 @@
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit.
+
+# Test libmultiprocess inside Bitcoin Core by replacing the subtree copy
+# with the version from this PR, then building and running IPC-related
+# unit & functional tests.
+
+name: Bitcoin Core CI
+
+on:
+  push:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  BITCOIN_REPO: bitcoin/bitcoin
+  LLVM_VERSION: 22
+  LIBCXX_DIR: /tmp/libcxx-build/
+
+jobs:
+  bitcoin-core:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 120
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: 'ASan + UBSan'
+            unit_test_runs: 15
+            functional_test_runs: 20
+            nproc_multiplier: 2
+            functional_timeout_factor: 40
+            runner: ubuntu-24.04
+            apt-llvm: true
+            packages: >-
+              ccache
+              clang-22
+              llvm-22
+              libclang-rt-22-dev
+              libevent-dev
+              libboost-dev
+              libsqlite3-dev
+              libcapnp-dev
+              capnproto
+              ninja-build
+              pkgconf
+              python3-pip
+            pip-packages: --break-system-packages pycapnp
+            cmake-args: |-
+              -DSANITIZERS=address,float-divide-by-zero,integer,undefined
+              -DCMAKE_C_COMPILER=clang
+              -DCMAKE_CXX_COMPILER=clang++
+              -DCMAKE_C_FLAGS=-ftrivial-auto-var-init=pattern
+              -DCMAKE_CXX_FLAGS=-ftrivial-auto-var-init=pattern
+
+          - name: 'macOS'
+            unit_test_runs: 50
+            functional_test_runs: 20
+            nproc_multiplier: 2
+            functional_timeout_factor: 40
+            runner: macos-15
+            brew-packages: ccache capnp boost libevent sqlite pkgconf ninja
+            pip-packages: --break-system-packages pycapnp
+            cmake-args: |-
+              -DREDUCE_EXPORTS=ON
+
+    env:
+      CCACHE_MAXSIZE: 400M
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+
+    steps:
+      - name: Checkout Bitcoin Core
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.BITCOIN_REPO }}
+          fetch-depth: 1
+
+      - name: Checkout libmultiprocess
+        uses: actions/checkout@v4
+        with:
+          path: _libmultiprocess
+
+      - name: Replace libmultiprocess subtree
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh replace_subtree
+
+      - name: Add LLVM apt repository
+        if: matrix.apt-llvm
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh add_llvm_apt_repository
+
+      - name: Install APT packages
+        if: matrix.packages
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh install_apt_packages ${{ matrix.packages }}
+
+      - name: Configure LLVM alternatives
+        if: matrix.packages
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh install_llvm_alternatives
+
+      - name: Install Homebrew packages
+        if: matrix.brew-packages
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh install_homebrew_packages ${{ matrix.brew-packages }}
+
+      - name: Install pip packages
+        if: matrix.pip-packages
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh install_pip_packages ${{ matrix.pip-packages }}
+
+      - name: Determine parallelism
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh determine_parallelism "${{ matrix.nproc_multiplier }}"
+
+      - name: Restore ccache
+        id: ccache-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }}
+          restore-keys: |
+            ccache-${{ matrix.name }}-${{ github.ref }}-
+            ccache-${{ matrix.name }}-
+
+      - name: Reset ccache stats
+        if: matrix.packages || matrix.brew-packages
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh reset_ccache_stats
+
+      - name: CMake configure
+        env:
+          BITCOIN_CORE_CMAKE_ARGS: ${{ matrix.cmake-args }}
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh configure_bitcoin_core
+
+      - name: Build
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_bitcoin_core
+
+      - name: Show ccache stats
+        if: matrix.packages || matrix.brew-packages
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh show_ccache_stats
+
+      - name: Run IPC unit tests
+        env:
+          ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1
+          LSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/lsan
+          UBSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh run_ipc_unit_tests "${{ matrix.unit_test_runs }}"
+
+      - name: Run IPC functional tests
+        env:
+          ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1
+          LSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/lsan
+          UBSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1
+          CI_FAILFAST_TEST_LEAVE_DANGLING: 1
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh run_ipc_functional_tests "${{ matrix.functional_test_runs }}" "${{ matrix.functional_timeout_factor }}"
+
+      - name: Save ccache
+        uses: actions/cache/save@v4
+        if: github.ref == 'refs/heads/master' || steps.ccache-restore.outputs.cache-hit != 'true'
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }}
+
+  bitcoin-core-tsan:
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 180
+
+    strategy:
+      matrix:
+        include:
+          - name: TSan
+            unit_test_runs: 8
+            functional_test_runs: 25
+            nproc_multiplier: 2
+            functional_timeout_factor: 40
+
+    env:
+      CCACHE_MAXSIZE: 400M
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      LIBCXX_FLAGS: >-
+        -fsanitize=thread
+        -nostdinc++
+        -nostdlib++
+        -isystem /tmp/libcxx-build/include/c++/v1
+        -L/tmp/libcxx-build/lib
+        -Wl,-rpath,/tmp/libcxx-build/lib
+        -lc++
+        -lc++abi
+        -lpthread
+        -Wno-unused-command-line-argument
+      TSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1
+
+    steps:
+      - name: Checkout Bitcoin Core
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.BITCOIN_REPO }}
+          fetch-depth: 1
+
+      - name: Checkout libmultiprocess
+        uses: actions/checkout@v4
+        with:
+          path: _libmultiprocess
+
+      - name: Add LLVM apt repository
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh add_llvm_apt_repository
+
+      - name: Install packages
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh install_tsan_packages
+
+      - name: Determine parallelism
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh determine_parallelism "${{ matrix.nproc_multiplier }}"
+
+      - name: Restore instrumented libc++ cache
+        id: libcxx-cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.LIBCXX_DIR }}
+          key: libcxx-Thread-llvmorg-${{ env.LLVM_VERSION }}.1.0
+
+      - name: Build instrumented libc++
+        if: steps.libcxx-cache.outputs.cache-hit != 'true'
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_instrumented_libcxx
+
+      - name: Determine host
+        id: host
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh determine_host
+
+      - name: Restore depends cache
+        id: depends-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            depends/built
+            depends/${{ steps.host.outputs.host }}
+          key: depends-tsan-${{ hashFiles('depends/packages/*.mk') }}-${{ env.LLVM_VERSION }}
+
+      - name: Build depends (stage 1, without IPC)
+        if: steps.depends-cache.outputs.cache-hit != 'true'
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_depends_without_ipc
+
+      - name: Save depends cache
+        uses: actions/cache/save@v4
+        if: steps.depends-cache.outputs.cache-hit != 'true'
+        with:
+          path: |
+            depends/built
+            depends/${{ steps.host.outputs.host }}
+          key: depends-tsan-${{ hashFiles('depends/packages/*.mk') }}-${{ env.LLVM_VERSION }}
+
+      - name: Replace libmultiprocess subtree
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh replace_subtree
+
+      - name: Build depends (stage 2, IPC packages including libmultiprocess)
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_depends_with_ipc
+
+      - name: Restore ccache
+        id: ccache-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-TSan-${{ github.ref }}-${{ github.sha }}
+          restore-keys: |
+            ccache-TSan-${{ github.ref }}-
+            ccache-TSan-
+
+      - name: Reset ccache stats
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh reset_ccache_stats
+
+      - name: CMake configure
+        env:
+          BITCOIN_CORE_CMAKE_ARGS: |-
+            -DSANITIZERS=thread
+            -DAPPEND_CPPFLAGS=-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES
+            -DCMAKE_TOOLCHAIN_FILE=depends/${{ steps.host.outputs.host }}/toolchain.cmake
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh configure_bitcoin_core
+
+      - name: Build
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_bitcoin_core
+
+      - name: Show ccache stats
+        run: _libmultiprocess/ci/scripts/ci_helpers.sh show_ccache_stats
+
+      - name: Run IPC unit tests
+        env:
+          LD_LIBRARY_PATH: depends/${{ steps.host.outputs.host }}/lib
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh run_ipc_unit_tests "${{ matrix.unit_test_runs }}"
+
+      - name: Run IPC functional tests
+        env:
+          LD_LIBRARY_PATH: depends/${{ steps.host.outputs.host }}/lib
+          CI_FAILFAST_TEST_LEAVE_DANGLING: 1
+        run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh run_ipc_functional_tests "${{ matrix.functional_test_runs }}" "${{ matrix.functional_timeout_factor }}"
+
+      - name: Save ccache
+        uses: actions/cache/save@v4
+        if: github.ref == 'refs/heads/master' || steps.ccache-restore.outputs.cache-hit != 'true'
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-TSan-${{ github.ref }}-${{ github.sha }}

src/ipc/libmultiprocess/.github/workflows/ci.yml

@@ -120,16 +120,21 @@ jobs:
         env:
           HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
         run: |
-          brew install --quiet ninja capnp
+          brew install --quiet ninja capnp llvm
 
       - name: Run CI script
         run: |
+          export PATH="$(brew --prefix llvm)/bin:$PATH"
           CI_CONFIG="ci/configs/macos.bash" bash ci/scripts/ci.sh
 
   build:
     runs-on: ubuntu-latest
 
     env:
+      NIXPKGS_CHANNEL: nixos-25.05
+      NIX_EXTRA_CONFIG: |
+        keep-env-derivations = true
+        keep-outputs = true
       NIX_EXTRA_CONFIG_ACT: |
         sandbox = false
         filter-syscalls = false
@@ -144,14 +149,33 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
+      - name: Determine CI configuration
+        id: config
+        env:
+          CI_CONFIG: ci/configs/${{ matrix.config }}.bash
+        run: ci/scripts/config.sh
+
       - name: Install Nix
         uses: cachix/install-nix-action@v31       # 2025-05-27, from https://github.com/cachix/install-nix-action/tags
         with:
-          nix_path: nixpkgs=channel:nixos-25.05   # latest release
+          nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/${{ steps.config.outputs.nixpkgs_rev }}.tar.gz
           # Act executes inside an unprivileged container (Docker or Podman),
           # so KVM support isn't available.
           enable_kvm: "${{ github.actor != 'nektos/act' }}"
-          extra_nix_config: ${{ github.actor == 'nektos/act' && env.NIX_EXTRA_CONFIG_ACT || '' }}
+          extra_nix_config: |
+            ${{ env.NIX_EXTRA_CONFIG }}
+            ${{ github.actor == 'nektos/act' && env.NIX_EXTRA_CONFIG_ACT || '' }}
+
+      - name: Cache Nix store
+        if: steps.config.outputs.cache_nix_store == 'true'
+        uses: nix-community/cache-nix-action@v7
+        with:
+          primary-key: nix-${{ runner.os }}-${{ matrix.config }}-${{ steps.config.outputs.nixpkgs_rev }}-${{ hashFiles('shell.nix', 'ci/patches/*.patch', format('ci/configs/{0}.bash', matrix.config)) }}
+          restore-prefixes-first-match: |
+            nix-${{ runner.os }}-${{ matrix.config }}-${{ steps.config.outputs.nixpkgs_rev }}-
+            nix-${{ runner.os }}-${{ matrix.config }}-
+            nix-${{ runner.os }}-
+          gc-max-store-size-linux: 10G
 
       - name: Run CI script
         env:

src/ipc/libmultiprocess/ci/configs/gnu32.bash

@@ -1,5 +1,7 @@
 CI_DESC="CI job cross-compiling to 32-bit"
 CI_DIR=build-gnu32
+# Cache the heaviest Nix job to stay within GitHub's cache budget.
+CI_CACHE_NIX_STORE=true
 NIX_ARGS=(
   --arg minimal true
   --arg crossPkgs 'import <nixpkgs> { crossSystem = { config = "i686-unknown-linux-gnu"; }; }'

src/ipc/libmultiprocess/ci/configs/macos.bash

@@ -1,5 +1,5 @@
 CI_DESC="CI config for macOS"
 CI_DIR=build-macos
 export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter"
-CMAKE_ARGS=(-G Ninja)
+CMAKE_ARGS=(-G Ninja -DMP_ENABLE_CLANG_TIDY=ON)
 BUILD_ARGS=(-k 0)