Add in-repo Complement tests

.github/workflows/latest_deps.yml

@@ -126,7 +126,6 @@ jobs:
           -exec echo "::endgroup::" \;
           || true
 
-
   sytest:
     needs: check_repo
     if: needs.check_repo.outputs.should_run_workflow == 'true'
@@ -181,7 +180,6 @@ jobs:
             /logs/results.tap
             /logs/**/*.log*
 
-
   complement:
     needs: check_repo
     if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
@@ -214,11 +212,53 @@ jobs:
           cache-dependency-path: complement/go.sum
           go-version-file: complement/go.mod
 
-      - run: |
+      - name: Run Complement Tests
+        id: run_complement_tests
+        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
+        #     are underpowered and don't like running tons of Synapse instances at once.
+        # -json: Output JSON format so that gotestfmt can parse it.
+        #
+        # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it
+        # later on for better formatting with gotestfmt. But we still want the command
+        # to output to the terminal as it runs so we can see what's happening in
+        # real-time.
+        run: |
           set -o pipefail
-          TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log
         shell: bash
-        name: Run Complement Tests
+        env:
+          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
+          TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1
+
+      - name: Formatted Complement test logs
+        # Always run this step if we attempted to run the Complement tests.
+        if: always() && steps.run_complement_tests.outcome != 'skipped'
+        run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
+
+      - name: Run in-repo Complement Tests
+        id: run_in_repo_complement_tests
+        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
+        #     are underpowered and don't like running tons of Synapse instances at once.
+        # -json: Output JSON format so that gotestfmt can parse it.
+        #
+        # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it
+        # later on for better formatting with gotestfmt. But we still want the command
+        # to output to the terminal as it runs so we can see what's happening in
+        # real-time.
+        run: |
+          set -o pipefail
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log
+        shell: bash
+        env:
+          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
+          TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1
+
+      - name: Formatted Complement test logs
+        # Always run this step if we attempted to run the Complement tests.
+        if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped'
+        run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
 
   # Open an issue if the build fails, so we know about it.
   # Only do this if we're not experimenting with this action in a PR.

.github/workflows/tests.yml

@@ -707,20 +707,19 @@ jobs:
           cache-dependency-path: complement/go.sum
           go-version-file: complement/go.mod
 
-        # use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once.
       - name: Run Complement Tests
         id: run_complement_tests
         # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
         #     are underpowered and don't like running tons of Synapse instances at once.
         # -json: Output JSON format so that gotestfmt can parse it.
         #
-        # tee /tmp/gotest.log: We tee the output to a file so that we can re-process it
+        # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it
         # later on for better formatting with gotestfmt. But we still want the command
         # to output to the terminal as it runs so we can see what's happening in
         # real-time.
         run: |
           set -o pipefail
-          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest.log
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log
         shell: bash
         env:
           POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
@@ -729,7 +728,30 @@ jobs:
       - name: Formatted Complement test logs
         # Always run this step if we attempted to run the Complement tests.
         if: always() && steps.run_complement_tests.outcome != 'skipped'
-        run: cat /tmp/gotest.log | gotestfmt -hide "successful-downloads,empty-packages"
+        run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
+
+      - name: Run in-repo Complement Tests
+        id: run_in_repo_complement_tests
+        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
+        #     are underpowered and don't like running tons of Synapse instances at once.
+        # -json: Output JSON format so that gotestfmt can parse it.
+        #
+        # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it
+        # later on for better formatting with gotestfmt. But we still want the command
+        # to output to the terminal as it runs so we can see what's happening in
+        # real-time.
+        run: |
+          set -o pipefail
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log
+        shell: bash
+        env:
+          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
+
+      - name: Formatted Complement test logs
+        # Always run this step if we attempted to run the Complement tests.
+        if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped'
+        run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
 
   cargo-test:
     if: ${{ needs.changes.outputs.rust == 'true' }}

.github/workflows/twisted_trunk.yml

@@ -12,10 +12,9 @@ on:
       twisted_ref:
         description: Commit, branch or tag to checkout from upstream Twisted.
         required: false
-        default: 'trunk'
+        default: "trunk"
         type: string
 
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -89,7 +88,6 @@ jobs:
           poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
           poetry install --no-interaction --extras "all test"
       - run: poetry run trial --jobs 2 tests
-
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -199,11 +197,53 @@ jobs:
           poetry lock
         working-directory: synapse
 
-      - run: |
+      - name: Run Complement Tests
+        id: run_complement_tests
+        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
+        #     are underpowered and don't like running tons of Synapse instances at once.
+        # -json: Output JSON format so that gotestfmt can parse it.
+        #
+        # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it
+        # later on for better formatting with gotestfmt. But we still want the command
+        # to output to the terminal as it runs so we can see what's happening in
+        # real-time.
+        run: |
           set -o pipefail
-          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log
         shell: bash
-        name: Run Complement Tests
+        env:
+          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
+          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: 1
+
+      - name: Formatted Complement test logs
+        # Always run this step if we attempted to run the Complement tests.
+        if: always() && steps.run_complement_tests.outcome != 'skipped'
+        run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
+
+      - name: Run in-repo Complement Tests
+        id: run_in_repo_complement_tests
+        # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes
+        #     are underpowered and don't like running tons of Synapse instances at once.
+        # -json: Output JSON format so that gotestfmt can parse it.
+        #
+        # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it
+        # later on for better formatting with gotestfmt. But we still want the command
+        # to output to the terminal as it runs so we can see what's happening in
+        # real-time.
+        run: |
+          set -o pipefail
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log
+        shell: bash
+        env:
+          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
+          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: 1
+
+      - name: Formatted Complement test logs
+        # Always run this step if we attempted to run the Complement tests.
+        if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped'
+        run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
 
   # open an issue if the build fails, so we know about it.
   open-issue:

changelog.d/19406.misc

@@ -0,0 +1 @@
+Add in-repo Complement tests so we can test Synapse specific behavior at an end-to-end level.

complement/.golangci.yml

@@ -0,0 +1,38 @@
+# Docs: https://golangci-lint.run/docs/configuration/file/
+#
+# Go formatting/linting rules
+#
+# This is run as part of the normal linting utility script,
+# `poetry run ./scripts-dev/lint.sh`
+
+version: "2"
+
+linters:
+  # Default set of linters.
+  # The value can be:
+  # - `standard`: https://golangci-lint.run/docs/linters/#enabled-by-default
+  # - `all`: enables all linters by default.
+  # - `none`: disables all linters by default.
+  # - `fast`: enables only linters considered as "fast" (`golangci-lint help linters --json | jq '[ .[] | select(.fast==true) ] | map(.name)'`).
+  # Default: standard
+  default: standard
+
+  # Enable specific linter.
+  # enable:
+  #   - example
+
+  # Disable specific linters.
+  disable:
+    # FIXME: Ideally, we'd enable the `bodyclose` lint but there are many
+    # false-positives (like https://github.com/timakin/bodyclose/issues/39) and just is
+    # not well-suited for our use case (https://github.com/timakin/bodyclose/issues/11 and
+    # https://github.com/timakin/bodyclose/issues/76).
+    - bodyclose
+
+formatters:
+  # Enable specific formatter.
+  # Default: [] (uses standard Go formatting)
+  enable:
+    - gofmt
+    - goimports
+    - golines

complement/README.md

@@ -0,0 +1,52 @@
+# In-repo Complement test suite
+
+Complement is a black box integration testing framework for Matrix homeservers. It
+allows us to write end-to-end tests that interact with real Synapse homeservers to
+ensure everything works at a holistic level.
+
+We use the in-repo test suite to test Synapse specific behaviors like the admin API.
+
+
+## Setup
+
+Nothing beyond a [normal Complement
+setup](https://github.com/matrix-org/complement?tab=readme-ov-file#running) (just Go and
+Docker).
+
+Then you can run the `multi_synapse` Complement tests:
+
+```shell
+./scripts-dev/complement.sh ./tests/multi_synapse/...
+```
+
+## Running tests
+
+Run tests from the [Complement](https://github.com/matrix-org/complement) repo:
+
+```shell
+# Run the tests
+./scripts-dev/complement.sh
+
+# To run a whole group of tests, you can specify part of the test path:
+scripts-dev/complement.sh ./tests/csapi/... -run TestRoomCreate
+# To run a specific test, you can specify the whole name structure:
+scripts-dev/complement.sh ./tests/csapi/... -run TestRoomCreate/Parallel/POST_/createRoom_makes_a_public_room
+# Generally though, the `-run` parameter accepts regex patterns, so you can match however you like:
+scripts-dev/complement.sh ./tests/... -run 'TestRoomCreate/Parallel/POST_/createRoom_makes_a_(.*)'
+
+```
+
+### Running in-repo tests
+
+In-repo Complement tests are tests that are vendored into this project.
+
+To run the in-repo Complement tests, use the `--in-repo` command line argument.
+
+```shell
+# Run only a specific test package.
+# Note: test packages are relative to the `./complement` directory in this project
+./scripts-dev/complement.sh --in-repo ./tests/...
+
+# Similarly, you can also use `-run` to specify all or part of a specific test path to run
+scripts-dev/complement.sh --in-repo ./tests/... -run TestIntraShardFederation
+```