From 1e5328cfd8b0800d34803ff3f29dfdcc089b0c33 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 3 Jul 2019 15:51:40 +0100 Subject: [PATCH 1/7] Add a sytest blacklist file --- sytest-blacklist | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 sytest-blacklist diff --git a/sytest-blacklist b/sytest-blacklist new file mode 100644 index 000000000000..75d5780eb2b2 --- /dev/null +++ b/sytest-blacklist @@ -0,0 +1,17 @@ +# Blacklisted due to https://github.com/matrix-org/synapse/issues/1679 +Remote room members also see posted message events +# Blacklisted due to https://github.com/matrix-org/synapse/issues/2065 +Guest users can accept invites to private rooms over federation +# Blacklisted due to https://github.com/vector-im/riot-web/issues/7211 +The only membership state included in a gapped incremental sync is for senders in the timeline +# Blacklisted due to https://github.com/matrix-org/synapse/issues/1658 +Newly created users see their own presence in /initialSync (SYT-34) +# Blacklisted due to https://github.com/matrix-org/synapse/issues/1396 +Should reject keys claiming to belong to a different user +# Blacklisted due to https://github.com/matrix-org/synapse/issues/2306 +Users appear/disappear from directory when join_rules are changed +Users appear/disappear from directory when history_visibility are changed +# Blacklisted due to https://github.com/matrix-org/synapse/issues/1531 +Enabling an unknown default rule fails with 404 +# Blacklisted due to https://github.com/matrix-org/synapse/issues/1663 +New federated private chats get full presence information (SYN-115) From 3a828d5d5cd4c6e560ee4df9020cf6a893a29c17 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 3 Jul 2019 15:56:08 +0100 Subject: [PATCH 2/7] Add changelog --- changelog.d/5611.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5611.feature diff --git a/changelog.d/5611.feature b/changelog.d/5611.feature new file mode 100644 index 000000000000..a2d169513912 --- /dev/null +++ b/changelog.d/5611.feature @@ -0,0 +1 @@ +Add a blacklist file to the repo to blacklist certain sytests from failing CI. From f51154344152ed0b536218843195e11234a1cd43 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 3 Jul 2019 16:01:24 +0100 Subject: [PATCH 3/7] Add blacklist to manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 2c59c7bdc29a..834ddfad39c6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ include demo/README include demo/demo.tls.dh include demo/*.py include demo/*.sh +include sytest-blacklist recursive-include synapse/storage/schema *.sql recursive-include synapse/storage/schema *.sql.postgres From 9ad8f8922a8f1ecfa883b83702aba41fc19aed2e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 4 Jul 2019 10:32:14 +0100 Subject: [PATCH 4/7] change newsfile type, explain blacklist contents --- changelog.d/{5611.feature => 5611.misc} | 0 sytest-blacklist | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+) rename changelog.d/{5611.feature => 5611.misc} (100%) diff --git a/changelog.d/5611.feature b/changelog.d/5611.misc similarity index 100% rename from changelog.d/5611.feature rename to changelog.d/5611.misc diff --git a/sytest-blacklist b/sytest-blacklist index 75d5780eb2b2..b760a48c57ca 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -1,17 +1,35 @@ +# This file serves as a blacklist for SyTest tests that we expect will fail in +# Synapse. +# +# Each line of this file is scanned by sytest during a run and if the line +# exactly matches the name of a test, it will be marked as "expected fail", +# meaning the test will still run, but failure will not mark the entire test +# suite as failing. +# +# Test names are encouraged to have a bug accompanied with them, serving as an +# explanation for why the test has been excluded. + # Blacklisted due to https://github.com/matrix-org/synapse/issues/1679 Remote room members also see posted message events + # Blacklisted due to https://github.com/matrix-org/synapse/issues/2065 Guest users can accept invites to private rooms over federation + # Blacklisted due to https://github.com/vector-im/riot-web/issues/7211 The only membership state included in a gapped incremental sync is for senders in the timeline + # Blacklisted due to https://github.com/matrix-org/synapse/issues/1658 Newly created users see their own presence in /initialSync (SYT-34) + # Blacklisted due to https://github.com/matrix-org/synapse/issues/1396 Should reject keys claiming to belong to a different user + # Blacklisted due to https://github.com/matrix-org/synapse/issues/2306 Users appear/disappear from directory when join_rules are changed Users appear/disappear from directory when history_visibility are changed + # Blacklisted due to https://github.com/matrix-org/synapse/issues/1531 Enabling an unknown default rule fails with 404 + # Blacklisted due to https://github.com/matrix-org/synapse/issues/1663 New federated private chats get full presence information (SYN-115) From ce00f717b84e83102062631768c044778bd53273 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 5 Jul 2019 00:46:49 +1000 Subject: [PATCH 5/7] always pull the latest sytest --- .buildkite/pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 513eb3bde9cc..060cedf33634 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -178,6 +178,7 @@ steps: - docker#v3.0.1: image: "matrixdotorg/sytest-synapse:py35" propagate-environment: true + always-pull: true retry: automatic: - exit_status: -1 @@ -197,6 +198,7 @@ steps: - docker#v3.0.1: image: "matrixdotorg/sytest-synapse:py35" propagate-environment: true + always-pull: true retry: automatic: - exit_status: -1 @@ -217,6 +219,7 @@ steps: - docker#v3.0.1: image: "matrixdotorg/sytest-synapse:py35" propagate-environment: true + always-pull: true soft_fail: true retry: automatic: From 6b66eafe6b33f050af5da8bcab33fe8e809911a4 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 5 Jul 2019 00:56:31 +1000 Subject: [PATCH 6/7] pull every time, and remove buildkite sytest script now it's in sytest --- .buildkite/pipeline.yml | 4 +- .buildkite/synapse_sytest.sh | 145 ----------------------------------- 2 files changed, 2 insertions(+), 147 deletions(-) delete mode 100644 .buildkite/synapse_sytest.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 060cedf33634..9fd5285c586d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -193,7 +193,7 @@ steps: POSTGRES: "1" command: - "bash .buildkite/merge_base_branch.sh" - - "bash .buildkite/synapse_sytest.sh" + - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: image: "matrixdotorg/sytest-synapse:py35" @@ -214,7 +214,7 @@ steps: WORKERS: "1" command: - "bash .buildkite/merge_base_branch.sh" - - "bash .buildkite/synapse_sytest.sh" + - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: image: "matrixdotorg/sytest-synapse:py35" diff --git a/.buildkite/synapse_sytest.sh b/.buildkite/synapse_sytest.sh deleted file mode 100644 index 3011b88bb799..000000000000 --- a/.buildkite/synapse_sytest.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash -# -# Fetch sytest, and then run the tests for synapse. The entrypoint for the -# sytest-synapse docker images. - -set -ex - -if [ -n "$BUILDKITE" ] -then - SYNAPSE_DIR=`pwd` -else - SYNAPSE_DIR="/src" -fi - -# Attempt to find a sytest to use. -# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image. -if [ -d "/sytest" ]; then - # If the user has mounted in a SyTest checkout, use that. - echo "Using local sytests..." - - # create ourselves a working directory and dos2unix some scripts therein - mkdir -p /work/jenkins - for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do - dos2unix -n "/sytest/$i" "/work/$i" - done - ln -sf /sytest/tests /work - ln -sf /sytest/keys /work - SYTEST_LIB="/sytest/lib" -else - if [ -n "BUILDKITE_BRANCH" ] - then - branch_name=$BUILDKITE_BRANCH - else - # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch. - branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop" - fi - - # Try and fetch the branch - echo "Trying to get same-named sytest branch..." - wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || { - # Probably a 404, fall back to develop - echo "Using develop instead..." - wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz - } - - mkdir -p /work - tar -C /work --strip-components=1 -xf sytest.tar.gz - SYTEST_LIB="/work/lib" -fi - -cd /work - -# PostgreSQL setup -if [ -n "$POSTGRES" ] -then - export PGUSER=postgres - export POSTGRES_DB_1=pg1 - export POSTGRES_DB_2=pg2 - - # Start the database - su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres - - # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse - jenkins/prep_sytest_for_postgres.sh - - # Make the test databases for the two Synapse servers that will be spun up - su -c 'psql -c "CREATE DATABASE pg1;"' postgres - su -c 'psql -c "CREATE DATABASE pg2;"' postgres - -fi - -if [ -n "$OFFLINE" ]; then - # if we're in offline mode, just put synapse into the virtualenv, and - # hope that the deps are up-to-date. - # - # (`pip install -e` likes to reinstall setuptools even if it's already installed, - # so we just run setup.py explicitly.) - # - (cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop) -else - # We've already created the virtualenv, but lets double check we have all - # deps. - /venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR - /venv/bin/pip install -q --upgrade --no-cache-dir \ - lxml psycopg2 coverage codecov tap.py - - # Make sure all Perl deps are installed -- this is done in the docker build - # so will only install packages added since the last Docker build - ./install-deps.pl -fi - - -# Run the tests ->&2 echo "+++ Running tests" - -RUN_TESTS=( - perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all -) - -TEST_STATUS=0 - -if [ -n "$WORKERS" ]; then - RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py) -else - RUN_TESTS+=(-I Synapse) -fi - -"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$? - -if [ $TEST_STATUS -ne 0 ]; then - >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS" -else - >&2 echo -e "run-tests \e[32mPASSED\e[0m" -fi - ->&2 echo "--- Copying assets" - -# Copy out the logs -mkdir -p /logs -cp results.tap /logs/results.tap -rsync --ignore-missing-args --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*" - -# Upload coverage to codecov and upload files, if running on Buildkite -if [ -n "$BUILDKITE" ] -then - /venv/bin/coverage combine || true - /venv/bin/coverage xml || true - /venv/bin/codecov -X gcov -f coverage.xml - - wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz - tar xvf buildkite.tar.gz - chmod +x ./buildkite-agent - - # Upload the files - ./buildkite-agent artifact upload "/logs/**/*.log*" - ./buildkite-agent artifact upload "/logs/results.tap" - - if [ $TEST_STATUS -ne 0 ]; then - # Annotate, if failure - /venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL" - fi -fi - - -exit $TEST_STATUS From 8afbc1279b0f4dd8351617932610f68d5c9fc9a8 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 5 Jul 2019 01:02:01 +1000 Subject: [PATCH 7/7] fix incorrect path --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 9fd5285c586d..dd0f98cba0ec 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -173,7 +173,7 @@ steps: queue: "medium" command: - "bash .buildkite/merge_base_branch.sh" - - "bash .buildkite/synapse_sytest.sh" + - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: image: "matrixdotorg/sytest-synapse:py35"