Delete unreferenced state groups in background

changelog.d/18254.feature

@@ -0,0 +1 @@
+Add background job to clear unreferenced state groups.

docs/development/database_schema.md

@@ -162,7 +162,7 @@ by a unique name, the current status (stored in JSON), and some dependency infor
 * Whether the update requires a previous update to be complete.
 * A rough ordering for which to complete updates.
 
-A new background updates needs to be added to the `background_updates` table:
+A new background update needs to be added to the `background_updates` table:
 
 ```sql
 INSERT INTO background_updates (ordering, update_name, depends_on, progress_json) VALUES

synapse/_scripts/synapse_port_db.py

@@ -192,6 +192,11 @@
 
 
 IGNORED_TABLES = {
+    # Porting the auto generated sequence in this table is non-trivial.
+    # None of the entries in this list are mandatory for Synapse to keep working.
+    # If state group disk space is an issue after the port, the
+    # `mark_unreferenced_state_groups_for_deletion_bg_update` background task can be run again.
+    "state_groups_pending_deletion",
     # We don't port these tables, as they're a faff and we can regenerate
     # them anyway.
     "user_directory",
@@ -217,6 +222,15 @@
 }
 
 
+# These background updates will not be applied upon creation of the postgres database.
+IGNORED_BACKGROUND_UPDATES = {
+    # Reapplying this background update to the postgres database is unnecessary after
+    # already having waited for the SQLite database to complete all running background
+    # updates.
+    "mark_unreferenced_state_groups_for_deletion_bg_update",
+}
+
+
 # Error returned by the run function. Used at the top-level part of the script to
 # handle errors and return codes.
 end_error: Optional[str] = None
@@ -688,6 +702,20 @@ def _is_sqlite_autovacuum_enabled(txn: LoggingTransaction) -> bool:
         # 0 means off. 1 means full. 2 means incremental.
         return autovacuum_setting != 0
 
+    async def remove_ignored_background_updates_from_database(self) -> None:
+        def _remove_delete_unreferenced_state_groups_bg_updates(
+            txn: LoggingTransaction,
+        ) -> None:
+            txn.execute(
+                "DELETE FROM background_updates WHERE update_name = ANY(?)",
+                (list(IGNORED_BACKGROUND_UPDATES),),
+            )
+
+        await self.postgres_store.db_pool.runInteraction(
+            "remove_delete_unreferenced_state_groups_bg_updates",
+            _remove_delete_unreferenced_state_groups_bg_updates,
+        )
+
     async def run(self) -> None:
         """Ports the SQLite database to a PostgreSQL database.
 
@@ -733,6 +761,8 @@ async def run(self) -> None:
                 self.hs_config.database.get_single_database()
             )
 
+            await self.remove_ignored_background_updates_from_database()
+
             await self.run_background_updates_on_postgres()
 
             self.progress.set_state("Creating port tables")

synapse/storage/controllers/purge_events.py

@@ -21,11 +21,18 @@
 
 import itertools
 import logging
-from typing import TYPE_CHECKING, Collection, Mapping, Set
+from typing import (
+    TYPE_CHECKING,
+    Collection,
+    Mapping,
+    Set,
+)
 
 from synapse.logging.context import nested_logging_context
 from synapse.metrics.background_process_metrics import wrap_as_background_process
+from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause
 from synapse.storage.databases import Databases
+from synapse.types.storage import _BackgroundUpdates
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -44,6 +51,11 @@ def __init__(self, hs: "HomeServer", stores: Databases):
                 self._delete_state_groups_loop, 60 * 1000
             )
 
+        self.stores.state.db_pool.updates.register_background_update_handler(
+            _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE,
+            self._background_delete_unrefereneced_state_groups,
+        )
+
     async def purge_room(self, room_id: str) -> None:
         """Deletes all record of a room"""
 
@@ -81,7 +93,8 @@ async def purge_history(
             )
 
     async def _find_unreferenced_groups(
-        self, state_groups: Collection[int]
+        self,
+        state_groups: Collection[int],
     ) -> Set[int]:
         """Used when purging history to figure out which state groups can be
         deleted.
@@ -203,3 +216,213 @@ async def _delete_state_groups(
             room_id,
             groups_to_sequences,
         )
+
+    async def _background_delete_unrefereneced_state_groups(
+        self, progress: dict, batch_size: int
+    ) -> int:
+        """This background update will slowly delete any unreferenced state groups"""
+
+        last_checked_state_group = progress.get("last_checked_state_group")
+
+        if last_checked_state_group is None:
+            # This is the first run.
+            last_checked_state_group = (
+                await self.stores.state.db_pool.simple_select_one_onecol(
+                    table="state_groups",
+                    keyvalues={},
+                    retcol="MAX(id)",
+                    allow_none=True,
+                    desc="get_max_state_group",
+                )
+            )
+            if last_checked_state_group is None:
+                # There are no state groups so the background process is finished.
+                await self.stores.state.db_pool.updates._end_background_update(
+                    _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE
+                )
+                return batch_size
+            last_checked_state_group += 1
+
+        (
+            last_checked_state_group,
+            final_batch,
+        ) = await self._delete_unreferenced_state_groups_batch(
+            last_checked_state_group,
+            batch_size,
+        )
+
+        if not final_batch:
+            # There are more state groups to check.
+            progress = {
+                "last_checked_state_group": last_checked_state_group,
+            }
+            await self.stores.state.db_pool.updates._background_update_progress(
+                _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE,
+                progress,
+            )
+        else:
+            # This background process is finished.
+            await self.stores.state.db_pool.updates._end_background_update(
+                _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE
+            )
+
+        return batch_size
+
+    async def _delete_unreferenced_state_groups_batch(
+        self,
+        last_checked_state_group: int,
+        batch_size: int,
+    ) -> tuple[int, bool]:
+        """Looks for unreferenced state groups starting from the last state group
+        checked and marks them for deletion.
+
+        Args:
+            last_checked_state_group: The last state group that was checked.
+            batch_size: How many state groups to process in this iteration.
+
+        Returns:
+            (last_checked_state_group, final_batch)
+        """
+
+        # Look for state groups that can be cleaned up.
+        def get_next_state_groups_txn(txn: LoggingTransaction) -> Set[int]:
+            state_group_sql = (
+                "SELECT id FROM state_groups WHERE id < ? ORDER BY id DESC LIMIT ?"
+            )
+            txn.execute(state_group_sql, (last_checked_state_group, batch_size))
+
+            next_set = {row[0] for row in txn}
+
+            return next_set
+
+        next_set = await self.stores.state.db_pool.runInteraction(
+            "get_next_state_groups", get_next_state_groups_txn
+        )
+
+        final_batch = False
+        if len(next_set) < batch_size:
+            final_batch = True
+        else:
+            last_checked_state_group = min(next_set)
+
+        if len(next_set) == 0:
+            return last_checked_state_group, final_batch
+
+        # Find all state groups that can be deleted if any of the original set are deleted.
+        to_delete = await self._find_unreferenced_groups_for_background_deletion(
+            next_set
+        )
+
+        if len(to_delete) == 0:
+            return last_checked_state_group, final_batch
+
+        await self.stores.state_deletion.mark_state_groups_as_pending_deletion(
+            to_delete
+        )
+
+        return last_checked_state_group, final_batch
+
+    async def _find_unreferenced_groups_for_background_deletion(
+        self,
+        state_groups: Set[int],
+    ) -> Set[int]:
+        """Used when deleting unreferenced state groups in the background to figure out
+        which state groups can be deleted.
+        To avoid increased DB usage due to de-deltaing state groups, this returns only
+        state groups which are free standing (ie. no shared edges with referenced groups) or
+        state groups which do not share edges which result in a future referenced group.
+
+        The following scenarios outline the possibilities based on state group data in
+        the DB.
+
+        ie. Free standing -> state groups 1-N would be returned:
+            SG_1
+            |
+            ...
+            |
+            SG_N
+
+        ie. Previous reference -> state groups 2-N would be returned:
+            SG_1 <- referenced by event
+            |
+            SG_2
+            |
+            ...
+            |
+            SG_N
+
+        ie. Future reference -> none of the following state groups would be returned:
+            SG_1
+            |
+            SG_2
+            |
+            ...
+            |
+            SG_N <- referenced by event
+
+        Args:
+            state_groups: Set of state groups referenced by events
+                that are going to be deleted.
+
+        Returns:
+            to_delete
+        """
+
+        to_delete = set()
+
+        # If any of the state groups have next edges and any of those next state groups
+        # are marked for deletion, the corresponding state groups in this batch can also
+        # be marked for deletion.
+        def get_next_state_groups_marked_for_deletion_txn(
+            txn: LoggingTransaction,
+        ) -> tuple[Set[int], Set[int]]:
+            clause, args = make_in_list_sql_clause(
+                self.stores.state.database_engine, "e.prev_state_group", state_groups
+            )
+            state_group_sql = f"""
+                SELECT e.prev_state_group, e.state_group, d.state_group
+                FROM state_group_edges AS e
+                LEFT JOIN state_groups_pending_deletion AS d ON (e.state_group = d.state_group)
+                WHERE {clause}
+            """
+            txn.execute(state_group_sql, args)
+
+            groups_with_next_edges = set()
+            next_marked_for_deletion = set()
+            for row in txn:
+                state_group = row[0]
+                next_edge = row[1]
+                pending_deletion = row[2]
+
+                if next_edge is not None:
+                    groups_with_next_edges.add(state_group)
+                if pending_deletion is not None:
+                    next_marked_for_deletion.add(state_group)
+
+            return groups_with_next_edges, next_marked_for_deletion
+
+        (
+            groups_with_next_edges,
+            next_marked_for_deletion,
+        ) = await self.stores.state.db_pool.runInteraction(
+            "get_next_state_groups_marked_for_deletion",
+            get_next_state_groups_marked_for_deletion_txn,
+        )
+        to_delete |= next_marked_for_deletion
+
+        # Any state groups that have next edges don't need further processing.
+        # They are either already in `to_delete` or their next edge leads to a
+        # referenced state group.
+        state_groups -= groups_with_next_edges
+
+        if len(state_groups) == 0:
+            return to_delete
+
+        # Determine if any of the remaining state groups are directly referenced.
+        referenced = await self.stores.main.get_referenced_state_groups(state_groups)
+        state_groups -= referenced
+
+        # Any unreferenced state groups can be marked for deletion.
+        to_delete |= state_groups
+
+        return to_delete

synapse/storage/databases/state/bg_updates.py

@@ -20,7 +20,15 @@
 #
 
 import logging
-from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from synapse.logging.opentracing import tag_args, trace
 from synapse.storage._base import SQLBaseStore

synapse/storage/databases/state/deletion.py

@@ -321,19 +321,43 @@ def _finish_persisting_txn(
     async def mark_state_groups_as_pending_deletion(
         self, state_groups: Collection[int]
     ) -> None:
-        """Mark the given state groups as pending deletion"""
+        """Mark the given state groups as pending deletion.
 
-        now = self._clock.time_msec()
+        If any of the state groups are already pending deletion, then those records are
+        left as is.
+        """
 
-        await self.db_pool.simple_upsert_many(
-            table="state_groups_pending_deletion",
-            key_names=("state_group",),
-            key_values=[(state_group,) for state_group in state_groups],
-            value_names=("insertion_ts",),
-            value_values=[(now,) for _ in state_groups],
-            desc="mark_state_groups_as_pending_deletion",
+        await self.db_pool.runInteraction(
+            "mark_state_groups_as_pending_deletion",
+            self._mark_state_groups_as_pending_deletion_txn,
+            state_groups,
         )
 
+    def _mark_state_groups_as_pending_deletion_txn(
+        self,
+        txn: LoggingTransaction,
+        state_groups: Collection[int],
+    ) -> None:
+        sql = """
+        INSERT INTO state_groups_pending_deletion (state_group, insertion_ts)
+        VALUES %s
+        ON CONFLICT (state_group)
+        DO NOTHING
+        """
+
+        now = self._clock.time_msec()
+        rows = [
+            (
+                state_group,
+                now,
+            )
+            for state_group in state_groups
+        ]
+        if isinstance(txn.database_engine, PostgresEngine):
+            txn.execute_values(sql % ("?",), rows, fetch=False)
+        else:
+            txn.execute_batch(sql % ("(?, ?)",), rows)
+
     async def mark_state_groups_as_used(self, state_groups: Collection[int]) -> None:
         """Mark the given state groups as now being referenced"""