From 7d61b0d8830f4f2e4afb42191a6d052b2b3557b8 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 3 Jun 2025 11:06:24 -0400 Subject: [PATCH 1/2] Fix registering of background updates for split main/state db Fixes #16054 --- changelog.d/18509.bugfix | 1 + .../databases/main/events_bg_updates.py | 29 +++++++++++++++-- synapse/storage/databases/state/bg_updates.py | 31 ------------------- synapse/types/storage/__init__.py | 10 ++++++ 4 files changed, 38 insertions(+), 33 deletions(-) create mode 100644 changelog.d/18509.bugfix diff --git a/changelog.d/18509.bugfix b/changelog.d/18509.bugfix new file mode 100644 index 00000000000..2c821d03957 --- /dev/null +++ b/changelog.d/18509.bugfix @@ -0,0 +1 @@ +Fix `KeyError` on background updates when using split main/state databases. diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 5c83a9f779e..339670b21ab 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -294,6 +294,27 @@ def __init__( where_clause="NOT outlier", ) + # These indices are needed to validate the foreign key constraint + # when events are deleted. + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="current_state_events_stream_ordering_idx", + table="current_state_events", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="room_memberships_stream_ordering_idx", + table="room_memberships", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="local_current_membership_stream_ordering_idx", + table="local_current_membership", + columns=["event_stream_ordering"], + ) + # Handle background updates for Sliding Sync tables # self.db_pool.updates.register_background_update_handler( @@ -2121,7 +2142,9 @@ def _find_previous_invite_or_knock_membership_txn( ) # Map of values to insert/update in the `sliding_sync_membership_snapshots` table - sliding_sync_membership_snapshots_insert_map: SlidingSyncMembershipSnapshotSharedInsertValues = {} + sliding_sync_membership_snapshots_insert_map: ( + SlidingSyncMembershipSnapshotSharedInsertValues + ) = {} if membership == Membership.JOIN: # If we're still joined, we can pull from current state. current_state_ids_map: StateMap[ @@ -2758,7 +2781,9 @@ def _resolve_stale_data_in_sliding_sync_joined_rooms_table( "progress_json": "{}", }, ) - depends_on = _BackgroundUpdates.SLIDING_SYNC_PREFILL_JOINED_ROOMS_TO_RECALCULATE_TABLE_BG_UPDATE + depends_on = ( + _BackgroundUpdates.SLIDING_SYNC_PREFILL_JOINED_ROOMS_TO_RECALCULATE_TABLE_BG_UPDATE + ) # Now kick-off the background update to catch-up with what we missed while Synapse # was downgraded. diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index 5b594fe8dd9..ac38b2ab19c 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -290,16 +290,6 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): STATE_GROUPS_ROOM_INDEX_UPDATE_NAME = "state_groups_room_id_idx" STATE_GROUP_EDGES_UNIQUE_INDEX_UPDATE_NAME = "state_group_edges_unique_idx" - CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "current_state_events_stream_ordering_idx" - ) - ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "room_memberships_stream_ordering_idx" - ) - LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "local_current_membership_stream_ordering_idx" - ) - def __init__( self, database: DatabasePool, @@ -336,27 +326,6 @@ def __init__( replaces_index="state_group_edges_idx", ) - # These indices are needed to validate the foreign key constraint - # when events are deleted. - self.db_pool.updates.register_background_index_update( - self.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="current_state_events_stream_ordering_idx", - table="current_state_events", - columns=["event_stream_ordering"], - ) - self.db_pool.updates.register_background_index_update( - self.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="room_memberships_stream_ordering_idx", - table="room_memberships", - columns=["event_stream_ordering"], - ) - self.db_pool.updates.register_background_index_update( - self.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="local_current_membership_stream_ordering_idx", - table="local_current_membership", - columns=["event_stream_ordering"], - ) - async def _background_deduplicate_state( self, progress: dict, batch_size: int ) -> int: diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index 378a15e038c..b01653246a9 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -38,6 +38,16 @@ class _BackgroundUpdates: EVENTS_JUMP_TO_DATE_INDEX = "events_jump_to_date_index" + CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "current_state_events_stream_ordering_idx" + ) + ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "room_memberships_stream_ordering_idx" + ) + LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "local_current_membership_stream_ordering_idx" + ) + SLIDING_SYNC_PREFILL_JOINED_ROOMS_TO_RECALCULATE_TABLE_BG_UPDATE = ( "sliding_sync_prefill_joined_rooms_to_recalculate_table_bg_update" ) From 38b882019d090dab85e081b55f7e0bd066884f34 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 3 Jun 2025 11:26:38 -0400 Subject: [PATCH 2/2] Lint --- rust/src/identifier.rs | 2 +- synapse/storage/databases/main/events_bg_updates.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/rust/src/identifier.rs b/rust/src/identifier.rs index b70f6a30c73..03d1ebdc8a3 100644 --- a/rust/src/identifier.rs +++ b/rust/src/identifier.rs @@ -27,7 +27,7 @@ pub enum IdentifierError { impl fmt::Display for IdentifierError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 339670b21ab..8a59091da0f 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -2142,9 +2142,7 @@ def _find_previous_invite_or_knock_membership_txn( ) # Map of values to insert/update in the `sliding_sync_membership_snapshots` table - sliding_sync_membership_snapshots_insert_map: ( - SlidingSyncMembershipSnapshotSharedInsertValues - ) = {} + sliding_sync_membership_snapshots_insert_map: SlidingSyncMembershipSnapshotSharedInsertValues = {} if membership == Membership.JOIN: # If we're still joined, we can pull from current state. current_state_ids_map: StateMap[ @@ -2781,9 +2779,7 @@ def _resolve_stale_data_in_sliding_sync_joined_rooms_table( "progress_json": "{}", }, ) - depends_on = ( - _BackgroundUpdates.SLIDING_SYNC_PREFILL_JOINED_ROOMS_TO_RECALCULATE_TABLE_BG_UPDATE - ) + depends_on = _BackgroundUpdates.SLIDING_SYNC_PREFILL_JOINED_ROOMS_TO_RECALCULATE_TABLE_BG_UPDATE # Now kick-off the background update to catch-up with what we missed while Synapse # was downgraded.