unify sql and build indexes in background

Author: H-Shay

synapse/storage/databases/main/filtering.py

@@ -16,15 +16,37 @@
 from typing import Optional, Tuple, Union, cast
 
 from canonicaljson import encode_canonical_json
+from typing_extensions import TYPE_CHECKING
 
 from synapse.api.errors import Codes, StoreError, SynapseError
 from synapse.storage._base import SQLBaseStore, db_to_json
-from synapse.storage.database import LoggingTransaction
+from synapse.storage.database import (
+    DatabasePool,
+    LoggingDatabaseConnection,
+    LoggingTransaction,
+)
 from synapse.types import JsonDict, UserID
 from synapse.util.caches.descriptors import cached
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 
 class FilteringWorkerStore(SQLBaseStore):
+    def __init__(
+        self,
+        database: DatabasePool,
+        db_conn: LoggingDatabaseConnection,
+        hs: "HomeServer",
+    ):
+        super().__init__(database, db_conn, hs)
+        self.db_pool.updates.register_background_index_update(
+            "full_users_filters_unique_idx",
+            index_name="full_users_unique_idx",
+            table="user_filters",
+            columns=["full_user_id, filter_id"],
+        )
+
     @cached(num_args=2)
     async def get_user_filter(
         self, user_localpart: str, filter_id: Union[int, str]

synapse/storage/databases/main/profile.py

@@ -11,15 +11,33 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from synapse.api.errors import StoreError
 from synapse.storage._base import SQLBaseStore
+from synapse.storage.database import DatabasePool, LoggingDatabaseConnection
 from synapse.storage.databases.main.roommember import ProfileInfo
 from synapse.types import UserID
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 
 class ProfileWorkerStore(SQLBaseStore):
+    def __init__(
+        self,
+        database: DatabasePool,
+        db_conn: LoggingDatabaseConnection,
+        hs: "HomeServer",
+    ):
+        super().__init__(database, db_conn, hs)
+        self.db_pool.updates.register_background_index_update(
+            "profiles_full_user_id_key_idx",
+            index_name="profiles_full_user_id_key",
+            table="profiles",
+            columns=["full_user_id"],
+        )
+
     async def get_profileinfo(self, user_localpart: str) -> ProfileInfo:
         try:
             profile = await self.db_pool.simple_select_one(

synapse/storage/schema/main/delta/75/01_add_profiles_full_user_id_column.sql.postgres synapse/storage/schema/main/delta/75/01_add_profiles_full_user_id_column.sql

@@ -15,7 +15,6 @@
 
 ALTER TABLE profiles ADD COLUMN full_user_id TEXT;
 
--- Add a new constraint on the new column, mirroring the `profiles_user_id_key`
+-- Make sure the column has a unique constraint, mirroring the `profiles_user_id_key`
 -- constraint.
-ALTER TABLE ONLY profiles
-    ADD CONSTRAINT profiles_full_user_id_key UNIQUE (full_user_id);
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES (7501, 'profiles_full_user_id_key_idx', '{}');

synapse/storage/schema/main/delta/75/01_add_profiles_full_user_id_column.sql.sqlite

@@ -17,7 +17,4 @@ ALTER TABLE user_filters ADD COLUMN full_user_id TEXT;
 
 -- Add a unique index on the new column, mirroring the `user_filters_unique` unique
 -- index.
-CREATE UNIQUE INDEX full_user_filters_unique ON user_filters (full_user_id, filter_id);
--- NB: This will lock the table for writes while the index is being built.
---     There are around 4,000,000 user_filters on matrix.org so we expect this to take
---     a couple of seconds at most.
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES (7502, 'full_users_filters_unique_idx', '{}');