Merge branch 'develop' of github.com:matrix-org/synapse into anoa/msc2403_complement_build_flag

Author: anoadragon453

changelog.d/12310.feature

@@ -0,0 +1 @@
+Add a configuration option to remove a specific set of rooms from sync responses.

changelog.d/12326.misc

@@ -0,0 +1 @@
+Fix typechecker problems exposed by signedjson 1.1.2.

docs/sample_config.yaml

@@ -539,6 +539,15 @@ templates:
   #
   #custom_template_directory: /path/to/custom/templates/
 
+# List of rooms to exclude from sync responses. This is useful for server
+# administrators wishing to group users into a room without these users being able
+# to see it from their client.
+#
+# By default, no room is excluded.
+#
+#exclude_rooms_from_sync:
+#    - !foo:example.com
+
 
 # Message retention policy at the server level.
 #

mypy.ini

@@ -273,6 +273,9 @@ ignore_missing_imports = True
 [mypy-ijson.*]
 ignore_missing_imports = True
 
+[mypy-importlib_metadata.*]
+ignore_missing_imports = True
+
 [mypy-jaeger_client.*]
 ignore_missing_imports = True
 

synapse/config/key.py

@@ -16,7 +16,7 @@
 import hashlib
 import logging
 import os
-from typing import Any, Dict, Iterator, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional
 
 import attr
 import jsonschema
@@ -38,6 +38,9 @@
 
 from ._base import Config, ConfigError
 
+if TYPE_CHECKING:
+    from signedjson.key import VerifyKeyWithExpiry
+
 INSECURE_NOTARY_ERROR = """\
 Your server is configured to accept key server responses without signature
 validation or TLS certificate validation. This is likely to be very insecure. If
@@ -300,16 +303,16 @@ def read_signing_keys(self, signing_key_path: str, name: str) -> List[SigningKey
 
     def read_old_signing_keys(
         self, old_signing_keys: Optional[JsonDict]
-    ) -> Dict[str, VerifyKey]:
+    ) -> Dict[str, "VerifyKeyWithExpiry"]:
         if old_signing_keys is None:
             return {}
         keys = {}
         for key_id, key_data in old_signing_keys.items():
             if is_signing_algorithm_supported(key_id):
                 key_base64 = key_data["key"]
                 key_bytes = decode_base64(key_base64)
-                verify_key = decode_verify_key_bytes(key_id, key_bytes)
-                verify_key.expired_ts = key_data["expired_ts"]
+                verify_key: "VerifyKeyWithExpiry" = decode_verify_key_bytes(key_id, key_bytes)  # type: ignore[assignment]
+                verify_key.expired = key_data["expired_ts"]
                 keys[key_id] = verify_key
             else:
                 raise ConfigError(
@@ -422,7 +425,7 @@ def _parse_key_servers(
         server_name = server["server_name"]
         result = TrustedKeyServer(server_name=server_name)
 
-        verify_keys = server.get("verify_keys")
+        verify_keys: Optional[Dict[str, str]] = server.get("verify_keys")
         if verify_keys is not None:
             result.verify_keys = {}
             for key_id, key_base64 in verify_keys.items():

synapse/config/server.py

@@ -680,6 +680,10 @@ def read_config(self, config, **kwargs):
             config.get("use_account_validity_in_account_status") or False
         )
 
+        self.rooms_to_exclude_from_sync: List[str] = (
+            config.get("exclude_rooms_from_sync") or []
+        )
+
     def has_tls_listener(self) -> bool:
         return any(listener.tls for listener in self.listeners)
 
@@ -1234,6 +1238,15 @@ def generate_config_section(
           # information about using custom templates.
           #
           #custom_template_directory: /path/to/custom/templates/
+
+        # List of rooms to exclude from sync responses. This is useful for server
+        # administrators wishing to group users into a room without these users being able
+        # to see it from their client.
+        #
+        # By default, no room is excluded.
+        #
+        #exclude_rooms_from_sync:
+        #    - !foo:example.com
         """
             % locals()
         )

synapse/crypto/keyring.py

@@ -176,7 +176,7 @@ def __init__(
         self._local_verify_keys: Dict[str, FetchKeyResult] = {}
         for key_id, key in hs.config.key.old_signing_keys.items():
             self._local_verify_keys[key_id] = FetchKeyResult(
-                verify_key=key, valid_until_ts=key.expired_ts
+                verify_key=key, valid_until_ts=key.expired
             )
 
         vk = get_verify_key(hs.signing_key)

synapse/events/builder.py

@@ -15,7 +15,7 @@
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 import attr
-from nacl.signing import SigningKey
+from signedjson.types import SigningKey
 
 from synapse.api.constants import MAX_DEPTH
 from synapse.api.room_versions import (

synapse/handlers/sync.py

@@ -298,6 +298,8 @@ def __init__(self, hs: "HomeServer"):
             expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE,
         )
 
+        self.rooms_to_exclude = hs.config.server.rooms_to_exclude_from_sync
+
     async def wait_for_sync_for_user(
         self,
         requester: Requester,
@@ -1607,13 +1609,15 @@ async def _generate_sync_entry_for_rooms(
         ignored_users = await self.store.ignored_users(user_id)
         if since_token:
             room_changes = await self._get_rooms_changed(
-                sync_result_builder, ignored_users
+                sync_result_builder, ignored_users, self.rooms_to_exclude
             )
             tags_by_room = await self.store.get_updated_tags(
                 user_id, since_token.account_data_key
             )
         else:
-            room_changes = await self._get_all_rooms(sync_result_builder, ignored_users)
+            room_changes = await self._get_all_rooms(
+                sync_result_builder, ignored_users, self.rooms_to_exclude
+            )
             tags_by_room = await self.store.get_tags_for_user(user_id)
 
         log_kv({"rooms_changed": len(room_changes.room_entries)})
@@ -1689,7 +1693,10 @@ async def _have_rooms_changed(
         return False
 
     async def _get_rooms_changed(
-        self, sync_result_builder: "SyncResultBuilder", ignored_users: FrozenSet[str]
+        self,
+        sync_result_builder: "SyncResultBuilder",
+        ignored_users: FrozenSet[str],
+        excluded_rooms: List[str],
     ) -> _RoomChanges:
         """Determine the changes in rooms to report to the user.
 
@@ -1721,7 +1728,7 @@ async def _get_rooms_changed(
         #       _have_rooms_changed. We could keep the results in memory to avoid a
         #       second query, at the cost of more complicated source code.
         membership_change_events = await self.store.get_membership_changes_for_user(
-            user_id, since_token.room_key, now_token.room_key
+            user_id, since_token.room_key, now_token.room_key, excluded_rooms
         )
 
         mem_change_events_by_room_id: Dict[str, List[EventBase]] = {}
@@ -1922,7 +1929,10 @@ async def _get_rooms_changed(
         )
 
     async def _get_all_rooms(
-        self, sync_result_builder: "SyncResultBuilder", ignored_users: FrozenSet[str]
+        self,
+        sync_result_builder: "SyncResultBuilder",
+        ignored_users: FrozenSet[str],
+        ignored_rooms: List[str],
     ) -> _RoomChanges:
         """Returns entries for all rooms for the user.
 
@@ -1933,7 +1943,7 @@ async def _get_all_rooms(
         Args:
             sync_result_builder
             ignored_users: Set of users ignored by user.
-
+            ignored_rooms: List of rooms to ignore.
         """
 
         user_id = sync_result_builder.sync_config.user.to_string()
@@ -1944,6 +1954,7 @@ async def _get_all_rooms(
         room_list = await self.store.get_rooms_for_local_user_where_membership_is(
             user_id=user_id,
             membership_list=Membership.LIST,
+            excluded_rooms=ignored_rooms,
         )
 
         room_entries = []

synapse/rest/key/v2/local_key_resource.py

@@ -76,17 +76,17 @@ def update_response_body(self, time_now_msec: int) -> None:
 
     def response_json_object(self) -> JsonDict:
         verify_keys = {}
-        for key in self.config.key.signing_key:
-            verify_key_bytes = key.verify_key.encode()
-            key_id = "%s:%s" % (key.alg, key.version)
+        for signing_key in self.config.key.signing_key:
+            verify_key_bytes = signing_key.verify_key.encode()
+            key_id = "%s:%s" % (signing_key.alg, signing_key.version)
             verify_keys[key_id] = {"key": encode_base64(verify_key_bytes)}
 
         old_verify_keys = {}
-        for key_id, key in self.config.key.old_signing_keys.items():
-            verify_key_bytes = key.encode()
+        for key_id, old_signing_key in self.config.key.old_signing_keys.items():
+            verify_key_bytes = old_signing_key.encode()
             old_verify_keys[key_id] = {
                 "key": encode_base64(verify_key_bytes),
-                "expired_ts": key.expired_ts,
+                "expired_ts": old_signing_key.expired,
             }
 
         json_object = {

synapse/rest/key/v2/remote_key_resource.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import logging
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Dict, Set
 
 from signedjson.sign import sign_json
 
@@ -149,7 +149,7 @@ async def query_keys(
 
         cached = await self.store.get_server_keys_json(store_queries)
 
-        json_results = set()
+        json_results: Set[bytes] = set()
 
         time_now_ms = self.clock.time_msec()
 
@@ -234,8 +234,8 @@ async def query_keys(
             await self.query_keys(request, query, query_remote_on_cache_miss=False)
         else:
             signed_keys = []
-            for key_json in json_results:
-                key_json = json_decoder.decode(key_json.decode("utf-8"))
+            for key_json_raw in json_results:
+                key_json = json_decoder.decode(key_json_raw.decode("utf-8"))
                 for signing_key in self.config.key.key_server_signing_keys:
                     key_json = sign_json(
                         key_json, self.config.server.server_name, signing_key

synapse/storage/databases/main/roommember.py

@@ -361,7 +361,10 @@ async def get_invite_for_local_user_in_room(
         return None
 
     async def get_rooms_for_local_user_where_membership_is(
-        self, user_id: str, membership_list: Collection[str]
+        self,
+        user_id: str,
+        membership_list: Collection[str],
+        excluded_rooms: Optional[List[str]] = None,
     ) -> List[RoomsForUser]:
         """Get all the rooms for this *local* user where the membership for this user
         matches one in the membership list.
@@ -372,6 +375,7 @@ async def get_rooms_for_local_user_where_membership_is(
             user_id: The user ID.
             membership_list: A list of synapse.api.constants.Membership
                 values which the user must be in.
+            excluded_rooms: A list of rooms to ignore.
 
         Returns:
             The RoomsForUser that the user matches the membership types.
@@ -386,12 +390,19 @@ async def get_rooms_for_local_user_where_membership_is(
             membership_list,
         )
 
-        # Now we filter out forgotten rooms
-        forgotten_rooms = await self.get_forgotten_rooms_for_user(user_id)
-        return [room for room in rooms if room.room_id not in forgotten_rooms]
+        # Now we filter out forgotten and excluded rooms
+        rooms_to_exclude: Set[str] = await self.get_forgotten_rooms_for_user(user_id)
+
+        if excluded_rooms is not None:
+            rooms_to_exclude.update(set(excluded_rooms))
+
+        return [room for room in rooms if room.room_id not in rooms_to_exclude]
 
     def _get_rooms_for_local_user_where_membership_is_txn(
-        self, txn, user_id: str, membership_list: List[str]
+        self,
+        txn,
+        user_id: str,
+        membership_list: List[str],
     ) -> List[RoomsForUser]:
         # Paranoia check.
         if not self.hs.is_mine_id(user_id):

synapse/storage/databases/main/stream.py

@@ -36,7 +36,7 @@
 """
 
 import logging
-from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Any, Collection, Dict, List, Optional, Set, Tuple
 
 import attr
 from frozendict import frozendict
@@ -585,7 +585,11 @@ def f(txn: LoggingTransaction) -> List[_EventDictReturn]:
         return ret, key
 
     async def get_membership_changes_for_user(
-        self, user_id: str, from_key: RoomStreamToken, to_key: RoomStreamToken
+        self,
+        user_id: str,
+        from_key: RoomStreamToken,
+        to_key: RoomStreamToken,
+        excluded_rooms: Optional[List[str]] = None,
     ) -> List[EventBase]:
         """Fetch membership events for a given user.
 
@@ -610,23 +614,29 @@ def f(txn: LoggingTransaction) -> List[_EventDictReturn]:
             min_from_id = from_key.stream
             max_to_id = to_key.get_max_stream_pos()
 
+            args: List[Any] = [user_id, min_from_id, max_to_id]
+
+            ignore_room_clause = ""
+            if excluded_rooms is not None and len(excluded_rooms) > 0:
+                ignore_room_clause = "AND e.room_id NOT IN (%s)" % ",".join(
+                    "?" for _ in excluded_rooms
+                )
+                args = args + excluded_rooms
+
             sql = """
                 SELECT m.event_id, instance_name, topological_ordering, stream_ordering
                 FROM events AS e, room_memberships AS m
                 WHERE e.event_id = m.event_id
                     AND m.user_id = ?
                     AND e.stream_ordering > ? AND e.stream_ordering <= ?
+                    %s
                 ORDER BY e.stream_ordering ASC
-            """
-            txn.execute(
-                sql,
-                (
-                    user_id,
-                    min_from_id,
-                    max_to_id,
-                ),
+            """ % (
+                ignore_room_clause,
             )
 
+            txn.execute(sql, args)
+
             rows = [
                 _EventDictReturn(event_id, None, stream_ordering)
                 for event_id, instance_name, topological_ordering, stream_ordering in txn

tests/crypto/test_event_signing.py

@@ -28,8 +28,8 @@
 SIGNING_KEY_SEED = decode_base64("YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1")
 
 KEY_ALG = "ed25519"
-KEY_VER = 1
-KEY_NAME = "%s:%d" % (KEY_ALG, KEY_VER)
+KEY_VER = "1"
+KEY_NAME = "%s:%s" % (KEY_ALG, KEY_VER)
 
 HOSTNAME = "domain"
 
@@ -39,7 +39,7 @@ def setUp(self):
         # NB: `signedjson` expects `nacl.signing.SigningKey` instances which have been
         # monkeypatched to include new `alg` and `version` attributes. This is captured
         # by the `signedjson.types.SigningKey` protocol.
-        self.signing_key: signedjson.types.SigningKey = nacl.signing.SigningKey(
+        self.signing_key: signedjson.types.SigningKey = nacl.signing.SigningKey(  # type: ignore[assignment]
             SIGNING_KEY_SEED
         )
         self.signing_key.alg = KEY_ALG