media/create: add /create endpoint

See matrix-org/matrix-spec-proposals#2246 for details

Signed-off-by: Sumner Evans <[email protected]>

Author: sumnerevans

flake.nix

@@ -104,6 +104,7 @@
                   libxml2
                   libxslt
                   sqlite
+                  gcc
 
                   # Native dependencies for unit tests (SyTest also requires OpenSSL).
                   openssl
@@ -152,7 +153,7 @@
 
                 # Postgres is needed to run Synapse with postgres support and
                 # to run certain unit tests that require postgres.
-                services.postgres.enable = true;
+                services.postgres.enable = false;
 
                 # On the first invocation of `devenv up`, create a database for
                 # Synapse to store data in.
@@ -168,7 +169,7 @@
                 '';
 
                 # Redis is needed in order to run Synapse in worker mode.
-                services.redis.enable = true;
+                services.redis.enable = false;
 
                 # Configure and start Synapse. Before starting Synapse, this shell code:
                 #  * generates a default homeserver.yaml config file if one does not exist, and
@@ -178,22 +179,6 @@
                 process.before = ''
                   python -m synapse.app.homeserver -c homeserver.yaml --generate-config --server-name=synapse.dev --report-stats=no
                   mkdir -p homeserver-config-overrides.d
-                  cat > homeserver-config-overrides.d/database.yaml << EOF
-                  ## Do not edit this file. This file is generated by flake.nix
-                  database:
-                    name: psycopg2
-                    args:
-                      user: synapse_user
-                      database: synapse
-                      host: $PGHOST
-                      cp_min: 5
-                      cp_max: 10
-                  EOF
-                  cat > homeserver-config-overrides.d/redis.yaml << EOF
-                  ## Do not edit this file. This file is generated by flake.nix
-                  redis:
-                    enabled: true
-                  EOF
                 '';
                 # Start synapse when `devenv up` is run.
                 processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d";

synapse/media/media_repository.py

@@ -78,6 +78,7 @@ def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastores().main
         self.max_upload_size = hs.config.media.max_upload_size
         self.max_image_pixels = hs.config.media.max_image_pixels
+        self.unused_expiration_time = hs.config.media.unused_expiration_time
 
         Thumbnailer.set_limits(self.max_image_pixels)
 
@@ -183,6 +184,29 @@ def mark_recently_accessed(self, server_name: Optional[str], media_id: str) -> N
         else:
             self.recently_accessed_locals.add(media_id)
 
+    @trace
+    async def create_media_id(self, auth_user: UserID) -> Tuple[str, int]:
+        """Create and store a media ID for a local user and return the MXC URI and its
+        expiration.
+
+        Args:
+            auth_user: The user_id of the uploader
+
+        Returns:
+            A tuple containing the MXC URI of the stored content and the timestamp at
+            which the MXC URI expires.
+        """
+        media_id = random_string(24)
+        now = self.clock.time_msec()
+        unused_expires_at = now + self.unused_expiration_time
+        await self.store.store_local_media_id(
+            media_id=media_id,
+            time_now_ms=now,
+            user_id=auth_user,
+            unused_expires_at=unused_expires_at,
+        )
+        return f"mxc://{self.server_name}/{media_id}", unused_expires_at
+
     @trace
     async def create_content(
         self,

synapse/rest/media/create_resource.py

@@ -0,0 +1,88 @@
+# Copyright 2023 Beeper Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from synapse.api.errors import LimitExceededError
+from synapse.api.ratelimiting import Ratelimiter
+from synapse.http.server import respond_with_json
+from synapse.http.servlet import RestServlet
+from synapse.http.site import SynapseRequest
+
+if TYPE_CHECKING:
+    from synapse.media.media_repository import MediaRepository
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class CreateResource(RestServlet):
+    PATTERNS = [re.compile("/_matrix/media/v1/create")]
+    isLeaf = True
+
+    def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"):
+        super().__init__()
+
+        self.media_repo = media_repo
+        self.clock = hs.get_clock()
+        self.auth = hs.get_auth()
+
+        # A rate limiter for creating new media IDs.
+        self._create_media_rate_limiter = Ratelimiter(
+            store=hs.get_datastores().main,
+            clock=self.clock,
+            cfg=hs.config.ratelimiting.rc_media_create,
+        )
+
+    async def _async_render_OPTIONS(self, request: SynapseRequest) -> None:
+        respond_with_json(request, 200, {}, send_cors=True)
+
+    async def _async_render_POST(self, request: SynapseRequest) -> None:
+        requester = await self.auth.get_user_by_req(request)
+
+        # If the create media requests for the user are over the limit, drop them.
+        await self._create_media_rate_limiter.ratelimit(requester)
+
+        (
+            reached_pending_limit,
+            first_expiration_ts,
+        ) = await self.media_repo.reached_pending_media_limit(
+            requester.user, self.max_pending_media_uploads
+        )
+        if reached_pending_limit:
+            raise LimitExceededError(
+                limiter_name="max_pending_media_uploads",
+                retry_after_ms=first_expiration_ts - self.clock.time_msec(),
+            )
+
+        content_uri, unused_expires_at = await self.media_repo.create_media_id(
+            requester.user
+        )
+
+        logger.info(
+            "Created Media URI %r that if unused will expire at %d",
+            content_uri,
+            unused_expires_at,
+        )
+        respond_with_json(
+            request,
+            200,
+            {
+                "content_uri": content_uri,
+                "unused_expires_at": unused_expires_at,
+            },
+            send_cors=True,
+        )

synapse/rest/media/media_repository_resource.py

@@ -18,6 +18,7 @@
 from synapse.http.server import HttpServer, JsonResource
 
 from .config_resource import MediaConfigResource
+from .create_resource import CreateResource
 from .download_resource import DownloadResource
 from .preview_url_resource import PreviewUrlResource
 from .thumbnail_resource import ThumbnailResource
@@ -91,7 +92,7 @@ def register_servlets(http_server: HttpServer, hs: "HomeServer") -> None:
 
         # Note that many of these should not exist as v1 endpoints, but empirically
         # a lot of traffic still goes to them.
-
+        CreateResource(hs, media_repo).register(http_server)
         UploadResource(hs, media_repo).register(http_server)
         DownloadResource(hs, media_repo).register(http_server)
         ThumbnailResource(hs, media_repo, media_repo.media_storage).register(

synapse/storage/databases/main/media_repository.py

@@ -330,6 +330,25 @@ def _get_local_media_ids_txn(txn: LoggingTransaction) -> List[str]:
             "get_local_media_ids", _get_local_media_ids_txn
         )
 
+    @trace
+    async def store_local_media_id(
+        self,
+        media_id: str,
+        time_now_ms: int,
+        user_id: UserID,
+        unused_expires_at: int,
+    ) -> None:
+        await self.db_pool.simple_insert(
+            "local_media_repository",
+            {
+                "media_id": media_id,
+                "created_ts": time_now_ms,
+                "user_id": user_id.to_string(),
+                "unused_expires_at": unused_expires_at,
+            },
+            desc="store_local_media_id",
+        )
+
     @trace
     async def store_local_media(
         self,

synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql

@@ -0,0 +1,22 @@
+/* Copyright 2023 Beeper Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Add new colums to the `local_media_repository` to keep track of when the
+-- media ID must be used by. This is to support async uploads (see MSC2246).
+
+ALTER TABLE local_media_repository
+  ADD COLUMN unused_expires_at BIGINT DEFAULT NULL;
+
+CREATE INDEX CONCURRENTLY ON local_media_repository (unused_expires_at);