Support nested Matrix clients via the widget API (#2473)

* WIP RoomWidgetClient

* Wait for the widget API to become ready before backfilling

* Add support for sending user-defined encrypted to-device messages

This is a port of the same change from the robertlong/group-call branch.

* Fix tests

* Emit an event when the client receives TURN servers

* Expose the method in MatrixClient

* Override the encryptAndSendToDevices method

* Add support for TURN servers in embedded mode and make calls mostly work

* Don't put unclonable objects into VoIP events

RoomWidget clients were unable to send m.call.candidate events, because
the candidate objects were not clonable for use with postMessage.
Converting such objects to their canonical JSON form before attempting
to send them over the wire solves this.

* Fix types

* Fix more types

* Fix lint

* Upgrade matrix-widget-api

* Save lockfile

* Untangle dependencies to fix tests

* Add some preliminary tests

* Fix tests

* Fix indirect export

* Add more tests

* Resolve TODOs

* Add queueToDevice to RoomWidgetClient

Author: robintown

package.json

@@ -61,6 +61,7 @@
     "content-type": "^1.0.4",
     "loglevel": "^1.7.1",
     "matrix-events-sdk": "^0.0.1-beta.7",
+    "matrix-widget-api": "^1.0.0",
     "p-retry": "4",
     "qs": "^6.9.6",
     "request": "^2.88.2",
@@ -101,6 +102,7 @@
     "exorcist": "^2.0.0",
     "fake-indexeddb": "^4.0.0",
     "jest": "^28.0.0",
+    "jest-environment-jsdom": "^28.1.3",
     "jest-localstorage-mock": "^2.4.6",
     "jest-sonar-reporter": "^2.0.0",
     "jsdoc": "^3.6.6",

spec/unit/embedded.spec.ts

@@ -0,0 +1,264 @@
+/**
+ * @jest-environment jsdom
+ */
+
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// We have to use EventEmitter here to mock part of the matrix-widget-api
+// project, which doesn't know about our TypeEventEmitter implementation at all
+// eslint-disable-next-line no-restricted-imports
+import { EventEmitter } from "events";
+import { MockedObject } from "jest-mock";
+import {
+    WidgetApi,
+    WidgetApiToWidgetAction,
+    MatrixCapabilities,
+    ITurnServer,
+} from "matrix-widget-api";
+
+import { createRoomWidgetClient } from "../../src/matrix";
+import { MatrixClient, ClientEvent, ITurnServer as IClientTurnServer } from "../../src/client";
+import { SyncState } from "../../src/sync";
+import { ICapabilities } from "../../src/embedded";
+import { MatrixEvent } from "../../src/models/event";
+import { ToDeviceBatch } from "../../src/models/ToDeviceMessage";
+import { DeviceInfo } from "../../src/crypto/deviceinfo";
+
+class MockWidgetApi extends EventEmitter {
+    public start = jest.fn();
+    public requestCapability = jest.fn();
+    public requestCapabilities = jest.fn();
+    public requestCapabilityToSendState = jest.fn();
+    public requestCapabilityToReceiveState = jest.fn();
+    public requestCapabilityToSendToDevice = jest.fn();
+    public requestCapabilityToReceiveToDevice = jest.fn();
+    public sendStateEvent = jest.fn();
+    public sendToDevice = jest.fn();
+    public readStateEvents = jest.fn(() => []);
+    public getTurnServers = jest.fn(() => []);
+
+    public transport = { reply: jest.fn() };
+}
+
+describe("RoomWidgetClient", () => {
+    let widgetApi: MockedObject<WidgetApi>;
+    let client: MatrixClient;
+
+    beforeEach(() => {
+        widgetApi = new MockWidgetApi() as unknown as MockedObject<WidgetApi>;
+    });
+
+    afterEach(() => {
+        client.stopClient();
+    });
+
+    const makeClient = async (capabilities: ICapabilities): Promise<void> => {
+        const baseUrl = "https://example.org";
+        client = createRoomWidgetClient(widgetApi, capabilities, "!1:example.org", { baseUrl });
+        expect(widgetApi.start).toHaveBeenCalled(); // needs to have been called early in order to not miss messages
+        widgetApi.emit("ready");
+        await client.startClient();
+    };
+
+    describe("state events", () => {
+        const event = new MatrixEvent({
+            type: "org.example.foo",
+            event_id: "$sfkjfsksdkfsd",
+            room_id: "!1:example.org",
+            sender: "@alice:example.org",
+            state_key: "bar",
+            content: { hello: "world" },
+        }).getEffectiveEvent();
+
+        it("sends", async () => {
+            await makeClient({ sendState: [{ eventType: "org.example.foo", stateKey: "bar" }] });
+            expect(widgetApi.requestCapabilityToSendState).toHaveBeenCalledWith("org.example.foo", "bar");
+            await client.sendStateEvent("!1:example.org", "org.example.foo", { hello: "world" }, "bar");
+            expect(widgetApi.sendStateEvent).toHaveBeenCalledWith("org.example.foo", "bar", { hello: "world" });
+        });
+
+        it("refuses to send to other rooms", async () => {
+            await makeClient({ sendState: [{ eventType: "org.example.foo", stateKey: "bar" }] });
+            expect(widgetApi.requestCapabilityToSendState).toHaveBeenCalledWith("org.example.foo", "bar");
+            await expect(client.sendStateEvent("!2:example.org", "org.example.foo", { hello: "world" }, "bar"))
+                .rejects.toBeDefined();
+        });
+
+        it("receives", async () => {
+            await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] });
+            expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar");
+
+            const emittedEvent = new Promise<MatrixEvent>(resolve => client.once(ClientEvent.Event, resolve));
+            const emittedSync = new Promise<SyncState>(resolve => client.once(ClientEvent.Sync, resolve));
+            widgetApi.emit(
+                `action:${WidgetApiToWidgetAction.SendEvent}`,
+                new CustomEvent(`action:${WidgetApiToWidgetAction.SendEvent}`, { detail: { data: event } }),
+            );
+
+            // The client should've emitted about the received event
+            expect((await emittedEvent).getEffectiveEvent()).toEqual(event);
+            expect(await emittedSync).toEqual(SyncState.Syncing);
+            // It should've also inserted the event into the room object
+            const room = client.getRoom("!1:example.org");
+            expect(room.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event);
+        });
+
+        it("backfills", async () => {
+            widgetApi.readStateEvents.mockImplementation(async (eventType, limit, stateKey) =>
+                eventType === "org.example.foo" && (limit ?? Infinity) > 0 && stateKey === "bar"
+                    ? [event]
+                    : [],
+            );
+
+            await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] });
+            expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar");
+
+            const room = client.getRoom("!1:example.org");
+            expect(room.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event);
+        });
+    });
+
+    describe("to-device messages", () => {
+        const unencryptedContentMap = {
+            "@alice:example.org": { "*": { hello: "alice!" } },
+            "@bob:example.org": { bobDesktop: { hello: "bob!" } },
+        };
+
+        it("sends unencrypted (sendToDevice)", async () => {
+            await makeClient({ sendToDevice: ["org.example.foo"] });
+            expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo");
+
+            await client.sendToDevice("org.example.foo", unencryptedContentMap);
+            expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", false, unencryptedContentMap);
+        });
+
+        it("sends unencrypted (queueToDevice)", async () => {
+            await makeClient({ sendToDevice: ["org.example.foo"] });
+            expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo");
+
+            const batch: ToDeviceBatch = {
+                eventType: "org.example.foo",
+                batch: [
+                    { userId: "@alice:example.org", deviceId: "*", payload: { hello: "alice!" } },
+                    { userId: "@bob:example.org", deviceId: "bobDesktop", payload: { hello: "bob!" } },
+                ],
+            };
+            await client.queueToDevice(batch);
+            expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", false, unencryptedContentMap);
+        });
+
+        it("sends encrypted (encryptAndSendToDevices)", async () => {
+            await makeClient({ sendToDevice: ["org.example.foo"] });
+            expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo");
+
+            const payload = { type: "org.example.foo", hello: "world" };
+            await client.encryptAndSendToDevices(
+                [
+                    { userId: "@alice:example.org", deviceInfo: new DeviceInfo("aliceWeb") },
+                    { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobDesktop") },
+                ],
+                payload,
+            );
+            expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", true, {
+                "@alice:example.org": { aliceWeb: payload },
+                "@bob:example.org": { bobDesktop: payload },
+            });
+        });
+
+        it.each([
+            { encrypted: false, title: "unencrypted" },
+            { encrypted: true, title: "encrypted" },
+        ])("receives $title", async ({ encrypted }) => {
+            await makeClient({ receiveToDevice: ["org.example.foo"] });
+            expect(widgetApi.requestCapabilityToReceiveToDevice).toHaveBeenCalledWith("org.example.foo");
+
+            const event = {
+                type: "org.example.foo",
+                sender: "@alice:example.org",
+                encrypted,
+                content: { hello: "world" },
+            };
+
+            const emittedEvent = new Promise<MatrixEvent>(resolve => client.once(ClientEvent.ToDeviceEvent, resolve));
+            const emittedSync = new Promise<SyncState>(resolve => client.once(ClientEvent.Sync, resolve));
+            widgetApi.emit(
+                `action:${WidgetApiToWidgetAction.SendToDevice}`,
+                new CustomEvent(`action:${WidgetApiToWidgetAction.SendToDevice}`, { detail: { data: event } }),
+            );
+
+            expect((await emittedEvent).getEffectiveEvent()).toEqual({
+                type: event.type,
+                sender: event.sender,
+                content: event.content,
+            });
+            expect((await emittedEvent).isEncrypted()).toEqual(encrypted);
+            expect(await emittedSync).toEqual(SyncState.Syncing);
+        });
+    });
+
+    it("gets TURN servers", async () => {
+        const server1: ITurnServer = {
+            uris: [
+                "turn:turn.example.com:3478?transport=udp",
+                "turn:10.20.30.40:3478?transport=tcp",
+                "turns:10.20.30.40:443?transport=tcp",
+            ],
+            username: "1443779631:@user:example.com",
+            password: "JlKfBy1QwLrO20385QyAtEyIv0=",
+        };
+        const server2: ITurnServer = {
+            uris: [
+                "turn:turn.example.com:3478?transport=udp",
+                "turn:10.20.30.40:3478?transport=tcp",
+                "turns:10.20.30.40:443?transport=tcp",
+            ],
+            username: "1448999322:@user:example.com",
+            password: "hunter2",
+        };
+        const clientServer1: IClientTurnServer = {
+            urls: server1.uris,
+            username: server1.username,
+            credential: server1.password,
+        };
+        const clientServer2: IClientTurnServer = {
+            urls: server2.uris,
+            username: server2.username,
+            credential: server2.password,
+        };
+
+        let emitServer2: () => void;
+        const getServer2 = new Promise<ITurnServer>(resolve => emitServer2 = () => resolve(server2));
+        widgetApi.getTurnServers.mockImplementation(async function* () {
+            yield server1;
+            yield await getServer2;
+        });
+
+        await makeClient({ turnServers: true });
+        expect(widgetApi.requestCapability).toHaveBeenCalledWith(MatrixCapabilities.MSC3846TurnServers);
+
+        // The first server should've arrived immediately
+        expect(client.getTurnServers()).toEqual([clientServer1]);
+
+        // Subsequent servers arrive asynchronously and should emit an event
+        const emittedServer = new Promise<IClientTurnServer[]>(resolve =>
+            client.once(ClientEvent.TurnServers, resolve),
+        );
+        emitServer2();
+        expect(await emittedServer).toEqual([clientServer2]);
+        expect(client.getTurnServers()).toEqual([clientServer2]);
+    });
+});

src/ToDeviceMessageQueue.ts

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 import { logger } from "./logger";
-import { MatrixClient } from "./matrix";
+import { MatrixClient } from "./client";
 import { IndexedToDeviceBatch, ToDeviceBatch, ToDeviceBatchWithTxnId, ToDevicePayload } from "./models/ToDeviceMessage";
 import { MatrixScheduler } from "./scheduler";
 

src/client.ts

@@ -72,6 +72,7 @@ import {
     CryptoEvent,
     CryptoEventHandlerMap,
     fixBackupKey,
+    ICryptoCallbacks,
     IBootstrapCrossSigningOpts,
     ICheckOwnCrossSigningTrustOpts,
     IMegolmSessionData,
@@ -101,29 +102,9 @@ import {
 } from "./crypto/keybackup";
 import { IIdentityServerProvider } from "./@types/IIdentityServerProvider";
 import { MatrixScheduler } from "./scheduler";
-import {
-    IAuthData,
-    ICryptoCallbacks,
-    IMinimalEvent,
-    IRoomEvent,
-    IStateEvent,
-    NotificationCountType,
-    BeaconEvent,
-    BeaconEventHandlerMap,
-    RoomEvent,
-    RoomEventHandlerMap,
-    RoomMemberEvent,
-    RoomMemberEventHandlerMap,
-    RoomStateEvent,
-    RoomStateEventHandlerMap,
-    INotificationsResponse,
-    IFilterResponse,
-    ITagsResponse,
-    IStatusResponse,
-    IPushRule,
-    PushRuleActionName,
-    IAuthDict,
-} from "./matrix";
+import { BeaconEvent, BeaconEventHandlerMap } from "./models/beacon";
+import { IAuthData, IAuthDict } from "./interactive-auth";
+import { IMinimalEvent, IRoomEvent, IStateEvent } from "./sync-accumulator";
 import {
     CrossSigningKey,
     IAddSecretStorageKeyOpts,
@@ -138,7 +119,9 @@ import { VerificationRequest } from "./crypto/verification/request/VerificationR
 import { VerificationBase as Verification } from "./crypto/verification/Base";
 import * as ContentHelpers from "./content-helpers";
 import { CrossSigningInfo, DeviceTrustLevel, ICacheCallbacks, UserTrustLevel } from "./crypto/CrossSigning";
-import { Room } from "./models/room";
+import { Room, NotificationCountType, RoomEvent, RoomEventHandlerMap } from "./models/room";
+import { RoomMemberEvent, RoomMemberEventHandlerMap } from "./models/room-member";
+import { RoomStateEvent, RoomStateEventHandlerMap } from "./models/room-state";
 import {
     IAddThreePidOnlyBody,
     IBindThreePidBody,
@@ -156,6 +139,10 @@ import {
     ISearchOpts,
     ISendEventResponse,
     IUploadOpts,
+    INotificationsResponse,
+    IFilterResponse,
+    ITagsResponse,
+    IStatusResponse,
 } from "./@types/requests";
 import {
     EventType,
@@ -185,7 +172,16 @@ import {
 } from "./@types/search";
 import { ISynapseAdminDeactivateResponse, ISynapseAdminWhoisResponse } from "./@types/synapse";
 import { IHierarchyRoom } from "./@types/spaces";
-import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules";
+import {
+    IPusher,
+    IPusherRequest,
+    IPushRule,
+    IPushRules,
+    PushRuleAction,
+    PushRuleActionName,
+    PushRuleKind,
+    RuleId,
+} from "./@types/PushRules";
 import { IThreepid } from "./@types/threepids";
 import { CryptoStore } from "./crypto/store/base";
 import {

src/crypto/CrossSigning.ts

@@ -29,7 +29,7 @@ import { DeviceInfo } from "./deviceinfo";
 import { SecretStorage } from "./SecretStorage";
 import { ICrossSigningKey, ISignedKey, MatrixClient } from "../client";
 import { OlmDevice } from "./OlmDevice";
-import { ICryptoCallbacks } from "../matrix";
+import { ICryptoCallbacks } from ".";
 import { ISignatures } from "../@types/signed";
 import { CryptoStore } from "./store/base";
 import { ISecretStorageKeyInfo } from "./api";

src/crypto/EncryptionSetup.ts

@@ -19,16 +19,15 @@ import { MatrixEvent } from "../models/event";
 import { createCryptoStoreCacheCallbacks, ICacheCallbacks } from "./CrossSigning";
 import { IndexedDBCryptoStore } from './store/indexeddb-crypto-store';
 import { Method, PREFIX_UNSTABLE } from "../http-api";
-import { Crypto, IBootstrapCrossSigningOpts } from "./index";
+import { Crypto, ICryptoCallbacks, IBootstrapCrossSigningOpts } from "./index";
 import {
     ClientEvent,
-    CrossSigningKeys,
     ClientEventHandlerMap,
+    CrossSigningKeys,
     ICrossSigningKey,
-    ICryptoCallbacks,
     ISignedKey,
     KeySignatures,
-} from "../matrix";
+} from "../client";
 import { ISecretStorageKeyInfo } from "./api";
 import { IKeyBackupInfo } from "./keybackup";
 import { TypedEventEmitter } from "../models/typed-event-emitter";