Merge pull request #648 from vector-im/dbkr/tiles_for_everyone

dbkr · web-flow · commit b8af9a073378 · 2022-10-25T12:56:43.000+01:00
Show tiles for members we're trying to connect to
diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json
@@ -2,6 +2,7 @@
   "{{count}} people connected|one": "{{count}} person connected",
   "{{count}} people connected|other": "{{count}} people connected",
   "{{displayName}}, your call is now ended": "{{displayName}}, your call is now ended",
+  "{{name}} (Connecting...)": "{{name}} (Connecting...)",
   "{{name}} is presenting": "{{name}} is presenting",
   "{{name}} is talking…": "{{name}} is talking…",
   "{{names}}, {{name}}": "{{names}}, {{name}}",
diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx
@@ -76,7 +76,6 @@ export function GroupCallView({
     toggleScreensharing,
     requestingScreenshare,
     isScreensharing,
-    localScreenshareFeed,
     screenshareFeeds,
     participants,
     unencryptedEventsFromUsers,
@@ -221,6 +220,7 @@ export function GroupCallView({
           client={client}
           roomName={groupCall.room.name}
           avatarUrl={avatarUrl}
+          participants={participants}
           microphoneMuted={microphoneMuted}
           localVideoMuted={localVideoMuted}
           toggleLocalVideoMuted={toggleLocalVideoMuted}
@@ -230,7 +230,6 @@ export function GroupCallView({
           onLeave={onLeave}
           toggleScreensharing={toggleScreensharing}
           isScreensharing={isScreensharing}
-          localScreenshareFeed={localScreenshareFeed}
           screenshareFeeds={screenshareFeeds}
           roomIdOrAlias={roomIdOrAlias}
           unencryptedEventsFromUsers={unencryptedEventsFromUsers}
diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx
@@ -70,6 +70,7 @@ const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
 interface Props {
   client: MatrixClient;
   groupCall: GroupCall;
+  participants: RoomMember[];
   roomName: string;
   avatarUrl: string;
   microphoneMuted: boolean;
@@ -82,14 +83,16 @@ interface Props {
   onLeave: () => void;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   roomIdOrAlias: string;
   unencryptedEventsFromUsers: Set<string>;
   hideHeader: boolean;
 }
 
-export interface Participant {
+// Represents something that should get a tile on the layout,
+// ie. a user's video feed or a screen share feed.
+export interface TileDescriptor {
   id: string;
+  member: RoomMember;
   focused: boolean;
   presenter: boolean;
   callFeed?: CallFeed;
@@ -99,6 +102,7 @@ export interface Participant {
 export function InCallView({
   client,
   groupCall,
+  participants,
   roomName,
   avatarUrl,
   microphoneMuted,
@@ -111,7 +115,6 @@ export function InCallView({
   toggleScreensharing,
   isScreensharing,
   screenshareFeeds,
-  localScreenshareFeed,
   roomIdOrAlias,
   unencryptedEventsFromUsers,
   hideHeader,
@@ -185,39 +188,48 @@ export function InCallView({
   }, [setLayout]);
 
   const items = useMemo(() => {
-    const participants: Participant[] = [];
-
-    for (const callFeed of userMediaFeeds) {
-      participants.push({
-        id: callFeed.stream.id,
-        callFeed,
-        focused:
-          screenshareFeeds.length === 0 && callFeed.userId === activeSpeaker,
-        isLocal: callFeed.isLocal(),
+    const tileDescriptors: TileDescriptor[] = [];
+
+    // one tile for each participants, to start with (we want a tile for everyone we
+    // think should be in the call, even if we don't have a media feed for them yet)
+    for (const p of participants) {
+      const userMediaFeed = userMediaFeeds.find((f) => f.userId === p.userId);
+
+      // NB. this assumes that the same user can't join more than once from multiple
+      // devices, but the participants are just RoomMembers, so this assumption is baked
+      // into GroupCall itself.
+      tileDescriptors.push({
+        id: p.userId,
+        member: p,
+        callFeed: userMediaFeed,
+        focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker,
+        isLocal: p.userId === client.getUserId(),
         presenter: false,
       });
     }
 
-    for (const callFeed of screenshareFeeds) {
-      const userMediaItem = participants.find(
-        (item) => item.callFeed.userId === callFeed.userId
+    // add the screenshares too
+    for (const screenshareFeed of screenshareFeeds) {
+      const userMediaItem = tileDescriptors.find(
+        (item) => item.member.userId === screenshareFeed.userId
       );
 
       if (userMediaItem) {
         userMediaItem.presenter = true;
       }
 
-      participants.push({
-        id: callFeed.stream.id,
-        callFeed,
+      tileDescriptors.push({
+        id: screenshareFeed.stream.id,
+        member: userMediaItem?.member,
+        callFeed: screenshareFeed,
         focused: true,
-        isLocal: callFeed.isLocal(),
+        isLocal: screenshareFeed.isLocal(),
         presenter: false,
       });
     }
 
-    return participants;
-  }, [userMediaFeeds, activeSpeaker, screenshareFeeds]);
+    return tileDescriptors;
+  }, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]);
 
   // The maximised participant: either the participant that the user has
   // manually put in fullscreen, or the focused (active) participant if the
@@ -281,7 +293,13 @@ export function InCallView({
 
     return (
       <VideoGrid items={items} layout={layout} disableAnimations={isSafari}>
-        {({ item, ...rest }: { item: Participant; [x: string]: unknown }) => (
+        {({
+          item,
+          ...rest
+        }: {
+          item: TileDescriptor;
+          [x: string]: unknown;
+        }) => (
           <VideoTileContainer
             key={item.id}
             item={item}
diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts
@@ -51,7 +51,6 @@ export interface UseGroupCallReturnType {
   requestingScreenshare: boolean;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   localDesktopCapturerSourceId: string;
   participants: RoomMember[];
   hasLocalParticipant: boolean;
@@ -68,7 +67,6 @@ interface State {
   microphoneMuted: boolean;
   localVideoMuted: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   localDesktopCapturerSourceId: string;
   isScreensharing: boolean;
   requestingScreenshare: boolean;
@@ -89,7 +87,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
       localVideoMuted,
       isScreensharing,
       screenshareFeeds,
-      localScreenshareFeed,
       localDesktopCapturerSourceId,
       participants,
       hasLocalParticipant,
@@ -107,7 +104,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     localVideoMuted: false,
     isScreensharing: false,
     screenshareFeeds: [],
-    localScreenshareFeed: null,
     localDesktopCapturerSourceId: null,
     requestingScreenshare: false,
     participants: [],
@@ -135,7 +131,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
         microphoneMuted: groupCall.isMicrophoneMuted(),
         localVideoMuted: groupCall.isLocalVideoMuted(),
         isScreensharing: groupCall.isScreensharing(),
-        localScreenshareFeed: groupCall.localScreenshareFeed,
         localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
         screenshareFeeds: [...groupCall.screenshareFeeds],
         participants: [...groupCall.participants],
@@ -172,12 +167,11 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
 
     function onLocalScreenshareStateChanged(
       isScreensharing: boolean,
-      localScreenshareFeed: CallFeed,
+      _localScreenshareFeed: CallFeed,
       localDesktopCapturerSourceId: string
     ): void {
       updateState({
         isScreensharing,
-        localScreenshareFeed,
         localDesktopCapturerSourceId,
       });
     }
@@ -228,7 +222,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
       microphoneMuted: groupCall.isMicrophoneMuted(),
       localVideoMuted: groupCall.isLocalVideoMuted(),
       isScreensharing: groupCall.isScreensharing(),
-      localScreenshareFeed: groupCall.localScreenshareFeed,
       localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
       screenshareFeeds: [...groupCall.screenshareFeeds],
       participants: [...groupCall.participants],
@@ -412,7 +405,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     requestingScreenshare,
     isScreensharing,
     screenshareFeeds,
-    localScreenshareFeed,
     localDesktopCapturerSourceId,
     participants,
     hasLocalParticipant,
diff --git a/src/video-grid/AudioContainer.tsx b/src/video-grid/AudioContainer.tsx
@@ -16,15 +16,15 @@ limitations under the License.
 
 import React, { FC, useEffect, useRef } from "react";
 
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 import { useCallFeed } from "./useCallFeed";
 import { useMediaStreamTrackCount } from "./useMediaStream";
 
 // XXX: These in fact do not render anything but to my knowledge this is the
 // only way to a hook on an array
 
 interface AudioForParticipantProps {
-  item: Participant;
+  item: TileDescriptor;
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }
@@ -78,7 +78,7 @@ export const AudioForParticipant: FC<AudioForParticipantProps> = ({
 };
 
 interface AudioContainerProps {
-  items: Participant[];
+  items: TileDescriptor[];
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }
diff --git a/src/video-grid/VideoGrid.stories.tsx b/src/video-grid/VideoGrid.stories.tsx
@@ -16,11 +16,12 @@ limitations under the License.
 
 import React, { useState } from "react";
 import { useMemo } from "react";
+import { RoomMember } from "matrix-js-sdk";
 
 import { VideoGrid, useVideoGridLayout } from "./VideoGrid";
 import { VideoTile } from "./VideoTile";
 import { Button } from "../button";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 export default {
   title: "VideoGrid",
@@ -33,10 +34,11 @@ export const ParticipantsTest = () => {
   const { layout, setLayout } = useVideoGridLayout(false);
   const [participantCount, setParticipantCount] = useState(1);
 
-  const items: Participant[] = useMemo(
+  const items: TileDescriptor[] = useMemo(
     () =>
       new Array(participantCount).fill(undefined).map((_, i) => ({
         id: (i + 1).toString(),
+        member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`),
         focused: false,
         presenter: false,
       })),
@@ -77,6 +79,7 @@ export const ParticipantsTest = () => {
               key={item.id}
               name={`User ${item.id}`}
               disableSpeakingIndicator={items.length < 3}
+              hasFeed={true}
               {...rest}
             />
           )}
diff --git a/src/video-grid/VideoGrid.tsx b/src/video-grid/VideoGrid.tsx
@@ -23,7 +23,7 @@ import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/typ
 
 import styles from "./VideoGrid.module.css";
 import { Layout } from "../room/GridLayoutMenu";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface TilePosition {
   x: number;
@@ -36,7 +36,7 @@ interface TilePosition {
 interface Tile {
   key: Key;
   order: number;
-  item: Participant;
+  item: TileDescriptor;
   remove: boolean;
   focused: boolean;
   presenter: boolean;
@@ -693,12 +693,12 @@ interface ChildrenProperties extends ReactDOMAttributes {
   };
   width: number;
   height: number;
-  item: Participant;
+  item: TileDescriptor;
   [index: string]: unknown;
 }
 
 interface VideoGridProps {
-  items: Participant[];
+  items: TileDescriptor[];
   layout: Layout;
   disableAnimations?: boolean;
   children: (props: ChildrenProperties) => React.ReactNode;
diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx
@@ -26,6 +26,7 @@ import { AudioButton, FullscreenButton } from "../button/Button";
 
 interface Props {
   name: string;
+  hasFeed: Boolean;
   speaking?: boolean;
   audioMuted?: boolean;
   videoMuted?: boolean;
@@ -47,6 +48,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
   (
     {
       name,
+      hasFeed,
       speaking,
       audioMuted,
       videoMuted,
@@ -90,6 +92,8 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       }
     }
 
+    const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name });
+
     return (
       <animated.div
         className={classNames(styles.videoTile, className, {
@@ -120,7 +124,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             <div className={classNames(styles.infoBubble, styles.memberName)}>
               {audioMuted && !videoMuted && <MicMutedIcon />}
               {videoMuted && <VideoMutedIcon />}
-              <span title={name}>{name}</span>
+              <span title={caption}>{caption}</span>
             </div>
           ))}
         <video ref={mediaRef} playsInline disablePictureInPicture />
diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx
diff --git a/src/video-grid/useCallFeed.ts b/src/video-grid/useCallFeed.ts
diff --git a/src/video-grid/useFullscreen.tsx b/src/video-grid/useFullscreen.tsx
diff --git a/src/video-grid/useMediaStream.ts b/src/video-grid/useMediaStream.ts
