Populate info.duration for audio & video file uploads (#11225)

* Improve m.file m.image m.audio m.video types

* Populate `info.duration` for audio & video file uploads

* Fix tests

* Iterate types

* Improve coverage

* Fix test

* Add small delay to stabilise cypress test

* Fix test idempotency

* Improve coverage

* Slow down

* iterate

Author: t3chguy

cypress/e2e/right-panel/file-panel.spec.ts

@@ -183,7 +183,7 @@ describe("FilePanel", () => {
             });
         });
 
-        it("should render the audio pleyer and play the audio file on the panel", () => {
+        it("should render the audio player and play the audio file on the panel", () => {
             // Upload an image file
             uploadFile("cypress/fixtures/1sec.ogg");
 
@@ -202,10 +202,14 @@ describe("FilePanel", () => {
                                 cy.contains(".mx_AudioPlayer_byline", "(3.56 KB)").should("exist"); // actual size
                             });
 
+                            // Assert that the duration counter is 00:01 before clicking the play button
+                            cy.contains(".mx_AudioPlayer_mediaInfo time", "00:01").should("exist");
+
                             // Assert that the counter is zero before clicking the play button
                             cy.contains(".mx_AudioPlayer_seek [role='timer']", "00:00").should("exist");
 
                             // Click the play button
+                            cy.wait(500);
                             cy.findByRole("button", { name: "Play" }).click();
 
                             // Assert that the pause button is rendered

src/ContentMessages.ts

@@ -33,7 +33,14 @@ import {
 import { THREAD_RELATION_TYPE } from "matrix-js-sdk/src/models/thread";
 import { removeElement } from "matrix-js-sdk/src/utils";
 
-import { IEncryptedFile, IMediaEventContent, IMediaEventInfo } from "./customisations/models/IMediaEventContent";
+import {
+    AudioInfo,
+    EncryptedFile,
+    ImageInfo,
+    IMediaEventContent,
+    IMediaEventInfo,
+    VideoInfo,
+} from "./customisations/models/IMediaEventContent";
 import dis from "./dispatcher/dispatcher";
 import { _t } from "./languageHandler";
 import Modal from "./Modal";
@@ -146,11 +153,7 @@ const ALWAYS_INCLUDE_THUMBNAIL = ["image/avif", "image/webp"];
  * @param {File} imageFile The image to read and thumbnail.
  * @return {Promise} A promise that resolves with the attachment info.
  */
-async function infoForImageFile(
-    matrixClient: MatrixClient,
-    roomId: string,
-    imageFile: File,
-): Promise<Partial<IMediaEventInfo>> {
+async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File): Promise<ImageInfo> {
     let thumbnailType = "image/png";
     if (imageFile.type === "image/jpeg") {
         thumbnailType = "image/jpeg";
@@ -184,16 +187,59 @@ async function infoForImageFile(
     return imageInfo;
 }
 
+/**
+ * Load a file into a newly created audio element and load the metadata
+ *
+ * @param {File} audioFile The file to load in an audio element.
+ * @return {Promise} A promise that resolves with the audio element.
+ */
+function loadAudioElement(audioFile: File): Promise<HTMLAudioElement> {
+    return new Promise((resolve, reject) => {
+        // Load the file into a html element
+        const audio = document.createElement("audio");
+        audio.preload = "metadata";
+        audio.muted = true;
+
+        const reader = new FileReader();
+
+        reader.onload = function (ev): void {
+            audio.onloadedmetadata = async function (): Promise<void> {
+                resolve(audio);
+            };
+            audio.onerror = function (e): void {
+                reject(e);
+            };
+
+            audio.src = ev.target?.result as string;
+        };
+        reader.onerror = function (e): void {
+            reject(e);
+        };
+        reader.readAsDataURL(audioFile);
+    });
+}
+
+/**
+ * Read the metadata for an audio file.
+ *
+ * @param {File} audioFile The audio to read.
+ * @return {Promise} A promise that resolves with the attachment info.
+ */
+async function infoForAudioFile(audioFile: File): Promise<AudioInfo> {
+    const audio = await loadAudioElement(audioFile);
+    return { duration: Math.ceil(audio.duration * 1000) };
+}
+
 /**
  * Load a file into a newly created video element and pull some strings
  * in an attempt to guarantee the first frame will be showing.
  *
- * @param {File} videoFile The file to load in an video element.
- * @return {Promise} A promise that resolves with the video image element.
+ * @param {File} videoFile The file to load in a video element.
+ * @return {Promise} A promise that resolves with the video element.
  */
 function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
     return new Promise((resolve, reject) => {
-        // Load the file into an html element
+        // Load the file into a html element
         const video = document.createElement("video");
         video.preload = "metadata";
         video.playsInline = true;
@@ -237,20 +283,17 @@ function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
  * @param {File} videoFile The video to read and thumbnail.
  * @return {Promise} A promise that resolves with the attachment info.
  */
-function infoForVideoFile(
-    matrixClient: MatrixClient,
-    roomId: string,
-    videoFile: File,
-): Promise<Partial<IMediaEventInfo>> {
+function infoForVideoFile(matrixClient: MatrixClient, roomId: string, videoFile: File): Promise<VideoInfo> {
     const thumbnailType = "image/jpeg";
 
-    let videoInfo: Partial<IMediaEventInfo>;
+    const videoInfo: VideoInfo = {};
     return loadVideoElement(videoFile)
         .then((video) => {
+            videoInfo.duration = Math.ceil(video.duration * 1000);
             return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
         })
         .then((result) => {
-            videoInfo = result.info;
+            Object.assign(videoInfo, result.info);
             return uploadFile(matrixClient, roomId, result.thumbnail);
         })
         .then((result) => {
@@ -299,7 +342,7 @@ export async function uploadFile(
     file: File | Blob,
     progressHandler?: UploadOpts["progressHandler"],
     controller?: AbortController,
-): Promise<{ url?: string; file?: IEncryptedFile }> {
+): Promise<{ url?: string; file?: EncryptedFile }> {
     const abortController = controller ?? new AbortController();
 
     // If the room is encrypted then encrypt the file before uploading it.
@@ -329,7 +372,7 @@ export async function uploadFile(
             file: {
                 ...encryptResult.info,
                 url,
-            } as IEncryptedFile,
+            } as EncryptedFile,
         };
     } else {
         const { content_uri: url } = await matrixClient.uploadContent(file, { progressHandler, abortController });
@@ -546,6 +589,14 @@ export default class ContentMessages {
                 }
             } else if (file.type.indexOf("audio/") === 0) {
                 content.msgtype = MsgType.Audio;
+                try {
+                    const audioInfo = await infoForAudioFile(file);
+                    Object.assign(content.info, audioInfo);
+                } catch (e) {
+                    // Failed to process audio file, fall back to uploading an m.file
+                    logger.error(e);
+                    content.msgtype = MsgType.File;
+                }
             } else if (file.type.indexOf("video/") === 0) {
                 content.msgtype = MsgType.Video;
                 try {

src/components/views/messages/MFileBody.tsx

@@ -198,7 +198,7 @@ export default class MFileBody extends React.Component<IProps, IState> {
         const isEncrypted = this.props.mediaEventHelper?.media.isEncrypted;
         const contentUrl = this.getContentUrl();
         const contentFileSize = this.content.info ? this.content.info.size : null;
-        const fileType = this.content.info ? this.content.info.mimetype : "application/octet-stream";
+        const fileType = this.content.info?.mimetype ?? "application/octet-stream";
 
         let placeholder: React.ReactNode = null;
         if (this.props.showGenericPlaceholder) {

src/components/views/messages/MImageBody.tsx

@@ -29,7 +29,7 @@ import SettingsStore from "../../../settings/SettingsStore";
 import Spinner from "../elements/Spinner";
 import { Media, mediaFromContent } from "../../../customisations/Media";
 import { BLURHASH_FIELD, createThumbnail } from "../../../utils/image-media";
-import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
+import { ImageContent } from "../../../customisations/models/IMediaEventContent";
 import ImageView from "../elements/ImageView";
 import { IBodyProps } from "./IBodyProps";
 import { ImageSize, suggestedSize as suggestedImageSize } from "../../../settings/enums/ImageSize";
@@ -102,7 +102,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
                 return;
             }
 
-            const content = this.props.mxEvent.getContent<IMediaEventContent>();
+            const content = this.props.mxEvent.getContent<ImageContent>();
             const httpUrl = this.state.contentUrl;
             if (!httpUrl) return;
             const params: Omit<ComponentProps<typeof ImageView>, "onFinished"> = {
@@ -212,7 +212,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
         const thumbWidth = 800;
         const thumbHeight = 600;
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         const media = mediaFromContent(content);
         const info = content.info;
 
@@ -287,7 +287,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
             contentUrl = this.getContentUrl();
         }
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         let isAnimated = mayBeAnimated(content.info?.mimetype);
 
         // If there is no included non-animated thumbnail then we will generate our own, we can't depend on the server
@@ -317,7 +317,13 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
                     }
 
                     if (isAnimated) {
-                        const thumb = await createThumbnail(img, img.width, img.height, content.info!.mimetype, false);
+                        const thumb = await createThumbnail(
+                            img,
+                            img.width,
+                            img.height,
+                            content.info?.mimetype ?? "image/jpeg",
+                            false,
+                        );
                         thumbUrl = URL.createObjectURL(thumb.thumbnail);
                     }
                 } catch (error) {
@@ -381,7 +387,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
         }
     }
 
-    protected getBanner(content: IMediaEventContent): ReactNode {
+    protected getBanner(content: ImageContent): ReactNode {
         // Hide it for the threads list & the file panel where we show it as text anyway.
         if (
             [TimelineRenderingType.ThreadsList, TimelineRenderingType.File].includes(this.context.timelineRenderingType)
@@ -395,7 +401,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
     protected messageContent(
         contentUrl: string | null,
         thumbUrl: string | null,
-        content: IMediaEventContent,
+        content: ImageContent,
         forcedHeight?: number,
     ): ReactNode {
         if (!thumbUrl) thumbUrl = contentUrl; // fallback
@@ -591,7 +597,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
     }
 
     public render(): React.ReactNode {
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
 
         if (this.state.error) {
             let errorText = _t("Unable to show image due to error");

src/components/views/messages/MImageReplyBody.tsx

@@ -17,7 +17,7 @@ limitations under the License.
 import React from "react";
 
 import MImageBody from "./MImageBody";
-import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
+import { ImageContent } from "../../../customisations/models/IMediaEventContent";
 
 const FORCED_IMAGE_HEIGHT = 44;
 
@@ -35,7 +35,7 @@ export default class MImageReplyBody extends MImageBody {
             return super.render();
         }
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         const thumbnail = this.state.contentUrl
             ? this.messageContent(this.state.contentUrl, this.state.thumbUrl, content, FORCED_IMAGE_HEIGHT)
             : undefined;