Populate info.duration for audio & video file uploads

src/ContentMessages.ts

@@ -33,7 +33,14 @@ import {
 import { THREAD_RELATION_TYPE } from "matrix-js-sdk/src/models/thread";
 import { removeElement } from "matrix-js-sdk/src/utils";
 
-import { IEncryptedFile, IMediaEventContent, IMediaEventInfo } from "./customisations/models/IMediaEventContent";
+import {
+    AudioInfo,
+    IEncryptedFile,
+    ImageInfo,
+    IMediaEventContent,
+    IMediaEventInfo,
+    VideoInfo,
+} from "./customisations/models/IMediaEventContent";
 import dis from "./dispatcher/dispatcher";
 import { _t } from "./languageHandler";
 import Modal from "./Modal";
@@ -146,11 +153,7 @@ const ALWAYS_INCLUDE_THUMBNAIL = ["image/avif", "image/webp"];
  * @param {File} imageFile The image to read and thumbnail.
  * @return {Promise} A promise that resolves with the attachment info.
  */
-async function infoForImageFile(
-    matrixClient: MatrixClient,
-    roomId: string,
-    imageFile: File,
-): Promise<Partial<IMediaEventInfo>> {
+async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File): Promise<ImageInfo> {
     let thumbnailType = "image/png";
     if (imageFile.type === "image/jpeg") {
         thumbnailType = "image/jpeg";
@@ -184,16 +187,59 @@ async function infoForImageFile(
     return imageInfo;
 }
 
+/**
+ * Load a file into a newly created audio element and load the metadata
+ *
+ * @param {File} audioFile The file to load in an audio element.
+ * @return {Promise} A promise that resolves with the audio element.
+ */
+function loadAudioElement(audioFile: File): Promise<HTMLAudioElement> {
+    return new Promise((resolve, reject) => {
+        // Load the file into a html element
+        const audio = document.createElement("audio");
+        audio.preload = "metadata";
+        audio.muted = true;
+
+        const reader = new FileReader();
+
+        reader.onload = function (ev): void {
+            audio.onloadedmetadata = async function (): Promise<void> {
+                resolve(audio);
+            };
+            audio.onerror = function (e): void {
+                reject(e);
+            };
+
+            audio.src = ev.target?.result as string;
+        };
+        reader.onerror = function (e): void {
+            reject(e);
+        };
+        reader.readAsDataURL(audioFile);
+    });
+}
+
+/**
+ * Read the metadata for an audio file.
+ *
+ * @param {File} audioFile The audio to read.
+ * @return {Promise} A promise that resolves with the attachment info.
+ */
+async function infoForAudioFile(audioFile: File): Promise<AudioInfo> {
+    const audio = await loadAudioElement(audioFile);
+    return { duration: Math.ceil(audio.duration * 1000) };
+}
+
 /**
  * Load a file into a newly created video element and pull some strings
  * in an attempt to guarantee the first frame will be showing.
  *
- * @param {File} videoFile The file to load in an video element.
- * @return {Promise} A promise that resolves with the video image element.
+ * @param {File} videoFile The file to load in a video element.
+ * @return {Promise} A promise that resolves with the video element.
  */
 function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
     return new Promise((resolve, reject) => {
-        // Load the file into an html element
+        // Load the file into a html element
         const video = document.createElement("video");
         video.preload = "metadata";
         video.playsInline = true;
@@ -237,20 +283,17 @@ function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
  * @param {File} videoFile The video to read and thumbnail.
  * @return {Promise} A promise that resolves with the attachment info.
  */
-function infoForVideoFile(
-    matrixClient: MatrixClient,
-    roomId: string,
-    videoFile: File,
-): Promise<Partial<IMediaEventInfo>> {
+function infoForVideoFile(matrixClient: MatrixClient, roomId: string, videoFile: File): Promise<VideoInfo> {
     const thumbnailType = "image/jpeg";
 
-    let videoInfo: Partial<IMediaEventInfo>;
+    const videoInfo: VideoInfo = {};
     return loadVideoElement(videoFile)
         .then((video) => {
+            videoInfo.duration = Math.ceil(video.duration * 1000);
             return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
         })
         .then((result) => {
-            videoInfo = result.info;
+            Object.assign(videoInfo, result.info);
             return uploadFile(matrixClient, roomId, result.thumbnail);
         })
         .then((result) => {
@@ -546,6 +589,14 @@ export default class ContentMessages {
                 }
             } else if (file.type.indexOf("audio/") === 0) {
                 content.msgtype = MsgType.Audio;
+                try {
+                    const audioInfo = await infoForAudioFile(file);
+                    Object.assign(content.info, audioInfo);
+                } catch (e) {
+                    // Failed to process audio file, fall back to uploading an m.file
+                    logger.error(e);
+                    content.msgtype = MsgType.File;
+                }
             } else if (file.type.indexOf("video/") === 0) {
                 content.msgtype = MsgType.Video;
                 try {

src/components/views/messages/MFileBody.tsx

@@ -198,7 +198,7 @@ export default class MFileBody extends React.Component<IProps, IState> {
         const isEncrypted = this.props.mediaEventHelper?.media.isEncrypted;
         const contentUrl = this.getContentUrl();
         const contentFileSize = this.content.info ? this.content.info.size : null;
-        const fileType = this.content.info ? this.content.info.mimetype : "application/octet-stream";
+        const fileType = this.content.info?.mimetype ?? "application/octet-stream";
 
         let placeholder: React.ReactNode = null;
         if (this.props.showGenericPlaceholder) {

src/components/views/messages/MImageBody.tsx

@@ -29,7 +29,7 @@ import SettingsStore from "../../../settings/SettingsStore";
 import Spinner from "../elements/Spinner";
 import { Media, mediaFromContent } from "../../../customisations/Media";
 import { BLURHASH_FIELD, createThumbnail } from "../../../utils/image-media";
-import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
+import { ImageContent } from "../../../customisations/models/IMediaEventContent";
 import ImageView from "../elements/ImageView";
 import { IBodyProps } from "./IBodyProps";
 import { ImageSize, suggestedSize as suggestedImageSize } from "../../../settings/enums/ImageSize";
@@ -102,7 +102,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
                 return;
             }
 
-            const content = this.props.mxEvent.getContent<IMediaEventContent>();
+            const content = this.props.mxEvent.getContent<ImageContent>();
             const httpUrl = this.state.contentUrl;
             if (!httpUrl) return;
             const params: Omit<ComponentProps<typeof ImageView>, "onFinished"> = {
@@ -212,7 +212,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
         const thumbWidth = 800;
         const thumbHeight = 600;
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         const media = mediaFromContent(content);
         const info = content.info;
 
@@ -287,7 +287,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
             contentUrl = this.getContentUrl();
         }
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         let isAnimated = mayBeAnimated(content.info?.mimetype);
 
         // If there is no included non-animated thumbnail then we will generate our own, we can't depend on the server
@@ -317,7 +317,13 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
                     }
 
                     if (isAnimated) {
-                        const thumb = await createThumbnail(img, img.width, img.height, content.info!.mimetype, false);
+                        const thumb = await createThumbnail(
+                            img,
+                            img.width,
+                            img.height,
+                            content.info?.mimetype ?? "image/jpeg",
+                            false,
+                        );
                         thumbUrl = URL.createObjectURL(thumb.thumbnail);
                     }
                 } catch (error) {
@@ -381,7 +387,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
         }
     }
 
-    protected getBanner(content: IMediaEventContent): ReactNode {
+    protected getBanner(content: ImageContent): ReactNode {
         // Hide it for the threads list & the file panel where we show it as text anyway.
         if (
             [TimelineRenderingType.ThreadsList, TimelineRenderingType.File].includes(this.context.timelineRenderingType)
@@ -395,7 +401,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
     protected messageContent(
         contentUrl: string | null,
         thumbUrl: string | null,
-        content: IMediaEventContent,
+        content: ImageContent,
         forcedHeight?: number,
     ): ReactNode {
         if (!thumbUrl) thumbUrl = contentUrl; // fallback
@@ -591,7 +597,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
     }
 
     public render(): React.ReactNode {
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
 
         if (this.state.error) {
             let errorText = _t("Unable to show image due to error");

src/components/views/messages/MImageReplyBody.tsx

@@ -17,7 +17,7 @@ limitations under the License.
 import React from "react";
 
 import MImageBody from "./MImageBody";
-import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
+import { ImageContent } from "../../../customisations/models/IMediaEventContent";
 
 const FORCED_IMAGE_HEIGHT = 44;
 
@@ -35,7 +35,7 @@ export default class MImageReplyBody extends MImageBody {
             return super.render();
         }
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         const thumbnail = this.state.contentUrl
             ? this.messageContent(this.state.contentUrl, this.state.thumbUrl, content, FORCED_IMAGE_HEIGHT)
             : undefined;

src/customisations/models/IMediaEventContent.ts

@@ -16,6 +16,10 @@
 
 // TODO: These types should be elsewhere.
 
+import { MsgType } from "matrix-js-sdk/src/matrix";
+
+import { BLURHASH_FIELD } from "../../utils/image-media";
+
 export interface IEncryptedFile {
     url: string;
     key: {
@@ -30,31 +34,67 @@ export interface IEncryptedFile {
     v: string;
 }
 
-export interface IMediaEventInfo {
-    thumbnail_url?: string; // eslint-disable-line camelcase
-    thumbnail_file?: IEncryptedFile; // eslint-disable-line camelcase
-    thumbnail_info?: {
-        // eslint-disable-line camelcase
-        mimetype: string;
-        w?: number;
-        h?: number;
-        size?: number;
-    };
-    mimetype: string;
+interface ThumbnailInfo {
+    mimetype?: string;
     w?: number;
     h?: number;
     size?: number;
 }
 
-export interface IMediaEventContent {
-    msgtype: string;
-    body?: string;
-    filename?: string; // `m.file` optional field
-    url?: string; // required on unencrypted media
-    file?: IEncryptedFile; // required for *encrypted* media
-    info?: IMediaEventInfo;
+interface BaseInfo {
+    mimetype?: string;
+    size?: number;
+}
+
+export interface FileInfo extends BaseInfo {
+    [BLURHASH_FIELD]?: string;
+    thumbnail_file?: IEncryptedFile;
+    thumbnail_info?: ThumbnailInfo;
+    thumbnail_url?: string;
+}
+
+export interface ImageInfo extends FileInfo, ThumbnailInfo {}
+
+export interface AudioInfo extends BaseInfo {
+    duration?: number;
+}
+
+export interface VideoInfo extends AudioInfo, ImageInfo {}
+
+export type IMediaEventInfo = FileInfo | ImageInfo | AudioInfo | VideoInfo;
+
+interface BaseContent {
+    body: string;
 }
 
+interface BaseFileContent extends BaseContent {
+    file?: IEncryptedFile;
+    url?: string;
+}
+
+export interface FileContent extends BaseFileContent {
+    filename?: string;
+    info?: FileInfo;
+    msgtype: MsgType.File;
+}
+
+export interface ImageContent extends BaseFileContent {
+    info?: ImageInfo;
+    msgtype: MsgType.Image;
+}
+
+export interface AudioContent extends BaseFileContent {
+    info?: AudioInfo;
+    msgtype: MsgType.Audio;
+}
+
+export interface VideoContent extends BaseFileContent {
+    info?: VideoInfo;
+    msgtype: MsgType.Video;
+}
+
+export type IMediaEventContent = FileContent | ImageContent | AudioContent | VideoContent;
+
 export interface IPreparedMedia extends IMediaObject {
     thumbnail?: IMediaObject;
 }
@@ -73,12 +113,17 @@ export interface IMediaObject {
  */
 export function prepEventContentAsMedia(content: Partial<IMediaEventContent>): IPreparedMedia {
     let thumbnail: IMediaObject | undefined;
-    if (content?.info?.thumbnail_url) {
+    if (typeof content?.info === "object" && "thumbnail_url" in content.info && content.info.thumbnail_url) {
         thumbnail = {
             mxc: content.info.thumbnail_url,
             file: content.info.thumbnail_file,
         };
-    } else if (content?.info?.thumbnail_file?.url) {
+    } else if (
+        typeof content?.info === "object" &&
+        "thumbnail_file" in content.info &&
+        typeof content?.info?.thumbnail_file === "object" &&
+        content?.info?.thumbnail_file?.url
+    ) {
         thumbnail = {
             mxc: content.info.thumbnail_file.url,
             file: content.info.thumbnail_file,

src/utils/MediaEventHelper.ts

@@ -21,7 +21,7 @@ import { logger } from "matrix-js-sdk/src/logger";
 import { LazyValue } from "./LazyValue";
 import { Media, mediaFromContent } from "../customisations/Media";
 import { decryptFile } from "./DecryptFile";
-import { IMediaEventContent } from "../customisations/models/IMediaEventContent";
+import { FileContent, ImageContent, IMediaEventContent } from "../customisations/models/IMediaEventContent";
 import { IDestroyable } from "./IDestroyable";
 
 // TODO: We should consider caching the blobs. https://github.com/vector-im/element-web/issues/17192
@@ -48,7 +48,7 @@ export class MediaEventHelper implements IDestroyable {
 
     public get fileName(): string {
         return (
-            this.event.getContent<IMediaEventContent>().filename ||
+            this.event.getContent<FileContent>().filename ||
             this.event.getContent<IMediaEventContent>().body ||
             "download"
         );
@@ -92,7 +92,7 @@ export class MediaEventHelper implements IDestroyable {
         if (!this.media.hasThumbnail) return Promise.resolve(null);
 
         if (this.media.isEncrypted) {
-            const content = this.event.getContent<IMediaEventContent>();
+            const content = this.event.getContent<ImageContent>();
             if (content.info?.thumbnail_file) {
                 return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info);
             } else {