diff --git a/package-lock.json b/package-lock.json index fedd0d066c6..21ad7b77725 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,8 @@ "date-fns": "^2.29.3", "dotenv": "^16.0.3", "express": "^4.21.2", + "extendable-media-recorder": "^9.2.19", + "extendable-media-recorder-wav-encoder": "^7.0.123", "file-type": "^19.4.1", "google-auth-library": "^9.13.0", "handlebars": "^4.7.8", @@ -55,6 +57,7 @@ "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^10.0.0", + "wavefile": "^11.0.0", "zod": "^3.22.3" }, "devDependencies": { @@ -1320,9 +1323,10 @@ } }, "node_modules/@babel/runtime": { - "version": "7.25.6", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.25.6.tgz", - "integrity": "sha512-VBj9MYyDb9tuLq7yzqjgzt6Q+IBQLrGZfdjOekyEirZPHxXWoTSGUTMrpsfi58Up73d13NfYLv8HT9vmznjzhQ==", + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.7.tgz", + "integrity": "sha512-AOPI3D+a8dXnja+iwsUqGRjr1BbZIe771sXdapOtYI531gSqpi92vXivKcq2asu/DFpdl1ceFAKZyRzK2PCVcQ==", + "license": "MIT", "dependencies": { "regenerator-runtime": "^0.14.0" }, @@ -5011,6 +5015,19 @@ "node": ">=8.0.0" } }, + "node_modules/automation-events": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/automation-events/-/automation-events-7.1.5.tgz", + "integrity": "sha512-1t/XcUE/nhHtAyePNZHQMU3/Ka9E0CB1aA8FpzSwc1EBHMsZBtX6H5sBNngCL7pmlxoaDrpcwwHE7Kd6lkYoWA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=18.2.0" + } + }, "node_modules/autoprefixer": { "version": "10.4.20", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.20.tgz", @@ -5243,6 +5260,18 @@ "node": ">=8" } }, + "node_modules/broker-factory": { + "version": "3.0.109", + "resolved": "https://registry.npmjs.org/broker-factory/-/broker-factory-3.0.109.tgz", + "integrity": "sha512-yl0S910a6rStKy5Pb4VxtpgyrY0vWoSoXRnAou22jUlK0ikakBSeIh1M4wSl3PgZ8BvYQ9Vt2o3L7hIp9738nw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "fast-unique-numbers": "^9.0.15", + "tslib": "^2.8.1", + "worker-factory": "^7.0.36" + } + }, "node_modules/brotli": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", @@ -6884,6 +6913,56 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "node_modules/extendable-media-recorder": { + "version": "9.2.19", + "resolved": "https://registry.npmjs.org/extendable-media-recorder/-/extendable-media-recorder-9.2.19.tgz", + "integrity": "sha512-7/YoyoqGGaphJkP6NgxgrDBAE4EfJF6CODwL0siKvJnQZ6YMYbUMdCS5sNRfIDqcnd5oR7I6EpsJHvLl+7R4AQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "media-encoder-host": "^9.0.14", + "multi-buffer-data-view": "^6.0.17", + "recorder-audio-worklet": "^6.0.41", + "standardized-audio-context": "^25.3.77", + "subscribable-things": "^2.1.47", + "tslib": "^2.8.1" + } + }, + "node_modules/extendable-media-recorder-wav-encoder": { + "version": "7.0.123", + "resolved": "https://registry.npmjs.org/extendable-media-recorder-wav-encoder/-/extendable-media-recorder-wav-encoder-7.0.123.tgz", + "integrity": "sha512-tQFU8UD0HiQNI4F7dzPXR4gskXijpn9dv/wJUH0l+CGCZN4Q8WcEYJPI8CYxHqGOeMaFzjvBlqkkSIFZuEeVMg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "extendable-media-recorder-wav-encoder-broker": "^7.0.113", + "extendable-media-recorder-wav-encoder-worker": "^8.0.110", + "tslib": "^2.8.1" + } + }, + "node_modules/extendable-media-recorder-wav-encoder-broker": { + "version": "7.0.113", + "resolved": "https://registry.npmjs.org/extendable-media-recorder-wav-encoder-broker/-/extendable-media-recorder-wav-encoder-broker-7.0.113.tgz", + "integrity": "sha512-KkgSfoG7UKxK9tXQa4d/AsDf92bY2FTbg9H6GWMbbOA3IRuChTd70hBwPAk/WNsqPfGUxgJ8Swdq0PX43q9+mg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "broker-factory": "^3.0.109", + "extendable-media-recorder-wav-encoder-worker": "^8.0.110", + "tslib": "^2.8.1" + } + }, + "node_modules/extendable-media-recorder-wav-encoder-worker": { + "version": "8.0.110", + "resolved": "https://registry.npmjs.org/extendable-media-recorder-wav-encoder-worker/-/extendable-media-recorder-wav-encoder-worker-8.0.110.tgz", + "integrity": "sha512-EvpJglMZuqkOu7HErjvV7hn9BtDSuo2PKaw8cw3oWqzXhKn8NdhX5hkqTpSWuXObfM1bnxxsUYeenG5Yd6z2Ew==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "tslib": "^2.8.1", + "worker-factory": "^7.0.36" + } + }, "node_modules/fast-copy": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz", @@ -6952,6 +7031,19 @@ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, + "node_modules/fast-unique-numbers": { + "version": "9.0.15", + "resolved": "https://registry.npmjs.org/fast-unique-numbers/-/fast-unique-numbers-9.0.15.tgz", + "integrity": "sha512-vHj0sfq6yB37b/RAAsAJ2DzIp0LR5NlUit7nYFp2YfTUcKL9m/Yk0f0kvYPV4oiuFYXdtO5scs3LQX7qiPAVYQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=18.2.0" + } + }, "node_modules/fast-xml-parser": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", @@ -8857,6 +8949,43 @@ "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==", "dev": true }, + "node_modules/media-encoder-host": { + "version": "9.0.14", + "resolved": "https://registry.npmjs.org/media-encoder-host/-/media-encoder-host-9.0.14.tgz", + "integrity": "sha512-paFcVqASpwhpOhvopkzNBRtNd7BQtZic8Eq0GggyImTMcN6xXS0ixJ5egEOEVrHHws/fAoRH0j39ddX5EWQznw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "media-encoder-host-broker": "^8.0.13", + "media-encoder-host-worker": "^10.0.13", + "tslib": "^2.8.1" + } + }, + "node_modules/media-encoder-host-broker": { + "version": "8.0.13", + "resolved": "https://registry.npmjs.org/media-encoder-host-broker/-/media-encoder-host-broker-8.0.13.tgz", + "integrity": "sha512-mcbATyu/NbebqNQYQypcwuDJ3sCZf3EQtlUMm4viZQ4aMpNy85mte/E3WGzA+VZNKdWR7IavmvmhO2lgJWpYQA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "broker-factory": "^3.0.109", + "fast-unique-numbers": "^9.0.15", + "media-encoder-host-worker": "^10.0.13", + "tslib": "^2.8.1" + } + }, + "node_modules/media-encoder-host-worker": { + "version": "10.0.13", + "resolved": "https://registry.npmjs.org/media-encoder-host-worker/-/media-encoder-host-worker-10.0.13.tgz", + "integrity": "sha512-Yg1+xi8203jcpWYj7/o1JTy0k4MBTpZGo9tG4EwEb/Est1iWXhdKhVYjN4Uj/E3CCgrXr8WNO5ih47cWRSSWtw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "extendable-media-recorder-wav-encoder-broker": "^7.0.113", + "tslib": "^2.8.1", + "worker-factory": "^7.0.36" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -9422,6 +9551,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/multi-buffer-data-view": { + "version": "6.0.17", + "resolved": "https://registry.npmjs.org/multi-buffer-data-view/-/multi-buffer-data-view-6.0.17.tgz", + "integrity": "sha512-L76daSLXwMS1T8uL6RUmYyVL3L5hGor5UwDoTqFBYwlnvhtfpvVS6lu5oePrXi0B+X2JKTXZxBerqqb4Q7fM+g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=18.2.0" + } + }, "node_modules/mute-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", @@ -10837,6 +10979,32 @@ "node": ">= 12.13.0" } }, + "node_modules/recorder-audio-worklet": { + "version": "6.0.41", + "resolved": "https://registry.npmjs.org/recorder-audio-worklet/-/recorder-audio-worklet-6.0.41.tgz", + "integrity": "sha512-sCgsdBgAMlD5dbJ6NRFOyCZsYuS7y7Twg/p8Eff8gy9FOZIxFiYdxu/s/1oLuMiTkSsR5K9ejPSN5vthYGtSFQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "broker-factory": "^3.0.109", + "fast-unique-numbers": "^9.0.15", + "recorder-audio-worklet-processor": "^5.0.30", + "standardized-audio-context": "^25.3.77", + "subscribable-things": "^2.1.47", + "tslib": "^2.8.1", + "worker-factory": "^7.0.36" + } + }, + "node_modules/recorder-audio-worklet-processor": { + "version": "5.0.30", + "resolved": "https://registry.npmjs.org/recorder-audio-worklet-processor/-/recorder-audio-worklet-processor-5.0.30.tgz", + "integrity": "sha512-HL6ySdEbD6f909MLG5S7qg2gZBzxxHI9F/P5G9dMEDm3mejzZQx9zvP+Uw0SCIBmz+mztOQOkxCF3tc6FLCe3w==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "tslib": "^2.8.1" + } + }, "node_modules/regenerator-runtime": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", @@ -11044,6 +11212,12 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rxjs-interop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/rxjs-interop/-/rxjs-interop-2.0.0.tgz", + "integrity": "sha512-ASEq9atUw7lualXB+knvgtvwkCEvGWV2gDD/8qnASzBkzEARZck9JAyxmY8OS6Nc1pCPEgDTKNcx+YqqYfzArw==", + "license": "MIT" + }, "node_modules/sade": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", @@ -11585,6 +11759,17 @@ "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", "dev": true }, + "node_modules/standardized-audio-context": { + "version": "25.3.77", + "resolved": "https://registry.npmjs.org/standardized-audio-context/-/standardized-audio-context-25.3.77.tgz", + "integrity": "sha512-Ki9zNz6pKcC5Pi+QPjPyVsD9GwJIJWgryji0XL9cAJXMGyn+dPOf6Qik1AHei0+UNVcc4BOCa0hWLBzlwqsW/A==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.25.6", + "automation-events": "^7.0.9", + "tslib": "^2.7.0" + } + }, "node_modules/static-eval": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.0.2.tgz", @@ -11800,6 +11985,17 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/subscribable-things": { + "version": "2.1.47", + "resolved": "https://registry.npmjs.org/subscribable-things/-/subscribable-things-2.1.47.tgz", + "integrity": "sha512-LEBz6py2ZombbDAziKv9mBI5n/CFb4kZ2KGyLwoazB/G91x+7s1st/rE1YAIPj/kjCEPmry9BjnRQ7NCBcq85A==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "rxjs-interop": "^2.0.0", + "tslib": "^2.8.1" + } + }, "node_modules/sucrase": { "version": "3.35.0", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", @@ -12468,9 +12664,10 @@ "devOptional": true }, "node_modules/tslib": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", - "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" }, "node_modules/type-check": { "version": "0.4.0", @@ -12971,6 +13168,18 @@ "node": ">=14" } }, + "node_modules/wavefile": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz", + "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==", + "license": "MIT", + "bin": { + "wavefile": "bin/wavefile.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/web-streams-polyfill": { "version": "4.0.0-beta.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", @@ -13079,6 +13288,17 @@ "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==" }, + "node_modules/worker-factory": { + "version": "7.0.36", + "resolved": "https://registry.npmjs.org/worker-factory/-/worker-factory-7.0.36.tgz", + "integrity": "sha512-gSYPHZRUBXUkON4LSyA7/GAVmBuJDU1WeJhFFp+ZmlxNKInWCvxG1UAS5KnwcayTfGHGVEvM1BnfgmUu8OtpnA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "fast-unique-numbers": "^9.0.15", + "tslib": "^2.8.1" + } + }, "node_modules/wrap-ansi": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", diff --git a/package.json b/package.json index 38fe6c3b834..8edf3535325 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,8 @@ "date-fns": "^2.29.3", "dotenv": "^16.0.3", "express": "^4.21.2", + "extendable-media-recorder": "^9.2.19", + "extendable-media-recorder-wav-encoder": "^7.0.123", "file-type": "^19.4.1", "google-auth-library": "^9.13.0", "handlebars": "^4.7.8", @@ -108,6 +110,7 @@ "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^10.0.0", + "wavefile": "^11.0.0", "zod": "^3.22.3" }, "optionalDependencies": { diff --git a/src/lib/components/chat/ChatTranscribe.svelte b/src/lib/components/chat/ChatTranscribe.svelte new file mode 100644 index 00000000000..08d67439d38 --- /dev/null +++ b/src/lib/components/chat/ChatTranscribe.svelte @@ -0,0 +1,110 @@ + + +{#if recording} +
+ {lastWordFromValue} + +
+{/if} +
+ +
diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index 761fb8b2dac..9e1c256e2d2 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -35,6 +35,8 @@ import { cubicInOut } from "svelte/easing"; import type { ToolFront } from "$lib/types/Tool"; import { loginModalOpen } from "$lib/stores/loginModal"; + import ChatTranscribe from "./ChatTranscribe.svelte"; + import { isWebGPUSupported } from "$lib/utils/isWebGPUSupported"; export let messages: Message[] = []; export let loading = false; @@ -440,7 +442,11 @@ modelIsMultimodal={currentModel.multimodal} /> {/if} - + {#await isWebGPUSupported() then isWebGPUSupported} + {#if browser && isWebGPUSupported} + + {/if} + {/await} {#if loading}