multithreading for chrome and safari

Author: mtasic85

Diff for: README.md

@@ -18,12 +18,12 @@ cd llama-cpp-wasm
 ./build-multi-thread.sh
 ```
 
-Once build is complete you can find `llama.cpp` built in `docs/llama-st` and `docs/llama-mt` directory.
+Once build is complete you can find `llama.cpp` built in `dist/llama-st` and `dist/llama-mt` directory.
 
 
 ## Deploy
 
-Basically, you can copy/paste `docs/llama-st` or `docs/llama-mt` directory after build to your project and use as vanilla JavaScript library/module.
+Basically, you can copy/paste `dist/llama-st` or `dist/llama-mt` directory after build to your project and use as vanilla JavaScript library/module.
 
 
 **index.html**

Diff for: build-multi-thread.sh

@@ -3,6 +3,7 @@ set -e
 
 LLAMA_CPP_WASM_BUILD_DIR=build
 LLAMA_CPP_WASM_DIST_DIR=dist
+LLAMA_CPP_WASM_DIST_LLAMA_DIR=$LLAMA_CPP_WASM_DIST_DIR/llama-mt
 LLAMA_CPP_GIT_HASH="8c933b7"
 LLAMA_CPP_SOURCE_DIR=$LLAMA_CPP_WASM_BUILD_DIR/llama.cpp
 LLAMA_CPP_BUILD_DIR=$LLAMA_CPP_WASM_BUILD_DIR/build
@@ -27,20 +28,20 @@ cd $LLAMA_CPP_BUILD_DIR
 emcc --clear-cache
 emcmake cmake ../../$LLAMA_CPP_SOURCE_DIR
 # export EMCC_CFLAGS="-O3 -pthread -DNDEBUG -flto -s SHARED_MEMORY=1 -s EXPORT_ALL=1 -s EXPORT_ES6=1 -s MODULARIZE=1 -s INITIAL_MEMORY=2GB -s MAXIMUM_MEMORY=4GB -s ALLOW_MEMORY_GROWTH -s FORCE_FILESYSTEM=1 -s EXPORTED_FUNCTIONS=_main -s EXPORTED_RUNTIME_METHODS=callMain -s NO_EXIT_RUNTIME=1"
-export EMCC_CFLAGS="-O3 -pthread -DNDEBUG -flto -s SHARED_MEMORY=1 -s EXPORT_ALL=1 -s EXPORT_ES6=1 -s MODULARIZE=1 -s INITIAL_MEMORY=800MB -s MAXIMUM_MEMORY=4GB -s ALLOW_MEMORY_GROWTH -s FORCE_FILESYSTEM=1 -s EXPORTED_FUNCTIONS=_main -s EXPORTED_RUNTIME_METHODS=callMain -s NO_EXIT_RUNTIME=1"
+export EMCC_CFLAGS="-O3 -msimd128 -pthread -fno-rtti -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=32 -DNDEBUG -flto=full -s SHARED_MEMORY=1 -s EXPORT_ALL=1 -s EXPORT_ES6=1 -s MODULARIZE=1 -s INITIAL_MEMORY=800MB -s MAXIMUM_MEMORY=4GB -s ALLOW_MEMORY_GROWTH -s FORCE_FILESYSTEM=1 -s EXPORTED_FUNCTIONS=_main -s EXPORTED_RUNTIME_METHODS=callMain -s NO_EXIT_RUNTIME=1"
 emmake make main -j
 cd ../..
 
 #
 # bundle llama-cpp-wasm dist
 #
-if [ -d $LLAMA_CPP_WASM_DIST_DIR ]; then
-    rm -rf $LLAMA_CPP_WASM_DIST_DIR
+if [ -d $LLAMA_CPP_WASM_DIST_LLAMA_DIR ]; then
+    rm -rf $LLAMA_CPP_WASM_DIST_LLAMA_DIR
 fi
 
-mkdir -p $LLAMA_CPP_WASM_DIST_DIR
-cp -r src/* $LLAMA_CPP_WASM_DIST_DIR
-cp $LLAMA_CPP_BUILD_DIR/bin/main.* $LLAMA_CPP_WASM_DIST_DIR/llama
+mkdir -p $LLAMA_CPP_WASM_DIST_LLAMA_DIR
+cp -rv src/llama/* $LLAMA_CPP_WASM_DIST_LLAMA_DIR
+cp $LLAMA_CPP_BUILD_DIR/bin/main.* $LLAMA_CPP_WASM_DIST_LLAMA_DIR
 
 rm -rf docs/llama-mt
-mv $LLAMA_CPP_WASM_DIST_DIR/llama docs/llama-mt
+cp -rv $LLAMA_CPP_WASM_DIST_LLAMA_DIR docs/

Diff for: build-single-thread.sh

@@ -3,6 +3,7 @@ set -e
 
 LLAMA_CPP_WASM_BUILD_DIR=build
 LLAMA_CPP_WASM_DIST_DIR=dist
+LLAMA_CPP_WASM_DIST_LLAMA_DIR=$LLAMA_CPP_WASM_DIST_DIR/llama-st
 LLAMA_CPP_GIT_HASH="8c933b7"
 LLAMA_CPP_SOURCE_DIR=$LLAMA_CPP_WASM_BUILD_DIR/llama.cpp
 LLAMA_CPP_BUILD_DIR=$LLAMA_CPP_WASM_BUILD_DIR/build
@@ -27,20 +28,20 @@ cd $LLAMA_CPP_BUILD_DIR
 emcc --clear-cache
 emcmake cmake ../../$LLAMA_CPP_SOURCE_DIR
 # export EMCC_CFLAGS="-O3 -DNDEBUG -flto -s BUILD_AS_WORKER=1 -s EXPORT_ALL=1 -s EXPORT_ES6=1 -s MODULARIZE=1 -s INITIAL_MEMORY=2GB -s MAXIMUM_MEMORY=4GB -s ALLOW_MEMORY_GROWTH -s FORCE_FILESYSTEM=1 -s EXPORTED_FUNCTIONS=_main -s EXPORTED_RUNTIME_METHODS=callMain -s NO_EXIT_RUNTIME=1"
-export EMCC_CFLAGS="-O3 -DNDEBUG -flto -s BUILD_AS_WORKER=1 -s EXPORT_ALL=1 -s EXPORT_ES6=1 -s MODULARIZE=1 -s INITIAL_MEMORY=800MB -s MAXIMUM_MEMORY=4GB -s ALLOW_MEMORY_GROWTH -s FORCE_FILESYSTEM=1 -s EXPORTED_FUNCTIONS=_main -s EXPORTED_RUNTIME_METHODS=callMain -s NO_EXIT_RUNTIME=1"
+export EMCC_CFLAGS="-O3 -msimd128 -fno-rtti -DNDEBUG -flto=full -s BUILD_AS_WORKER=1 -s EXPORT_ALL=1 -s EXPORT_ES6=1 -s MODULARIZE=1 -s INITIAL_MEMORY=800MB -s MAXIMUM_MEMORY=4GB -s ALLOW_MEMORY_GROWTH -s FORCE_FILESYSTEM=1 -s EXPORTED_FUNCTIONS=_main -s EXPORTED_RUNTIME_METHODS=callMain -s NO_EXIT_RUNTIME=1"
 emmake make main -j
 cd ../..
 
 #
 # bundle llama-cpp-wasm dist
 #
-if [ -d $LLAMA_CPP_WASM_DIST_DIR ]; then
-    rm -rf $LLAMA_CPP_WASM_DIST_DIR
+if [ -d $LLAMA_CPP_WASM_DIST_LLAMA_DIR ]; then
+    rm -rf $LLAMA_CPP_WASM_DIST_LLAMA_DIR
 fi
 
-mkdir -p $LLAMA_CPP_WASM_DIST_DIR
-cp -r src/* $LLAMA_CPP_WASM_DIST_DIR
-cp $LLAMA_CPP_BUILD_DIR/bin/main.* $LLAMA_CPP_WASM_DIST_DIR/llama
+mkdir -p $LLAMA_CPP_WASM_DIST_LLAMA_DIR
+cp -rv src/llama/* $LLAMA_CPP_WASM_DIST_LLAMA_DIR
+cp $LLAMA_CPP_BUILD_DIR/bin/main.* $LLAMA_CPP_WASM_DIST_LLAMA_DIR
 
 rm -rf docs/llama-st
-mv $LLAMA_CPP_WASM_DIST_DIR/llama docs/llama-st
+cp -rv $LLAMA_CPP_WASM_DIST_LLAMA_DIR docs/

Diff for: build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -e
+
+./build-single-thread.sh
+./build-multi-thread.sh

Diff for: docs/example-multi-thread.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>llama-cpp-wasm wasm32 multithreading</title>
+    <title>llama-cpp-wasm multithreading</title>
     <link rel="icon" type="image/png" href="favicon.png" />
 
     <!-- picocss -->
@@ -16,10 +16,7 @@
   <body>
     <header class="container">
       <hgroup>
-        <h1><a href="/">llama-cpp-wasm</a></h1>
-        <br />
-
-        <h1><mark>multi-threaded</mark> wasm32 </h1>
+        <h1><a href="/">llama-cpp-wasm</a> &nbsp; &#128007; <mark>multithreading</mark> wasm32 </h1>
         <br />
 
         <p> WebAssembly (Wasm) Build and Bindings for <a href="https://github.com/ggerganov/llama.cpp" target="_blank">llama.cpp</a>. </p>
@@ -31,9 +28,6 @@ <h1><mark>multi-threaded</mark> wasm32 </h1>
         <p> Repository: <a href="https://github.com/tangledgroup/llama-cpp-wasm"> https://github.com/tangledgroup/llama-cpp-wasm </a></p>
         <br />
 
-        <p> Recommended browser: &#129395; <b>Firefox</b></p>
-        <br />
-
         <p> When you click <b>Run</b>, model will be first downloaded and cached in browser. </p>
       </hgroup>
     </header>
@@ -60,7 +54,8 @@ <h2> Demo </h2>
 
         <label> Result: </label>
 
-        <textarea id="result" name="result" rows="10" autocomplete="off"></textarea>
+        <!-- <textarea id="result" name="result" rows="10" autocomplete="off"></textarea> -->
+        <pre id="result" name="result"></pre>
       </section>
 
       <section>
@@ -79,6 +74,6 @@ <h2> Demo </h2>
     </main>
 
     <!-- example -->
-    <script type="module" src="example-multi-thread.js?v=240213-4"></script>
+    <script type="module" src="example-multi-thread.js?v=240213-5"></script>
   </body>
 </html>

Diff for: docs/example-multi-thread.js

@@ -8,7 +8,7 @@ const buttonRunProgressGenerating = document.querySelector("#run-progress-genera
 const selectModel = document.querySelector("select#model");
 const modelProgress = document.querySelector("#model-progress");
 const textareaPrompt = document.querySelector("textarea#prompt");
-const textareaResult = document.querySelector("textarea#result");
+const textareaResult = document.querySelector("#result");
 
 const onModelLoaded = () => {
   const prompt = textareaPrompt.value;
@@ -34,7 +34,8 @@ const onMessageChunk = (text) => {
     buttonRunProgressGenerating.removeAttribute("hidden");
   }
 
-  textareaResult.value += text;
+  // textareaResult.value += text;
+  textareaResult.innerText += text;
 };
 
 const onComplete = () => {
@@ -50,7 +51,8 @@ buttonRun.addEventListener("click", (e) => {
   buttonRun.setAttribute("hidden", "hidden");
   buttonRunProgressLoadingModel.removeAttribute("hidden");
   modelProgress.removeAttribute("hidden");
-  textareaResult.value = "";
+  // textareaResult.value = "";
+  textareaResult.innerText = "";
 
   if (app && app.url == selectModel.value) {
     onModelLoaded();

Diff for: docs/example-single-thread.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>llama-cpp-wasm wasm32 single thread</title>
+    <title>llama-cpp-wasm single thread</title>
     <link rel="icon" type="image/png" href="favicon.png" />
 
     <!-- picocss -->
@@ -16,10 +16,7 @@
   <body>
     <header class="container">
       <hgroup>
-        <h1><a href="/">llama-cpp-wasm</a></h1>
-        <br />
-        
-        <h2><mark>single-threaded</mark> wasm32 </h2>
+        <h1><a href="/">llama-cpp-wasm</a> &nbsp; &#128034; <mark>single thread</mark> wasm32 </h2>
         <br />
 
         <p> WebAssembly (Wasm) Build and Bindings for <a href="https://github.com/ggerganov/llama.cpp" target="_blank">llama.cpp</a>. </p>
@@ -31,9 +28,6 @@ <h2><mark>single-threaded</mark> wasm32 </h2>
         <p> Repository: <a href="https://github.com/tangledgroup/llama-cpp-wasm"> https://github.com/tangledgroup/llama-cpp-wasm </a></p>
         <br />
 
-        <p> Recommended browsers: &#128549; <b>Chrome</b>, &#128549; <b>Safari</b>, &#129395; <b>Firefox</b></p>
-        <br />
-
         <p> When you click <b>Run</b>, model will be first downloaded and cached in browser. </p>
       </hgroup>
     </header>
@@ -60,7 +54,8 @@ <h2> Demo </h2>
 
         <label> Result: </label>
 
-        <textarea id="result" name="result" rows="10" autocomplete="off"></textarea>
+        <!-- <textarea id="result" name="result" rows="10" autocomplete="off"></textarea> -->
+        <pre id="result" name="result"></pre>
       </section>
 
       <section>
@@ -79,6 +74,6 @@ <h2> Demo </h2>
     </main>
 
     <!-- example -->
-    <script type="module" src="example-single-thread.js?v=240213-4"></script>
+    <script type="module" src="example-single-thread.js?v=240213-5"></script>
   </body>
 </html>

Diff for: docs/example-single-thread.js

@@ -8,7 +8,7 @@ const buttonRunProgressGenerating = document.querySelector("#run-progress-genera
 const selectModel = document.querySelector("select#model");
 const modelProgress = document.querySelector("#model-progress");
 const textareaPrompt = document.querySelector("textarea#prompt");
-const textareaResult = document.querySelector("textarea#result");
+const textareaResult = document.querySelector("#result");
 
 const onModelLoaded = () => {
   const prompt = textareaPrompt.value;
@@ -34,7 +34,8 @@ const onMessageChunk = (text) => {
     buttonRunProgressGenerating.removeAttribute("hidden");
   }
 
-  textareaResult.value += text;
+  // textareaResult.value += text;
+  textareaResult.innerText += text;
 };
 
 const onComplete = () => {
@@ -50,7 +51,8 @@ buttonRun.addEventListener("click", (e) => {
   buttonRun.setAttribute("hidden", "hidden");
   buttonRunProgressLoadingModel.removeAttribute("hidden");
   modelProgress.removeAttribute("hidden");
-  textareaResult.value = "";
+  // textareaResult.value = "";
+  textareaResult.innerText = "";
 
   if (app && app.url == selectModel.value) {
     onModelLoaded();

Diff for: docs/index.html

@@ -40,8 +40,8 @@ <h1><a href="/">llama-cpp-wasm</a></h1>
         <h2> In-Browser Demos </h2>
 
         <ul>
-          <li><a href="/example-single-thread.html"> <b> single-threaded </b> wasm32 </a> (Recommended browsers: &#128549; <b>Chrome</b>, &#128549; <b>Safari</b>, &#129395; <b>Firefox</b>) </li>
-          <li><a href="/example-multi-thread.html"> <b> multi-threading </b> wasm32 </a> (Recommended browser: &#129395; <b>Firefox</b>)</li>
+          <li><a href="/example-single-thread.html"> &#128034; &nbsp; <b> single thread </b> wasm32 </a></li>
+          <li><a href="/example-multi-thread.html"> &#128007; &nbsp; <b> multithreading </b> wasm32 </a></li>
         </ul>
       </section>
 

Diff for: docs/llama-mt/main-worker.js

@@ -103,10 +103,10 @@ const run_main = (
         "--prompt", prompt.toString(),
     ];
 
-    // if (!!globalThis.SharedArrayBuffer) {
-    //     args.push("--threads");
-    //     args.push((navigator.hardwareConcurrency).toString());
-    // }
+    if (!!globalThis.SharedArrayBuffer) {
+        args.push("--threads");
+        args.push((navigator.hardwareConcurrency).toString());
+    }
 
     if (chatml) {
         args.push("--chatml");

Diff for: docs/llama-mt/main.js

@@ -103,10 +103,10 @@ const run_main = (
         "--prompt", prompt.toString(),
     ];
 
-    // if (!!globalThis.SharedArrayBuffer) {
-    //     args.push("--threads");
-    //     args.push((navigator.hardwareConcurrency).toString());
-    // }
+    if (!!globalThis.SharedArrayBuffer) {
+        args.push("--threads");
+        args.push((navigator.hardwareConcurrency).toString());
+    }
 
     if (chatml) {
         args.push("--chatml");

Diff for: docs/llama-mt/main.wasm

@@ -1,6 +1,6 @@
 {
   "name": "llama-cpp-wasm",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "WebAssembly (Wasm) Build and Bindings for llama.cpp",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"

Diff for: docs/llama-st/main-worker.js

@@ -103,10 +103,10 @@ const run_main = (
         "--prompt", prompt.toString(),
     ];
 
-    // if (!!globalThis.SharedArrayBuffer) {
-    //     args.push("--threads");
-    //     args.push((navigator.hardwareConcurrency).toString());
-    // }
+    if (!!globalThis.SharedArrayBuffer) {
+        args.push("--threads");
+        args.push((navigator.hardwareConcurrency).toString());
+    }
 
     if (chatml) {
         args.push("--chatml");