idk what changed here

.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+  "configurations": [
+    {
+      "name": "macos-clang-x64",
+      "includePath": [
+        "${workspaceFolder}/**"
+      ],
+      "compilerPath": "/usr/bin/clang",
+      "cStandard": "c23",
+      "cppStandard": "c++20",
+      "intelliSenseMode": "macos-clang-x64",
+      "compilerArgs": [
+        ""
+      ]
+    }
+  ],
+  "version": 4
+}

.vscode/launch.json

@@ -0,0 +1,13 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "C/C++ Runner: Debug Session",
+      "type": "lldb",
+      "request": "launch",
+      "args": [],
+      "cwd": "/Users/martia/phi-offline",
+      "program": "/Users/martia/phi-offline/build/Debug/outDebug"
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,59 @@
+{
+  "C_Cpp_Runner.cCompilerPath": "clang",
+  "C_Cpp_Runner.cppCompilerPath": "clang++",
+  "C_Cpp_Runner.debuggerPath": "lldb",
+  "C_Cpp_Runner.cStandard": "c23",
+  "C_Cpp_Runner.cppStandard": "c++20",
+  "C_Cpp_Runner.msvcBatchPath": "",
+  "C_Cpp_Runner.useMsvc": false,
+  "C_Cpp_Runner.warnings": [
+    "-Wall",
+    "-Wextra",
+    "-Wpedantic",
+    "-Wshadow",
+    "-Wformat=2",
+    "-Wcast-align",
+    "-Wconversion",
+    "-Wsign-conversion",
+    "-Wnull-dereference"
+  ],
+  "C_Cpp_Runner.msvcWarnings": [
+    "/W4",
+    "/permissive-",
+    "/w14242",
+    "/w14287",
+    "/w14296",
+    "/w14311",
+    "/w14826",
+    "/w44062",
+    "/w44242",
+    "/w14905",
+    "/w14906",
+    "/w14263",
+    "/w44265",
+    "/w14928"
+  ],
+  "C_Cpp_Runner.enableWarnings": true,
+  "C_Cpp_Runner.warningsAsError": false,
+  "C_Cpp_Runner.compilerArgs": [],
+  "C_Cpp_Runner.linkerArgs": [],
+  "C_Cpp_Runner.includePaths": [],
+  "C_Cpp_Runner.includeSearch": [
+    "*",
+    "**/*"
+  ],
+  "C_Cpp_Runner.excludeSearch": [
+    "**/build",
+    "**/build/**",
+    "**/.*",
+    "**/.*/**",
+    "**/.vscode",
+    "**/.vscode/**"
+  ],
+  "C_Cpp_Runner.useAddressSanitizer": false,
+  "C_Cpp_Runner.useUndefinedSanitizer": false,
+  "C_Cpp_Runner.useLeakSanitizer": false,
+  "C_Cpp_Runner.showCompilationTime": false,
+  "C_Cpp_Runner.useLinkTimeOptimization": false,
+  "C_Cpp_Runner.msvcSecureNoWarnings": false
+}

README.md

@@ -1,11 +1 @@
----
-title: Candle Phi Wasm Demo
-emoji: 🕯️ɸ
-colorFrom: gray
-colorTo: purple
-sdk: static
-pinned: false
-disable_embedding: true
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Candle phi 1.5 / 2 demo

build/m.js

@@ -453,7 +453,7 @@ async function __wbg_init(input) {
     if (wasm !== undefined) return wasm;
 
     if (typeof input === 'undefined') {
-        input = new URL('m_bg.wasm', import.meta.url);
+        input = new URL('agent.wasm', import.meta.url);
     }
     const imports = __wbg_get_imports();
 

phi-offline/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

phi-offline/README.md

@@ -0,0 +1,11 @@
+---
+title: Candle Phi Wasm Demo
+emoji: 🕯️ɸ
+colorFrom: gray
+colorTo: purple
+sdk: static
+pinned: false
+disable_embedding: true
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

phi-offline/build/m.d.ts

@@ -0,0 +1,66 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+*/
+export class Model {
+  free(): void;
+/**
+* @param {Uint8Array} weights
+* @param {Uint8Array} tokenizer
+* @param {Uint8Array} config
+* @param {boolean} quantized
+*/
+  constructor(weights: Uint8Array, tokenizer: Uint8Array, config: Uint8Array, quantized: boolean);
+/**
+* @param {string} prompt
+* @param {number} temp
+* @param {number} top_p
+* @param {number} repeat_penalty
+* @param {number} repeat_last_n
+* @param {bigint} seed
+* @returns {string}
+*/
+  init_with_prompt(prompt: string, temp: number, top_p: number, repeat_penalty: number, repeat_last_n: number, seed: bigint): string;
+/**
+* @returns {string}
+*/
+  next_token(): string;
+}
+
+export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
+
+export interface InitOutput {
+  readonly memory: WebAssembly.Memory;
+  readonly __wbg_model_free: (a: number) => void;
+  readonly model_load: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => void;
+  readonly model_init_with_prompt: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => void;
+  readonly model_next_token: (a: number, b: number) => void;
+  readonly main: (a: number, b: number) => number;
+  readonly __wbindgen_add_to_stack_pointer: (a: number) => number;
+  readonly __wbindgen_malloc: (a: number, b: number) => number;
+  readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+  readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+  readonly __wbindgen_exn_store: (a: number) => void;
+  readonly __wbindgen_start: () => void;
+}
+
+export type SyncInitInput = BufferSource | WebAssembly.Module;
+/**
+* Instantiates the given `module`, which can either be bytes or
+* a precompiled `WebAssembly.Module`.
+*
+* @param {SyncInitInput} module
+*
+* @returns {InitOutput}
+*/
+export function initSync(module: SyncInitInput): InitOutput;
+
+/**
+* If `module_or_path` is {RequestInfo} or {URL}, makes a request and
+* for everything else, calls `WebAssembly.instantiate` directly.
+*
+* @param {InitInput | Promise<InitInput>} module_or_path
+*
+* @returns {Promise<InitOutput>}
+*/
+export default function __wbg_init (module_or_path?: InitInput | Promise<InitInput>): Promise<InitOutput>;

phi-offline/build/m.js

@@ -0,0 +1,472 @@
+let wasm;
+
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
+
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+let cachedUint8Memory0 = null;
+
+function getUint8Memory0() {
+    if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) {
+        cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer);
+    }
+    return cachedUint8Memory0;
+}
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+}
+
+const heap = new Array(128).fill(undefined);
+
+heap.push(undefined, null, true, false);
+
+let heap_next = heap.length;
+
+function addHeapObject(obj) {
+    if (heap_next === heap.length) heap.push(heap.length + 1);
+    const idx = heap_next;
+    heap_next = heap[idx];
+
+    heap[idx] = obj;
+    return idx;
+}
+
+function getObject(idx) { return heap[idx]; }
+
+function dropObject(idx) {
+    if (idx < 132) return;
+    heap[idx] = heap_next;
+    heap_next = idx;
+}
+
+function takeObject(idx) {
+    const ret = getObject(idx);
+    dropObject(idx);
+    return ret;
+}
+
+let WASM_VECTOR_LEN = 0;
+
+function passArray8ToWasm0(arg, malloc) {
+    const ptr = malloc(arg.length * 1, 1) >>> 0;
+    getUint8Memory0().set(arg, ptr / 1);
+    WASM_VECTOR_LEN = arg.length;
+    return ptr;
+}
+
+let cachedInt32Memory0 = null;
+
+function getInt32Memory0() {
+    if (cachedInt32Memory0 === null || cachedInt32Memory0.byteLength === 0) {
+        cachedInt32Memory0 = new Int32Array(wasm.memory.buffer);
+    }
+    return cachedInt32Memory0;
+}
+
+const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
+
+const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
+    ? function (arg, view) {
+    return cachedTextEncoder.encodeInto(arg, view);
+}
+    : function (arg, view) {
+    const buf = cachedTextEncoder.encode(arg);
+    view.set(buf);
+    return {
+        read: arg.length,
+        written: buf.length
+    };
+});
+
+function passStringToWasm0(arg, malloc, realloc) {
+
+    if (realloc === undefined) {
+        const buf = cachedTextEncoder.encode(arg);
+        const ptr = malloc(buf.length, 1) >>> 0;
+        getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
+        WASM_VECTOR_LEN = buf.length;
+        return ptr;
+    }
+
+    let len = arg.length;
+    let ptr = malloc(len, 1) >>> 0;
+
+    const mem = getUint8Memory0();
+
+    let offset = 0;
+
+    for (; offset < len; offset++) {
+        const code = arg.charCodeAt(offset);
+        if (code > 0x7F) break;
+        mem[ptr + offset] = code;
+    }
+
+    if (offset !== len) {
+        if (offset !== 0) {
+            arg = arg.slice(offset);
+        }
+        ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
+        const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
+        const ret = encodeString(arg, view);
+
+        offset += ret.written;
+    }
+
+    WASM_VECTOR_LEN = offset;
+    return ptr;
+}
+
+function handleError(f, args) {
+    try {
+        return f.apply(this, args);
+    } catch (e) {
+        wasm.__wbindgen_exn_store(addHeapObject(e));
+    }
+}
+/**
+*/
+export class Model {
+
+    __destroy_into_raw() {
+        const ptr = this.__wbg_ptr;
+        this.__wbg_ptr = 0;
+
+        return ptr;
+    }
+
+    free() {
+        const ptr = this.__destroy_into_raw();
+        wasm.__wbg_model_free(ptr);
+    }
+    /**
+    * @param {Uint8Array} weights
+    * @param {Uint8Array} tokenizer
+    * @param {Uint8Array} config
+    * @param {boolean} quantized
+    */
+    constructor(weights, tokenizer, config, quantized) {
+        try {
+            const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+            const ptr0 = passArray8ToWasm0(weights, wasm.__wbindgen_malloc);
+            const len0 = WASM_VECTOR_LEN;
+            const ptr1 = passArray8ToWasm0(tokenizer, wasm.__wbindgen_malloc);
+            const len1 = WASM_VECTOR_LEN;
+            const ptr2 = passArray8ToWasm0(config, wasm.__wbindgen_malloc);
+            const len2 = WASM_VECTOR_LEN;
+            wasm.model_load(retptr, ptr0, len0, ptr1, len1, ptr2, len2, quantized);
+            var r0 = getInt32Memory0()[retptr / 4 + 0];
+            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r2 = getInt32Memory0()[retptr / 4 + 2];
+            if (r2) {
+                throw takeObject(r1);
+            }
+            this.__wbg_ptr = r0 >>> 0;
+            return this;
+        } finally {
+            wasm.__wbindgen_add_to_stack_pointer(16);
+        }
+    }
+    /**
+    * @param {string} prompt
+    * @param {number} temp
+    * @param {number} top_p
+    * @param {number} repeat_penalty
+    * @param {number} repeat_last_n
+    * @param {bigint} seed
+    * @returns {string}
+    */
+    init_with_prompt(prompt, temp, top_p, repeat_penalty, repeat_last_n, seed) {
+        let deferred3_0;
+        let deferred3_1;
+        try {
+            const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+            const ptr0 = passStringToWasm0(prompt, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+            const len0 = WASM_VECTOR_LEN;
+            wasm.model_init_with_prompt(retptr, this.__wbg_ptr, ptr0, len0, temp, top_p, repeat_penalty, repeat_last_n, seed);
+            var r0 = getInt32Memory0()[retptr / 4 + 0];
+            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r2 = getInt32Memory0()[retptr / 4 + 2];
+            var r3 = getInt32Memory0()[retptr / 4 + 3];
+            var ptr2 = r0;
+            var len2 = r1;
+            if (r3) {
+                ptr2 = 0; len2 = 0;
+                throw takeObject(r2);
+            }
+            deferred3_0 = ptr2;
+            deferred3_1 = len2;
+            return getStringFromWasm0(ptr2, len2);
+        } finally {
+            wasm.__wbindgen_add_to_stack_pointer(16);
+            wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
+        }
+    }
+    /**
+    * @returns {string}
+    */
+    next_token() {
+        let deferred2_0;
+        let deferred2_1;
+        try {
+            const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+            wasm.model_next_token(retptr, this.__wbg_ptr);
+            var r0 = getInt32Memory0()[retptr / 4 + 0];
+            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r2 = getInt32Memory0()[retptr / 4 + 2];
+            var r3 = getInt32Memory0()[retptr / 4 + 3];
+            var ptr1 = r0;
+            var len1 = r1;
+            if (r3) {
+                ptr1 = 0; len1 = 0;
+                throw takeObject(r2);
+            }
+            deferred2_0 = ptr1;
+            deferred2_1 = len1;
+            return getStringFromWasm0(ptr1, len1);
+        } finally {
+            wasm.__wbindgen_add_to_stack_pointer(16);
+            wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+        }
+    }
+}
+
+async function __wbg_load(module, imports) {
+    if (typeof Response === 'function' && module instanceof Response) {
+        if (typeof WebAssembly.instantiateStreaming === 'function') {
+            try {
+                return await WebAssembly.instantiateStreaming(module, imports);
+
+            } catch (e) {
+                if (module.headers.get('Content-Type') != 'application/wasm') {
+                    console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
+
+                } else {
+                    throw e;
+                }
+            }
+        }
+
+        const bytes = await module.arrayBuffer();
+        return await WebAssembly.instantiate(bytes, imports);
+
+    } else {
+        const instance = await WebAssembly.instantiate(module, imports);
+
+        if (instance instanceof WebAssembly.Instance) {
+            return { instance, module };
+
+        } else {
+            return instance;
+        }
+    }
+}
+
+function __wbg_get_imports() {
+    const imports = {};
+    imports.wbg = {};
+    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
+        const ret = new Error(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_new_abda76e883ba8a5f = function() {
+        const ret = new Error();
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_stack_658279fe44541cf6 = function(arg0, arg1) {
+        const ret = getObject(arg1).stack;
+        const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len1 = WASM_VECTOR_LEN;
+        getInt32Memory0()[arg0 / 4 + 1] = len1;
+        getInt32Memory0()[arg0 / 4 + 0] = ptr1;
+    };
+    imports.wbg.__wbg_error_f851667af71bcfc6 = function(arg0, arg1) {
+        let deferred0_0;
+        let deferred0_1;
+        try {
+            deferred0_0 = arg0;
+            deferred0_1 = arg1;
+            console.error(getStringFromWasm0(arg0, arg1));
+        } finally {
+            wasm.__wbindgen_free(deferred0_0, deferred0_1, 1);
+        }
+    };
+    imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
+        takeObject(arg0);
+    };
+    imports.wbg.__wbg_log_aa756f7b1647d2ab = function(arg0, arg1) {
+        console.log(getStringFromWasm0(arg0, arg1));
+    };
+    imports.wbg.__wbg_crypto_58f13aa23ffcb166 = function(arg0) {
+        const ret = getObject(arg0).crypto;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbindgen_is_object = function(arg0) {
+        const val = getObject(arg0);
+        const ret = typeof(val) === 'object' && val !== null;
+        return ret;
+    };
+    imports.wbg.__wbg_process_5b786e71d465a513 = function(arg0) {
+        const ret = getObject(arg0).process;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_versions_c2ab80650590b6a2 = function(arg0) {
+        const ret = getObject(arg0).versions;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_node_523d7bd03ef69fba = function(arg0) {
+        const ret = getObject(arg0).node;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbindgen_is_string = function(arg0) {
+        const ret = typeof(getObject(arg0)) === 'string';
+        return ret;
+    };
+    imports.wbg.__wbg_msCrypto_abcb1295e768d1f2 = function(arg0) {
+        const ret = getObject(arg0).msCrypto;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_require_2784e593a4674877 = function() { return handleError(function () {
+        const ret = module.require;
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbindgen_is_function = function(arg0) {
+        const ret = typeof(getObject(arg0)) === 'function';
+        return ret;
+    };
+    imports.wbg.__wbindgen_string_new = function(arg0, arg1) {
+        const ret = getStringFromWasm0(arg0, arg1);
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_randomFillSync_a0d98aa11c81fe89 = function() { return handleError(function (arg0, arg1) {
+        getObject(arg0).randomFillSync(takeObject(arg1));
+    }, arguments) };
+    imports.wbg.__wbg_getRandomValues_504510b5564925af = function() { return handleError(function (arg0, arg1) {
+        getObject(arg0).getRandomValues(getObject(arg1));
+    }, arguments) };
+    imports.wbg.__wbg_newnoargs_ccdcae30fd002262 = function(arg0, arg1) {
+        const ret = new Function(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_call_669127b9d730c650 = function() { return handleError(function (arg0, arg1) {
+        const ret = getObject(arg0).call(getObject(arg1));
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbindgen_object_clone_ref = function(arg0) {
+        const ret = getObject(arg0);
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_self_3fad056edded10bd = function() { return handleError(function () {
+        const ret = self.self;
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbg_window_a4f46c98a61d4089 = function() { return handleError(function () {
+        const ret = window.window;
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbg_globalThis_17eff828815f7d84 = function() { return handleError(function () {
+        const ret = globalThis.globalThis;
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbg_global_46f939f6541643c5 = function() { return handleError(function () {
+        const ret = global.global;
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbindgen_is_undefined = function(arg0) {
+        const ret = getObject(arg0) === undefined;
+        return ret;
+    };
+    imports.wbg.__wbg_call_53fc3abd42e24ec8 = function() { return handleError(function (arg0, arg1, arg2) {
+        const ret = getObject(arg0).call(getObject(arg1), getObject(arg2));
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbg_now_4579335d3581594c = function() {
+        const ret = Date.now();
+        return ret;
+    };
+    imports.wbg.__wbg_buffer_344d9b41efe96da7 = function(arg0) {
+        const ret = getObject(arg0).buffer;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_newwithbyteoffsetandlength_2dc04d99088b15e3 = function(arg0, arg1, arg2) {
+        const ret = new Uint8Array(getObject(arg0), arg1 >>> 0, arg2 >>> 0);
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_new_d8a000788389a31e = function(arg0) {
+        const ret = new Uint8Array(getObject(arg0));
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_set_dcfd613a3420f908 = function(arg0, arg1, arg2) {
+        getObject(arg0).set(getObject(arg1), arg2 >>> 0);
+    };
+    imports.wbg.__wbg_newwithlength_13b5319ab422dcf6 = function(arg0) {
+        const ret = new Uint8Array(arg0 >>> 0);
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_subarray_6ca5cfa7fbb9abbe = function(arg0, arg1, arg2) {
+        const ret = getObject(arg0).subarray(arg1 >>> 0, arg2 >>> 0);
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbindgen_throw = function(arg0, arg1) {
+        throw new Error(getStringFromWasm0(arg0, arg1));
+    };
+    imports.wbg.__wbindgen_memory = function() {
+        const ret = wasm.memory;
+        return addHeapObject(ret);
+    };
+
+    return imports;
+}
+
+function __wbg_init_memory(imports, maybe_memory) {
+
+}
+
+function __wbg_finalize_init(instance, module) {
+    wasm = instance.exports;
+    __wbg_init.__wbindgen_wasm_module = module;
+    cachedInt32Memory0 = null;
+    cachedUint8Memory0 = null;
+
+    wasm.__wbindgen_start();
+    return wasm;
+}
+
+function initSync(module) {
+    if (wasm !== undefined) return wasm;
+
+    const imports = __wbg_get_imports();
+
+    __wbg_init_memory(imports);
+
+    if (!(module instanceof WebAssembly.Module)) {
+        module = new WebAssembly.Module(module);
+    }
+
+    const instance = new WebAssembly.Instance(module, imports);
+
+    return __wbg_finalize_init(instance, module);
+}
+
+async function __wbg_init(input) {
+    if (wasm !== undefined) return wasm;
+
+    if (typeof input === 'undefined') {
+        input = new URL('m_bg.wasm', import.meta.url);
+    }
+    const imports = __wbg_get_imports();
+
+    if (typeof input === 'string' || (typeof Request === 'function' && input instanceof Request) || (typeof URL === 'function' && input instanceof URL)) {
+        input = fetch(input);
+    }
+
+    __wbg_init_memory(imports);
+
+    const { instance, module } = await __wbg_load(await input, imports);
+
+    return __wbg_finalize_init(instance, module);
+}
+
+export { initSync }
+export default __wbg_init;

phi-offline/build/m_bg.wasm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a62b31d8872b8c4c1f0f98dad8f2375d05ad120060731eb02ca21f827d188ac5
+size 4566571

phi-offline/build/m_bg.wasm.d.ts

@@ -0,0 +1,14 @@
+/* tslint:disable */
+/* eslint-disable */
+export const memory: WebAssembly.Memory;
+export function __wbg_model_free(a: number): void;
+export function model_load(a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number): void;
+export function model_init_with_prompt(a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number): void;
+export function model_next_token(a: number, b: number): void;
+export function main(a: number, b: number): number;
+export function __wbindgen_add_to_stack_pointer(a: number): number;
+export function __wbindgen_malloc(a: number, b: number): number;
+export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number;
+export function __wbindgen_free(a: number, b: number, c: number): void;
+export function __wbindgen_exn_store(a: number): void;
+export function __wbindgen_start(): void;

phi-offline/index.html

@@ -0,0 +1,377 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <meta content="text/htmlcharset=utf-8" http-equiv="Content-Type" />
+    <meta name="apple-mobile-web-app-capable" content="yes">
+    <link rel="shortcut icon" href="candle.png">
+    <link rel="apple-touch-icon" href="candle.png">
+    <title>Candle Phi Rust/WASM</title>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="stylesheet"
+        href="https://cdn.jsdelivr.net/gh/highlightjs/[email protected]/build/styles/default.min.css" />
+    <style>
+        @import url("https://fonts.googleapis.com/css2?family=Source+Code+Pro:wght@200300400&family=Source+Sans+3:wght@100200300400500600700800900&display=swap");
+
+        html,
+        body {
+            font-family: "Source Sans 3", sans-serif;
+        }
+
+        code,
+        output,
+        select,
+        pre {
+            font-family: "Source Code Pro", monospace;
+        }
+    </style>
+    <style type="text/tailwindcss">
+        .link { @apply underline hover:text-blue-500 hover:no-underline; }
+        </style>
+    <script src="https://cdn.tailwindcss.com"></script>
+    <script type="module">
+        import snarkdown from "https://cdn.skypack.dev/snarkdown"
+        import hljs from "https://cdn.skypack.dev/highlight.js"
+        // models base url
+        const MODELS = {
+            phi_1_5_q4k: {
+                base_url:
+                    "https://huggingface.co/lmz/candle-quantized-phi/resolve/main/",
+                model: "model-q4k.gguf",
+                tokenizer: "tokenizer.json",
+                config: "phi-1_5.json",
+                quantized: true,
+                seq_len: 2048,
+                size: "800 MB",
+            },
+            phi_1_5_q80: {
+                base_url:
+                    "https://huggingface.co/lmz/candle-quantized-phi/resolve/main/",
+                model: "model-q80.gguf",
+                tokenizer: "tokenizer.json",
+                config: "phi-1_5.json",
+                quantized: true,
+                seq_len: 2048,
+                size: "1.51 GB",
+            },
+            phi_2_0_q4k: {
+                base_url:
+                    "https://huggingface.co/radames/phi-2-quantized/resolve/main/",
+                model: [
+                    "model-v2-q4k.gguf_aa.part",
+                    "model-v2-q4k.gguf_ab.part",
+                    "model-v2-q4k.gguf_ac.part",
+                ],
+                tokenizer: "tokenizer.json",
+                config: "config.json",
+                quantized: true,
+                seq_len: 2048,
+                size: "1.57GB",
+            },
+            puffin_phi_v2_q4k: {
+                base_url:
+                    "https://huggingface.co/lmz/candle-quantized-phi/resolve/main/",
+                model: "model-puffin-phi-v2-q4k.gguf",
+                tokenizer: "tokenizer-puffin-phi-v2.json",
+                config: "puffin-phi-v2.json",
+                quantized: true,
+                seq_len: 2048,
+                size: "798 MB",
+            },
+            puffin_phi_v2_q80: {
+                base_url:
+                    "https://huggingface.co/lmz/candle-quantized-phi/resolve/main/",
+                model: "model-puffin-phi-v2-q80.gguf",
+                tokenizer: "tokenizer-puffin-phi-v2.json",
+                config: "puffin-phi-v2.json",
+                quantized: true,
+                seq_len: 2048,
+                size: "1.50 GB",
+            },
+        }
+
+        const phiWorker = new Worker("./phiWorker.js", {
+            type: "module",
+        })
+        async function generateSequence(prompt, controller) {
+            const getValue = (id) => document.querySelector(`#${id}`).value
+            const modelID = getValue("model")
+            const model = MODELS[modelID]
+            const weightsURL =
+                model.model instanceof Array
+                    ? model.model.map((m) => model.base_url + m)
+                    : model.base_url + model.model
+            const tokenizerURL = model.base_url + model.tokenizer
+            const configURL = model.base_url + model.config
+
+            // const prompt = getValue("prompt").trim()
+            const temperature = getValue("temperature")
+            const topP = getValue("top-p")
+            const repeatPenalty = getValue("repeat_penalty")
+            const seed = getValue("seed")
+            const maxSeqLen = getValue("max-seq")
+
+            function updateStatus(data) {
+                const outStatus = document.querySelector("#output-status")
+                const outGen = document.querySelector("#output-generation")
+                const outCounter = document.querySelector("#output-counter")
+
+                switch (data.status) {
+                    case "loading":
+                        outStatus.hidden = false
+                        outStatus.innerHTML = data.message.replaceAll("\n", "<br>\n")
+                        outGen.hidden = true
+                        outCounter.hidden = true
+                        break
+                    case "generating":
+                        const { message, prompt, sentence, tokensSec, totalTime } = data
+                        outStatus.hidden = true
+                        outCounter.hidden = false
+                        outGen.hidden = false
+                        outGen.innerHTML = snarkdown(prompt + sentence).replaceAll('\n', '<br>\n')
+                        outCounter.innerHTML = `${(totalTime / 1000).toFixed(
+                            2
+                        )}s (${tokensSec.toFixed(2)} tok/s)`
+                        hljs.highlightAll()
+                        break
+                    case "complete":
+                        outStatus.hidden = true
+                        outGen.hidden = false
+                        break
+                }
+            }
+
+            function decodeHtml(html) {
+                var txt = document.createElement('textarea')
+                txt.innerHTML = html
+                return txt.value
+            }
+
+            return new Promise((resolve, reject) => {
+                let TEXT = document.querySelector('textarea#terminate')?.innerText
+                if (TEXT === '') TEXT = decodeHtml(document.querySelector('textarea#terminate')?.innerHTML)
+                phiWorker.postMessage({
+                    weightsURL,
+                    modelID,
+                    tokenizerURL,
+                    configURL,
+                    quantized: model.quantized,
+                    prompt,
+                    temp: temperature,
+                    top_p: topP,
+                    repeatPenalty,
+                    seed: seed,
+                    maxSeqLen,
+                    command: "start",
+                    stuff: TEXT.split(',').map(e => e.trim())
+                })
+
+                const handleAbort = () => {
+                    phiWorker.postMessage({ command: "abort" })
+                }
+                const handleMessage = (event) => {
+                    const { status, error, message, prompt, sentence } = event.data
+                    if (status) updateStatus(event.data)
+                    if (error) {
+                        phiWorker.removeEventListener("message", handleMessage)
+                        reject(new Error(error))
+                    }
+                    if (status === "aborted") {
+                        phiWorker.removeEventListener("message", handleMessage)
+                        resolve(event.data)
+                    }
+                    if (status === "complete") {
+                        phiWorker.removeEventListener("message", handleMessage)
+                        resolve(event.data)
+                    }
+                }
+
+                controller.signal.addEventListener("abort", handleAbort)
+                phiWorker.addEventListener("message", handleMessage)
+            })
+        }
+
+        const form = document.querySelector("#form")
+        const prompt = document.querySelector("#prompt")
+        const clearBtn = document.querySelector("#clear-btn")
+        const runBtn = document.querySelector("#run")
+        const modelSelect = document.querySelector("#model")
+        let runController = new AbortController()
+        let isRunning = false
+
+        document.addEventListener("DOMContentLoaded", () => {
+            for (const [id, model] of Object.entries(MODELS)) {
+                const option = document.createElement("option")
+                option.value = id
+                option.innerText = `${id} (${model.size})`
+                modelSelect.appendChild(option)
+            }
+            const query = new URLSearchParams(window.location.search)
+            const modelID = query.get("model")
+            if (modelID) {
+                modelSelect.value = modelID
+            } else {
+                modelSelect.value = "phi_1_5_q4k"
+            }
+        })
+
+        const TEMPLATES = { entries: () => [] }
+
+        for (const [i, { title, prompt }] of TEMPLATES.entries()) {
+            const div = document.createElement("div")
+            const input = document.createElement("input")
+            input.type = "radio"
+            input.name = "task"
+            input.id = `templates-${i}`
+            input.classList.add("font-light", "cursor-pointer")
+            input.value = prompt
+            const label = document.createElement("label")
+            label.htmlFor = `templates-${i}`
+            label.classList.add("cursor-pointer")
+            label.innerText = title
+            div.appendChild(input)
+            div.appendChild(label)
+            promptTemplates.appendChild(div)
+        }
+
+        modelSelect.addEventListener("change", (e) => {
+            const query = new URLSearchParams(window.location.search)
+            query.set("model", e.target.value)
+            window.history.replaceState(
+                {},
+                "",
+                `${window.location.pathname}?${query}`
+            )
+            window.parent.postMessage({ queryString: "?" + query }, "*")
+            const model = MODELS[e.target.value]
+            document.querySelector("#max-seq").max = model.seq_len
+            document.querySelector("#max-seq").nextElementSibling.value = 200
+        })
+
+        form.addEventListener("submit", async (e) => {
+            e.preventDefault()
+            if (isRunning) {
+                stopRunning()
+            } else {
+                startRunning()
+                await generateSequence(document.querySelector(`#prompt`).value, runController)
+                stopRunning()
+            }
+        })
+
+        function startRunning() {
+            isRunning = true
+            runBtn.textContent = "Stop"
+        }
+
+        function stopRunning() {
+            runController.abort()
+            runController = new AbortController()
+            runBtn.textContent = "Run"
+            isRunning = false
+        }
+        clearBtn.addEventListener("click", (e) => {
+            e.preventDefault()
+            prompt.value = ""
+            clearBtn.classList.add("invisible")
+            runBtn.disabled = true
+            stopRunning()
+        })
+        prompt.addEventListener("input", (e) => {
+            runBtn.disabled = false
+            if (e.target.value.length > 0) {
+                clearBtn.classList.remove("invisible")
+            } else {
+                clearBtn.classList.add("invisible")
+            }
+        })
+    </script>
+</head>
+
+<body class="container max-w-4xl mx-auto p-4 text-gray-800">
+    <main class="grid grid-cols-1 gap-8 relative">
+        <span class="absolute text-5xl -ml-[1em]"> 🕯️ </span>
+        <div>
+            <h1 class="text-5xl font-bold">Candle Phi 1.5 / Phi 2.0</h1>
+        </div>
+        <div>
+            <p class="text-m max-w-lg">
+                <b>Note:</b>
+                When first run, the app will download and cache the model, which could
+                take a few minutes. The models are <b>~800MB</b> or <b>~1.57GB</b> in
+                size.
+            </p>
+        </div>
+        <div>
+            <label for="model" class="font-medium">Models Options: </label>
+            <select id="model" class="border-2 border-gray-500 rounded-md font-light"></select>
+        </div>
+        <form id="form" class="flex text-normal px-1 py-1 border border-gray-700 rounded-md items-center">
+            <input type="submit" hidden />
+            <textarea type="text" id="prompt" class="font-light text-lg w-full px-3 py-2 mx-1 resize-none outline-none"
+                oninput="this.style.height = 0; this.style.height = this.scrollHeight + 'px'"
+                placeholder="Add your prompt here..."></textarea>
+            <button id="clear-btn">
+                <svg fill="none" xmlns="http://www.w3.org/2000/svg" width="40" viewBox="0 0 70 40">
+                    <path opacity=".5" d="M39 .2v40.2" stroke="#1F2937" />
+                    <path d="M1.5 11.5 19 29.1m0-17.6L1.5 29.1" opacity=".5" stroke="#1F2937" stroke-width="2" />
+                </svg>
+            </button>
+            <button id="run"
+                class="bg-gray-700 hover:bg-gray-800 text-white font-normal py-2 w-16 rounded disabled:bg-gray-300 disabled:cursor-not-allowed">
+                Run
+            </button>
+        </form>
+        <details>
+            <summary class="font-medium cursor-pointer">Advanced Options</summary>
+            <div class="grid grid-cols-3 max-w-md items-center gap-3 py-3">
+                <label class="text-sm font-medium" for="max-seq">Maximum length
+                </label>
+                <input type="range" id="max-seq" name="max-seq" min="1" max="2048" step="1" value="200"
+                    oninput="this.nextElementSibling.value = Number(this.value)" />
+                <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
+                    200</output>
+                <label class="text-sm font-medium" for="temperature">Temperature</label>
+                <input type="range" id="temperature" name="temperature" min="0" max="2" step="0.01" value="0.50"
+                    oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" />
+                <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
+                    0.50</output>
+                <label class="text-sm font-medium" for="top-p">Top-p</label>
+                <input type="range" id="top-p" name="top-p" min="0" max="1" step="0.01" value="1.00"
+                    oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" />
+                <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
+                    1.00</output>
+                <label class="text-sm font-medium" for="repeat_penalty">Repeat Penalty</label>
+                <input type="range" id="repeat_penalty" name="repeat_penalty" min="1" max="2" step="0.01" value="1.10"
+                    oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" />
+                <output
+                    class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">1.10</output>
+                <label class="text-sm font-medium" for="seed">Seed</label>
+                <input type="number" id="seed" name="seed" value="299792458"
+                    class="font-light border border-gray-700 text-right rounded-md p-2" />
+                <button id="run"
+                    onclick="document.querySelector('#seed').value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)"
+                    class="bg-gray-700 hover:bg-gray-800 text-white font-normal py-1 w-[50px] rounded disabled:bg-gray-300 disabled:cursor-not-allowed text-sm">
+                    Rand
+                </button>
+                <label hidden class="text-sm font-medium" for="terminate">End tokens</label>
+                <textarea hidden type="text" id="terminate"
+                    class="font-light text-lg w-full px-3 py-2 mx-1 resize-none outline-none"
+                    style="padding-left: -10px; border: 1px solid black; border-radius: 5px; width: 500px"
+                    oninput="this.style.height = 0; this.style.height = this.scrollHeight + 'px'"
+                    placeholder="Add your terminate tokens here, Separated by `, `"><|endoftext|>, <|user|>, <|system|>, <|assistant|></textarea>
+            </div>
+        </details>
+        <div>
+            <h3 class="font-medium">Generation:</h3>
+            <div class="min-h-[250px] bg-slate-100 text-gray-500 p-4 rounded-md flex flex-col gap-2">
+                <div id="output-counter" hidden class="ml-auto font-semibold grid-rows-1"></div>
+                <p hidden id="output-generation" class="grid-rows-2 text-lg"></p>
+                <span id="output-status" class="m-auto font-light">No output yet</span>
+            </div>
+        </div>
+    </main>
+</body>
+
+</html>

phi-offline/phiWorker.js

@@ -0,0 +1,242 @@
+import init, { Model } from "./build/m.js"
+
+function fixTwo(x) { return Math.floor(x * 100) / 100 } 
+
+function humanSize(size) {
+    if(size < 1e3) return `${fixTwo(size)}b`
+    if(size < 1e6) return `${fixTwo(size/1e3)}kb`
+    if(size < 1e9) return `${fixTwo(size/1e6)}mb`
+    if(size < 1e12) return `${fixTwo(size/1e9)}gb`
+    return `${fixTwo(size/1e12)}tb`
+}
+
+function humanTime(seconds) {
+    const _year = 31536e3
+    const _mon = 2592e3
+    const _day = 864e2
+    const _hour = 36e2
+    const _min = 60
+    const _sec = 1
+
+    const year_rem = seconds % _year
+    const years = (seconds - year_rem) / _year
+
+    const month_rem = year_rem % _mon
+    const months = (year_rem - month_rem) / _mon
+
+    const day_rem = month_rem % _day
+    const days = (month_rem - day_rem) / _day
+
+    const hour_rem = day_rem % _hour
+    const hours = (day_rem - hour_rem) / _hour
+
+    const minute_rem = hour_rem % _min
+    const minutes = (hour_rem - minute_rem) / _min
+
+    const second_rem = minute_rem % _sec
+    const second = (minute_rem - second_rem) / _sec
+
+    return (years > 0 ? `${years} year${years == 1 ? '' : 's'} ` : '') + (months > 0 ? `${months} month${months == 1 ? '' : 's'} `: '') +
+        (days > 0 ? `${days} day${days == 1 ? '' : 's'} ` : '') + (hours > 0 ? `${hours} hour${hours == 1 ? '' : 's'} ` : '') +
+        (minutes > 0 ? `${minutes} minute${minutes == 1 ? '' : 's'} ` : '') + (seconds > 0 ? `${second} second${second == 1 ? '' : 's'} ` : '')
+}
+
+let lastSend = 0
+let lastTime = Infinity
+let times = [0, 0, 0, 0]
+
+async function fetchArrayBuffer(url) {
+    const cacheName = "phi-mixformer-candle-cache"
+    const cache = await caches.open(cacheName)
+    const cachedResponse = await cache.match(url)
+    if (cachedResponse) {
+        const data = await cachedResponse.arrayBuffer()
+        return new Uint8Array(data)
+    }
+    const res = await fetch(url, { cache: "force-cache" })
+    while (!res.body) { }
+    const reader = res.body.getReader()
+    const contentLength = +(res.headers.get('Content-Length') ?? 0)
+    let receivedLength = 0
+    let chunks = []
+    while (true) {
+        const { done, value } = await reader.read()
+        if (done) {
+            break
+        }
+        chunks.push(value)
+        receivedLength += value.length
+        if(Date.now() - lastSend > 250) {
+            times.push(receivedLength)
+            times = times.slice(1)
+            let max = [times[3] - times[2], times[2] - times[1], times[1] - times[0]]
+            let median = (max[0] + max[1] + max[2]) / 3
+            let lengthPerSecond = median * 4
+            let leftSize = contentLength - receivedLength
+            let leftTime = Math.abs(leftSize / lengthPerSecond)
+
+            if(leftTime > lastTime * 1.5 && lastTime != 0) leftTime = lastTime * 1.2
+            // if(leftTime > lastTime) leftTime = lastTime
+            lastTime = leftTime
+            let downloadMessage = `Downloading... ${fixTwo((receivedLength / contentLength) * 100)}% (${humanSize(Math.floor(receivedLength * 100) / 100)})
+Estimated time remaining: ${humanTime(leftTime)} (may be inaccurate)
+Total size: ${humanSize(fixTwo(contentLength))}
+Download URL: ${url}`
+            self.postMessage({ status: "loading", message: downloadMessage })
+            // console.log(downloadMessage)
+            lastSend = Date.now()
+        }
+    }
+    let chunksAll = new Uint8Array(receivedLength)
+    let position = 0
+    for (let chunk of chunks) {
+        chunksAll.set(chunk, position)
+        position += chunk.length
+    }
+    cache.put(url, new Response(chunksAll))
+    return chunksAll
+}
+
+async function concatenateArrayBuffers(urls) {
+    const arrayBuffers = await Promise.all(urls.map(url => fetchArrayBuffer(url)))
+
+    let totalLength = arrayBuffers.reduce((acc, arrayBuffer) => acc + arrayBuffer.byteLength, 0)
+    let concatenatedBuffer = new Uint8Array(totalLength)
+
+    let offset = 0
+    arrayBuffers.forEach(buffer => {
+        concatenatedBuffer.set(new Uint8Array(buffer), offset)
+        offset += buffer.byteLength
+    })
+    return concatenatedBuffer
+}
+
+class Phi {
+    static instance = {}
+
+    static async getInstance(
+        weightsURL,
+        modelID,
+        tokenizerURL,
+        configURL,
+        quantized
+    ) {
+        // load individual modelID only once
+        if (!this.instance[modelID]) {
+            await init()
+
+            self.postMessage({ status: "loading", message: "Loading Model" })
+            const [weightsArrayU8, tokenizerArrayU8, configArrayU8] =
+                await Promise.all([
+                    weightsURL instanceof Array ? concatenateArrayBuffers(weightsURL) : fetchArrayBuffer(weightsURL),
+                    fetchArrayBuffer(tokenizerURL),
+                    fetchArrayBuffer(configURL),
+                ])
+
+            this.instance[modelID] = new Model(
+                weightsArrayU8,
+                tokenizerArrayU8,
+                configArrayU8,
+                quantized
+            )
+        }
+        return this.instance[modelID]
+    }
+}
+
+let controller = null
+self.addEventListener("message", (event) => {
+    if (event.data.command === "start") {
+        controller = new AbortController()
+        generate(event.data)
+    } else if (event.data.command === "abort") {
+        controller.abort()
+    }
+})
+
+async function generate(data) {
+    const {
+        weightsURL,
+        modelID,
+        tokenizerURL,
+        configURL,
+        quantized,
+        prompt,
+        temp,
+        top_p,
+        repeatPenalty,
+        seed,
+        maxSeqLen,
+        stuff
+    } = data
+    try {
+        self.postMessage({ status: "loading", message: "Starting Phi" })
+        const model = await Phi.getInstance(
+            weightsURL,
+            modelID,
+            tokenizerURL,
+            configURL,
+            quantized
+        )
+
+        self.postMessage({ status: "loading", message: "Initializing model" })
+        const firstToken = model.init_with_prompt(
+            prompt,
+            temp,
+            top_p,
+            repeatPenalty,
+            64,
+            BigInt(seed)
+        )
+        const seq_len = 2048
+
+        let sentence = firstToken
+        let maxTokens = maxSeqLen ? maxSeqLen : seq_len - prompt.length - 1
+        let startTime = performance.now()
+        let tokensCount = 0
+        while (tokensCount < maxTokens) {
+            await new Promise(async (resolve) => {
+                if (controller && controller.signal.aborted) {
+                    self.postMessage({
+                        status: "aborted",
+                        message: "Aborted",
+                        output: prompt + sentence,
+                    })
+                    return
+                }
+                const token = await model.next_token()
+                const terminates = `<|endoftext|>, <|user|>, <|system|>, <|assistant|>`.split(', ').map(e => e.trim())
+                if (terminates.includes(token)) {
+                    self.postMessage({
+                        status: "complete",
+                        message: "complete",
+                        output: prompt + sentence,
+                    })
+                    return
+                }
+                const tokensSec =
+                    ((tokensCount + 1) / (performance.now() - startTime)) * 1000
+
+                sentence += token
+                self.postMessage({
+                    status: "generating",
+                    message: "Generating token",
+                    token: token,
+                    sentence: sentence,
+                    totalTime: performance.now() - startTime,
+                    tokensSec,
+                    prompt: prompt,
+                })
+                setTimeout(resolve, 0)
+            })
+            tokensCount++
+        }
+        self.postMessage({
+            status: "complete",
+            message: "complete",
+            output: prompt + sentence,
+        })
+    } catch (e) {
+        self.postMessage({ error: e })
+    }
+}

phi-offline/style.css

@@ -0,0 +1,28 @@
+body {
+	padding: 2rem;
+	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
+}
+
+h1 {
+	font-size: 16px;
+	margin-top: 0;
+}
+
+p {
+	color: rgb(107, 114, 128);
+	font-size: 15px;
+	margin-bottom: 10px;
+	margin-top: 5px;
+}
+
+.card {
+	max-width: 620px;
+	margin: 0 auto;
+	padding: 16px;
+	border: 1px solid lightgray;
+	border-radius: 16px;
+}
+
+.card p:last-child {
+	margin-bottom: 0;
+}