IQ.Pilot Release Commit @ f2a861c
This commit is contained in:
IQ.Lvbs CI [bot]
@@ -0,0 +1,927 @@
|
||||
window.TINYCHAT_ROOT = "/tinychat-browser/";
|
||||
const queryParams = new URLSearchParams(window.location.search);
|
||||
const normalizedParams = Object.fromEntries([...queryParams].map(([key, value]) => [key.toUpperCase(), value.toUpperCase()]));
|
||||
window.BACKEND = (normalizedParams["BACKEND"] === "WASM") ? "WASM" : "WebGPU";
|
||||
const isMobileAgent = /Mobi|Android|iPhone|iPad|iPod/i.test(navigator.userAgent);
|
||||
const hasTouchScreen = 'ontouchstart' in window || navigator.maxTouchPoints > 0;
|
||||
window.isMobile = isMobileAgent || hasTouchScreen;
|
||||
if (window.isMobile) document.documentElement.classList.add('mobile'); // prevent annoying auto-zoom when entering prompt on mobile
|
||||
// MODEL_BASE_URL is where the weights are hosted, WEBGPU_EXPORT is the JS-wrapped WebGPU code exported from tinygrad
|
||||
window.PC_MODEL_BASE_URL = ".";
|
||||
window.PC_WEBGPU_EXPORT = './net.js'
|
||||
window.PC_MAX_CONTEXT = 1024;
|
||||
window.MOBILE_MODEL_BASE_URL = ".";
|
||||
window.MOBILE_WEBGPU_EXPORT = './net.js'
|
||||
window.MOBILE_MAX_CONTEXT = 1024;
|
||||
|
||||
const tiktokenReady = (async () => {
|
||||
const { init, get_encoding, Tiktoken, load } = await import('./tiktoken.js');
|
||||
window.Tiktoken = Tiktoken;
|
||||
window.tiktokenInit = init;
|
||||
window.tiktokenGetEncoding = get_encoding;
|
||||
window.tiktokenLoad = load;
|
||||
})();
|
||||
|
||||
async function getDevice() {
|
||||
let adapter;
|
||||
try {
|
||||
adapter = await navigator.gpu.requestAdapter();
|
||||
if (!adapter) {
|
||||
this.loadingMessage = "Loading WASM (WebGPU not enabled):";
|
||||
throw new Error("No WebGPU adapter found");
|
||||
}
|
||||
} catch(error) {
|
||||
this.loadingMessage = "Loading WASM (WebGPU not enabled):";
|
||||
throw error;
|
||||
}
|
||||
const requiredLimits = {};
|
||||
const maxBufferSize = 322122544;
|
||||
requiredLimits.maxStorageBufferBindingSize = maxBufferSize;
|
||||
requiredLimits.maxBufferSize = maxBufferSize;
|
||||
requiredLimits.maxComputeInvocationsPerWorkgroup = 512; // may need to vary based on what the WEBGPU backend produces
|
||||
|
||||
try {
|
||||
return await adapter.requestDevice({ requiredLimits });
|
||||
} catch(error) {
|
||||
this.loadingMessage = "Loading WASM (WebGPU error):";
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// copied from examples/webgpu/stable_diffusion/index.html
|
||||
function initDb() {
|
||||
return new Promise((resolve, reject) => {
|
||||
let db;
|
||||
const request = indexedDB.open('tinydb', 1);
|
||||
request.onerror = (event) => {
|
||||
console.error('Database error:', event.target.error);
|
||||
resolve(null);
|
||||
};
|
||||
|
||||
request.onsuccess = (event) => {
|
||||
db = event.target.result;
|
||||
console.log("Db initialized.");
|
||||
resolve(db);
|
||||
};
|
||||
|
||||
request.onupgradeneeded = (event) => {
|
||||
db = event.target.result;
|
||||
if (!db.objectStoreNames.contains('tensors')) {
|
||||
db.createObjectStore('tensors', { keyPath: 'id' });
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// copied from examples/webgpu/stable_diffusion/index.html
|
||||
function readTensorFromDb(db, id) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (db == null) {
|
||||
resolve(null);
|
||||
}
|
||||
|
||||
const transaction = db.transaction(['tensors'], 'readonly');
|
||||
const store = transaction.objectStore('tensors');
|
||||
const request = store.get(id);
|
||||
|
||||
transaction.onabort = (event) => {
|
||||
console.log("Transaction error while reading tensor: " + event.target.error);
|
||||
resolve(null);
|
||||
};
|
||||
|
||||
request.onsuccess = (event) => {
|
||||
const result = event.target.result;
|
||||
if (result) {
|
||||
resolve(result);
|
||||
} else {
|
||||
resolve(null);
|
||||
}
|
||||
};
|
||||
|
||||
request.onerror = (event) => {
|
||||
console.error('Tensor retrieve failed: ', event.target.error);
|
||||
resolve(null);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function getAllKeysFromDb(db) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (db == null) {resolve([]);}
|
||||
const transaction = db.transaction(['tensors'], 'readonly');
|
||||
const store = transaction.objectStore('tensors');
|
||||
const request = store.getAllKeys();
|
||||
transaction.onabort = (event) => {
|
||||
console.log("Transaction error while reading IndexedDB keys: " + event.target.error);
|
||||
resolve([]);
|
||||
};
|
||||
request.onsuccess = function (event) {resolve(event.target.result);};
|
||||
request.onerror = (event) => {
|
||||
console.error('Retrieval of IndexedDB keys failed: ', event.target.error);
|
||||
resolve([]);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// modified from examples/webgpu/stable_diffusion/index.html
|
||||
function saveTensorToDb(db, id, tensor) {
|
||||
return readTensorFromDb(db, id).then((result) => {
|
||||
if (!result) {
|
||||
new Promise((resolve, reject) => {
|
||||
if (db == null) {
|
||||
resolve(null);
|
||||
}
|
||||
|
||||
const transaction = db.transaction(['tensors'], 'readwrite');
|
||||
const store = transaction.objectStore('tensors');
|
||||
const request = store.put({ id: id, content: tensor });
|
||||
|
||||
transaction.onabort = (event) => {
|
||||
console.log("Transaction error while saving tensor: " + event.target.error);
|
||||
resolve(null);
|
||||
};
|
||||
|
||||
request.onsuccess = () => {
|
||||
console.log('Tensor saved successfully.');
|
||||
resolve();
|
||||
};
|
||||
|
||||
request.onerror = (event) => {
|
||||
console.error('Tensor save failed:', event.target.error);
|
||||
resolve(null);
|
||||
};
|
||||
});
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}).catch(()=> null);
|
||||
}
|
||||
|
||||
function deleteTensorFromDb(db, id) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (db == null) {
|
||||
console.error("Database is not initialized.");
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const transaction = db.transaction(['tensors'], 'readwrite');
|
||||
const store = transaction.objectStore('tensors');
|
||||
const request = store.delete(id);
|
||||
|
||||
transaction.oncomplete = () => {
|
||||
console.log(`Tensor with ID '${id}' deleted successfully.`);
|
||||
resolve();
|
||||
};
|
||||
|
||||
transaction.onerror = (event) => {
|
||||
console.error("Transaction error while deleting tensor:", event.target.error);
|
||||
resolve(null);
|
||||
};
|
||||
|
||||
request.onerror = (event) => {
|
||||
console.error('Tensor deletion failed:', event.target.error);
|
||||
resolve(null);
|
||||
};
|
||||
|
||||
request.onsuccess = () => {
|
||||
console.log(`Delete request for tensor with ID '${id}' succeeded.`);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function makeProgress(total) {
|
||||
let acc = 0;
|
||||
const ret = function progress(amount, message) {
|
||||
if (amount >= 0) { // allow updating message only
|
||||
acc += amount;
|
||||
const percentage = total ? Math.trunc((acc / total) * 100) : 0;
|
||||
document.querySelector('.progress').style.width = `${percentage}%`;
|
||||
document.getElementById('progress-percentage').textContent = `${percentage}%`;
|
||||
}
|
||||
if (message) {
|
||||
this.loadingMessage = message;
|
||||
document.getElementById('loading-message').textContent = this.loadingMessage;
|
||||
}
|
||||
}.bind(this);
|
||||
ret.total = total;
|
||||
return ret;
|
||||
}
|
||||
|
||||
function sendMessageToWorker(worker, message) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const onMessage = (event) => {
|
||||
resolve(event.data);
|
||||
worker.removeEventListener('message', onMessage);
|
||||
worker.removeEventListener('error', onError);
|
||||
};
|
||||
|
||||
const onError = (error) => {
|
||||
reject(error);
|
||||
worker.removeEventListener('message', onMessage);
|
||||
worker.removeEventListener('error', onError);
|
||||
};
|
||||
|
||||
worker.addEventListener('message', onMessage);
|
||||
worker.addEventListener('error', onError);
|
||||
|
||||
if (message.header === "token") worker.postMessage(message.data);
|
||||
else if (message.header === "load_state_dict") {
|
||||
if (message.data === "done") worker.postMessage(message.data);
|
||||
else worker.postMessage(message.data, message.data.map(file => file.bytes.buffer));
|
||||
}
|
||||
else if (message.header === "init") worker.postMessage("init");
|
||||
});
|
||||
}
|
||||
|
||||
async function load_state_dict (data, device, progress) {
|
||||
let state_dict = data.metadata.state_dict;
|
||||
let completed = 0;
|
||||
|
||||
// modified from examples/webgpu/stable_diffusion/index.html getProgressDlForPart
|
||||
const loadPart = async (part) => {
|
||||
const response = await fetch(part);
|
||||
const res = new Response(new ReadableStream({
|
||||
async start(controller) {
|
||||
const reader = response.body.getReader();
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
progress(value.byteLength);
|
||||
controller.enqueue(value);
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
}));
|
||||
|
||||
return res.arrayBuffer();
|
||||
};
|
||||
|
||||
let db = await initDb();
|
||||
|
||||
const getPart = async(filename, hash) => {
|
||||
let part = await readTensorFromDb(db, hash);
|
||||
|
||||
if (part) {
|
||||
console.log(`Cache hit: ${filename}, hash: ${hash}`);
|
||||
progress(part.content.byteLength);
|
||||
return Promise.resolve(part.content);
|
||||
} else {
|
||||
console.log(`Cache miss: ${filename}, hash: ${hash}`);
|
||||
return loadPart(`${window.MODEL_BASE_URL}/${filename}`);
|
||||
}
|
||||
}
|
||||
|
||||
const correctHashes = data.metadata.files.map(file => file.hash)
|
||||
// delete unused cached buffers to free disk space -- if we update weights, user will otherwise have obsolete cached buffers
|
||||
const dbKeys = await getAllKeysFromDb(db);
|
||||
const correctHashesSet = new Set(correctHashes);
|
||||
const notInCorrectHashes = dbKeys.filter(key => !correctHashesSet.has(key));
|
||||
// await these right before starting to save new stuff
|
||||
const deletionPromises = notInCorrectHashes.map(async (hash) => deleteTensorFromDb(db, hash));
|
||||
|
||||
// instantiates empty weight buffers on WebGPU, attaches buffers to state_dict
|
||||
let model;
|
||||
if (window.BACKEND === "WebGPU") {
|
||||
//model = await transformer().setup(device, state_dict, progress);
|
||||
model = await transformer.setupNet(device, state_dict);
|
||||
progress(0.15 * progress.total);
|
||||
|
||||
}
|
||||
else if (window.BACKEND === "WASM") {
|
||||
progress(0.02 * progress.total);
|
||||
model = new Worker(`./worker.js?version=${Date.now()}`);
|
||||
await sendMessageToWorker(model, {header: "init"});
|
||||
progress(0.02 * progress.total);
|
||||
progress(0.11 * progress.total);
|
||||
}
|
||||
|
||||
const downloaded = [];
|
||||
const triggerChainDownload = async (toDownload) => {
|
||||
const numDownloaders = window.isMobile ? 4 : toDownload.length; // TODO: dynamically base this on DL file size? current assumption is 16 MiB chunks
|
||||
|
||||
const chainDownload = async() => {
|
||||
const file = toDownload.shift();
|
||||
loadPart(`${window.MODEL_BASE_URL}/${file.name}`) // triggers download
|
||||
.then(async (arraybuf) => {
|
||||
downloaded.push({ ...file, bytes: new Uint8Array(arraybuf)});
|
||||
// pause downloads if further processing is a bottleneck
|
||||
while (toDownload.length && downloaded.length >= numDownloaders) await new Promise(resolve => setTimeout(resolve, 5));
|
||||
if (toDownload.length && downloaded.length < numDownloaders) chainDownload(); // start next download
|
||||
})
|
||||
}
|
||||
for (let i=0; i<numDownloaders; i++) if (toDownload.length) chainDownload();
|
||||
}
|
||||
|
||||
const loadFileToStateDict = async(file) => {
|
||||
if (window.BACKEND === "WebGPU") {
|
||||
for (const part of file.parts) {
|
||||
if (part.empty) continue;
|
||||
part.bytes = (part.size === file.bytes.length) ? file.bytes : file.bytes.slice(part.file_start_pos, part.file_start_pos + part.size);
|
||||
device.queue.writeBuffer(state_dict[part.key].bytes, part.target_start_pos, part.bytes); // improves stability over mappedAtCreation writing
|
||||
part.bytes = null;
|
||||
}
|
||||
}
|
||||
else if (window.BACKEND === "WASM") {
|
||||
await sendMessageToWorker(model, {header: "load_state_dict", data: [file]});
|
||||
}
|
||||
file.bytes = null;
|
||||
}
|
||||
|
||||
if (window.BACKEND === "WebGPU") { // contiguous loading not needed for WebGPU stability
|
||||
const files = data.tensor_file_groups.flatMap(obj => obj.files);
|
||||
data.tensor_file_groups = [{contiguous: false, files: files}];
|
||||
}
|
||||
|
||||
for (const group of data.tensor_file_groups) {
|
||||
const contiguous = group.contiguous;
|
||||
const files = group.files;
|
||||
const tensor_file_indices = files.map(file => file.index);
|
||||
const contiguousFiles = [];
|
||||
const fileHashes = new Set(files.map(file => file.hash));
|
||||
const cachedFileHashes = new Set(dbKeys.filter(key => fileHashes.has(key)));
|
||||
const cachedFiles = files.filter(file => cachedFileHashes.has(file.hash));
|
||||
const toDownload = files.filter(file => !cachedFileHashes.has(file.hash));
|
||||
triggerChainDownload(toDownload);
|
||||
|
||||
const loadDelay = 5;
|
||||
await Promise.all(deletionPromises);
|
||||
|
||||
while (completed < files.length) {
|
||||
const start = performance.now();
|
||||
// prioritize files from downloaded queue, so we can continue downloading more files
|
||||
if (downloaded.length) {
|
||||
const file = downloaded.shift();
|
||||
await saveTensorToDb(db, file.hash, file.bytes); // for wasm, must await to prevent race between indexedDB and transfer to worker
|
||||
if (!contiguous) await loadFileToStateDict(file);
|
||||
else contiguousFiles.push(file);
|
||||
completed += 1;
|
||||
}
|
||||
else if (!downloaded.length && cachedFiles.length) {
|
||||
const file = cachedFiles.shift();
|
||||
file.bytes = await getPart(file.name, file.hash); // reads data from IndexedDB
|
||||
if (!contiguous) await loadFileToStateDict(file);
|
||||
else contiguousFiles.push(file);
|
||||
completed += 1;
|
||||
}
|
||||
const end = performance.now();
|
||||
const elapsed = end - start;
|
||||
if (elapsed < loadDelay) await new Promise(resolve => setTimeout(resolve, loadDelay - elapsed));
|
||||
}
|
||||
if (contiguous) {
|
||||
const orderMap = tensor_file_indices.reduce((acc, id, index) => {acc[id] = index; return acc;}, {});
|
||||
contiguousFiles.sort((a, b) => orderMap[a.index] - orderMap[b.index]); // glue files together in the right order
|
||||
await sendMessageToWorker(model, {header: "load_state_dict", data: contiguousFiles});
|
||||
}
|
||||
completed = 0;
|
||||
}
|
||||
|
||||
// initialize empty kv_caches, which were part of exported model's state_dict, but which we didn't want to package/download
|
||||
if (window.BACKEND === "WASM") {
|
||||
for (const [k, v] of Object.entries(state_dict).filter(([_, v]) => v.empty === true)) {
|
||||
v.parts[0].file_start_pos = 0;
|
||||
const file = { parts: v.parts, size: v.size, bytes: new Uint8Array(v.size).fill(0) };
|
||||
await loadFileToStateDict(file);
|
||||
}
|
||||
}
|
||||
|
||||
return model;
|
||||
};
|
||||
|
||||
document.addEventListener("alpine:init", () => {
|
||||
Alpine.data("state", () => ({
|
||||
// loadingMessage updates the user on page load progress, including weights download and decompression
|
||||
// if loadingMessage is not '', then prompt box will be hidden: this is default behavior on page load
|
||||
placeholderText: "Generating...",
|
||||
loadingMessage: `Loading ${window.BACKEND} model:`,
|
||||
// model
|
||||
nets: {},
|
||||
tokenizer: null,
|
||||
max_context: 1024,
|
||||
lastSeenToks: [],
|
||||
|
||||
progress: null,
|
||||
|
||||
async init() {
|
||||
var device = null;
|
||||
var webgpuErrorMessage = null;
|
||||
if (window.BACKEND === "WebGPU") {
|
||||
try {
|
||||
device = await getDevice.call(this);
|
||||
console.log("WebGPU device initialized");
|
||||
} catch (error) {
|
||||
window.BACKEND = "WASM";
|
||||
console.log(`error: ${error}\nFailed to launch WebGPU. Loading WASM model instead...`); // return;
|
||||
webgpuErrorMessage = this.loadingMessage;
|
||||
}
|
||||
}
|
||||
|
||||
window.MODEL_BASE_URL = (window.BACKEND === "WebGPU" && !window.isMobile) ? window.PC_MODEL_BASE_URL : window.MOBILE_MODEL_BASE_URL;
|
||||
this.max_context = (window.BACKEND === "WebGPU" && !window.isMobile) ? window.PC_MAX_CONTEXT : window.MOBILE_MAX_CONTEXT;
|
||||
|
||||
const kernelsReady = (async () => {
|
||||
if (window.BACKEND === "WASM") {var exports = await import(`./net_clang.js?version=${Date.now()}`);}
|
||||
else if (window.BACKEND === "WebGPU" && !window.isMobile) {var exports = await import(`${PC_WEBGPU_EXPORT}?version=${Date.now()}`);}
|
||||
else if (window.BACKEND === "WebGPU" && window.isMobile) {var exports = await import(`${MOBILE_WEBGPU_EXPORT}?version=${Date.now()}`);}
|
||||
self.transformer = exports.default;
|
||||
})();
|
||||
|
||||
const response = await fetch(`${window.MODEL_BASE_URL}/net_metadata.json`);
|
||||
// TODO: cache metadata (and everything else, including tokenizer)
|
||||
// TODO: use service worker to reload page when offline
|
||||
const data = await response.json();
|
||||
data.metadata.files = data.metadata.files.map((file, index) => ({...file, index}));
|
||||
const state_dict = data.metadata.state_dict;
|
||||
|
||||
/*
|
||||
- allocating memory to WASM on mobile has longstanding issues: https://github.com/WebAssembly/design/issues/1397
|
||||
|
||||
- the below pattern, while yielding a succesfully-functioning model when it doesn't crash, causes regular crashes on iOS Safari (iphone 15 iOS 18.3):
|
||||
- call WASM malloc (to fit all tensors, or one per tensor) for all tensors up front, then load tensor byte chunks into the buffers in random order
|
||||
|
||||
- the below pattern has been stable on iOS Safari (iphone 15 iOS 18.3):
|
||||
- call only one WASM malloc at a time before filling the allocated bytes, as small as possible (malloc up to 256 MiB has been tested)
|
||||
- fill the malloc'd memory in linear order from start to end (what has been tested is calling wasm.HEAPU8.set on 16 MiB chunks from start to end)
|
||||
- use ALLOW_MEMORY_GROWTH=1 in wasm compilation, minimize initial memory
|
||||
|
||||
- additional considerations affecting loading design, for WASM:
|
||||
- it seems that copying bytes into wasm memory cannot be zero-copy without sharedarraybuffer, which isn't currently used due to increased hosting complexity
|
||||
- non-zero copies create memory pressure, which is not reliably capped because of lack of control over garbage collection
|
||||
- to minimize peak memory pressure if GC is delayed, we process (i.e. download + copy into WASM) large tensors (> 16 MiB) one at a time, in descending size order
|
||||
*/
|
||||
data.tensor_file_groups = []; // see above: for WASM, limit processing of multi-file Tensors to one at a time, in descending order based on Tensor size
|
||||
const unsplit_tensors = [];
|
||||
const sortedEntries = Object.entries(state_dict).sort(([, objA], [, objB]) => objB.size - objA.size);
|
||||
|
||||
let totalSize = 0;
|
||||
const seen = new Set();
|
||||
for (const [k,v] of sortedEntries) {
|
||||
const files_in_tensor = [];
|
||||
for (const part of v.parts) {
|
||||
part.key = k;
|
||||
if (part.empty) state_dict[k].empty = true; // assumes no other parts of this weight exist and are non-empty
|
||||
else {
|
||||
const file = data.metadata.files[part.file];
|
||||
if (!seen.has(file.index)) {
|
||||
seen.add(file.index);
|
||||
files_in_tensor.push(file);
|
||||
}
|
||||
totalSize += part.size;
|
||||
part.dtype = v.dtype;
|
||||
if (!data.metadata.files[part.file].parts) data.metadata.files[part.file].parts = [];
|
||||
data.metadata.files[part.file].size ??= 0;
|
||||
data.metadata.files[part.file].size += part.size;
|
||||
data.metadata.files[part.file].parts.push(part);
|
||||
}
|
||||
}
|
||||
if (files_in_tensor.length > 1) data.tensor_file_groups.push({contiguous: true, files: files_in_tensor}); // [tensorN_file0, tensorN_file1, ...]
|
||||
else if (files_in_tensor.length > 0) unsplit_tensors.push(files_in_tensor[0]);
|
||||
}
|
||||
data.tensor_file_groups.push({contiguous: false, files: unsplit_tensors});
|
||||
|
||||
data.totalSize = totalSize;
|
||||
totalSize = totalSize / 0.8; // give space in progress bar for initializing model bufs, and tokenizer
|
||||
this.progress = makeProgress.call(this, totalSize); // creates closure with totalSize
|
||||
|
||||
try {
|
||||
this.progress(0.01 * totalSize, "Loading tokenizer:");
|
||||
const wasmResponse = await fetch(`${window.MODEL_BASE_URL}/tiktoken_bg.wasm`);
|
||||
this.progress(0.01 * totalSize);
|
||||
const wasmBytes = await wasmResponse.arrayBuffer();
|
||||
await tiktokenReady;
|
||||
await window.tiktokenInit((imports) => WebAssembly.instantiate(wasmBytes, imports));
|
||||
this.progress(0.01 * totalSize);
|
||||
|
||||
this.tokenizer = await createTokenizer(`${window.MODEL_BASE_URL}/llama3-2.tiktoken`);
|
||||
const tokenizer_works = (new TextDecoder().decode(this.tokenizer.decode(this.tokenizer.encode("hello world"))) === "hello world");
|
||||
console.log("tokenizer works:", tokenizer_works)
|
||||
this.progress(0.01 * totalSize);
|
||||
} catch (error) {this.progress(-1, `Error launching tokenizer: ${error}`); console.log(error); return;}
|
||||
|
||||
try {
|
||||
const loadModelMessage = (webgpuErrorMessage) ? webgpuErrorMessage : `Loading ${window.BACKEND} model:`
|
||||
this.progress(0, loadModelMessage);
|
||||
await kernelsReady;
|
||||
const model = await load_state_dict(data, device, this.progress);
|
||||
|
||||
if (window.BACKEND === "WebGPU") {
|
||||
this.nets = {"transformer": model};
|
||||
}
|
||||
else if (window.BACKEND === "WASM") {
|
||||
const msg = await sendMessageToWorker(model, {header: "load_state_dict", data: "done"});
|
||||
this.nets = {"transformer": async (tok, start_pos) => sendMessageToWorker(model, {header: "token", data: [tok, start_pos]})};
|
||||
}
|
||||
this.progress(0.01 * totalSize, `Launching ${window.BACKEND} model:`);
|
||||
this.loadingMessage = ""; // Triggers removal of loading bar, display of prompt box
|
||||
} catch (error) {this.progress(-1, `Error launching model: ${error}`); console.log(error); return;}
|
||||
},
|
||||
|
||||
// current state
|
||||
cstate: {
|
||||
time: null,
|
||||
messages: [],
|
||||
},
|
||||
|
||||
// historical state
|
||||
histories: JSON.parse(localStorage.getItem("histories")) || [],
|
||||
|
||||
home: 0,
|
||||
generating: false,
|
||||
maxContextReached: false,
|
||||
cancelGeneration: false,
|
||||
endpoint: `${window.location.origin}/v1`,
|
||||
|
||||
// performance tracking
|
||||
time_till_first: 0,
|
||||
tokens_per_second: 0,
|
||||
total_tokens: 0,
|
||||
max_context: 0,
|
||||
|
||||
removeHistory(cstate) {
|
||||
const index = this.histories.findIndex((state) => {
|
||||
return state.time === cstate.time;
|
||||
});
|
||||
if (index !== -1) {
|
||||
this.histories.splice(index, 1);
|
||||
localStorage.setItem("histories", JSON.stringify(this.histories));
|
||||
}
|
||||
},
|
||||
|
||||
async handleSend() {
|
||||
const el = document.getElementById("input-form");
|
||||
const value = el.value.trim();
|
||||
if (!value) return;
|
||||
|
||||
if (this.generating) return;
|
||||
this.maxContextReached = false;
|
||||
this.placeholderText = "Generating...";
|
||||
this.generating = true;
|
||||
this.cancelGeneration = false;
|
||||
if (this.home === 0) this.home = 1;
|
||||
|
||||
// ensure that going back in history will go back to home
|
||||
window.history.pushState({}, "", window.TINYCHAT_ROOT || "/");
|
||||
|
||||
// add message to list
|
||||
this.cstate.messages.push({ role: "user", content: value });
|
||||
|
||||
// clear textarea
|
||||
el.value = "";
|
||||
el.style.height = "auto";
|
||||
el.style.height = el.scrollHeight + "px";
|
||||
|
||||
// reset performance tracking
|
||||
const prefill_start = Date.now();
|
||||
let start_time = 0;
|
||||
let tokens = 0;
|
||||
this.tokens_per_second = 0;
|
||||
|
||||
let gottenFirstChunk = false;
|
||||
try {
|
||||
for await (
|
||||
const chunk of this.openaiChatCompletion(this.cstate.messages)
|
||||
) {
|
||||
if (!gottenFirstChunk) {
|
||||
this.cstate.messages.push({ role: "assistant", content: "" });
|
||||
gottenFirstChunk = true;
|
||||
}
|
||||
|
||||
// add chunk to the last message
|
||||
// TODO: handle localStorage overflow
|
||||
// possible example: this.cstate.messages[...] was undefined when trying to prompt within an old cstate (chat session)
|
||||
this.cstate.messages[this.cstate.messages.length - 1].content += chunk;
|
||||
|
||||
// calculate performance tracking
|
||||
tokens += 1;
|
||||
this.total_tokens += 1;
|
||||
if (start_time === 0) {
|
||||
start_time = Date.now();
|
||||
this.time_till_first = start_time - prefill_start;
|
||||
} else {
|
||||
const diff = Date.now() - start_time;
|
||||
if (diff > 0) {
|
||||
this.tokens_per_second = tokens / (diff / 1000);
|
||||
}
|
||||
}
|
||||
this.checkMaxContext(this.total_tokens);
|
||||
if (this.cancelGeneration) break;
|
||||
}
|
||||
} finally {
|
||||
// update the state in histories or add it if it doesn't exist
|
||||
const index = this.histories.findIndex((cstate) => {
|
||||
return cstate.time === this.cstate.time;
|
||||
});
|
||||
this.cstate.time = Date.now();
|
||||
if (index !== -1) {
|
||||
// update the time
|
||||
this.histories[index] = this.cstate;
|
||||
} else {
|
||||
this.histories.push(this.cstate);
|
||||
}
|
||||
// update in local storage
|
||||
localStorage.setItem("histories", JSON.stringify(this.histories));
|
||||
|
||||
if (!this.maxContextReached) this.generating = false;
|
||||
if (this.cancelGeneration && !this.maxContextReached) this.cstate = { time: null, messages: [] };
|
||||
}
|
||||
},
|
||||
|
||||
async handleEnter(event) {
|
||||
// if shift is not pressed
|
||||
if (!event.shiftKey) {
|
||||
event.preventDefault();
|
||||
await this.handleSend();
|
||||
}
|
||||
},
|
||||
|
||||
updateTotalTokens(messages) {
|
||||
try {
|
||||
let toks = [this.tokenizer.bos_id];
|
||||
messages.forEach((message) => {
|
||||
if (!message.role || !message.content) {
|
||||
throw new Error("Each message must have a 'role' and 'content' property.");
|
||||
}
|
||||
toks = toks.concat(this.tokenizer.encodeMessage(message.role, message.content));
|
||||
|
||||
if (messages.length > 0 && messages[messages.length - 1].role === "user") {
|
||||
toks = toks.concat(this.tokenizer.encodeRole("assistant"));
|
||||
}
|
||||
this.total_tokens = toks.length;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Error updating total tokens:", error);
|
||||
}
|
||||
},
|
||||
|
||||
checkMaxContext(num_tokens) {
|
||||
if (num_tokens >= this.max_context) {
|
||||
this.cancelGeneration = true;
|
||||
this.maxContextReached = true;
|
||||
this.placeholderText = `Max context reached: ${this.max_context} tokens`;
|
||||
}
|
||||
},
|
||||
|
||||
async *openaiChatCompletion(messages) {
|
||||
let tokens = [this.tokenizer.bos_id];
|
||||
for (const message of messages) {
|
||||
tokens = tokens.concat(this.tokenizer.encodeMessage(message.role, message.content));
|
||||
}
|
||||
tokens = tokens.concat(this.tokenizer.encodeRole("assistant"));
|
||||
this.checkMaxContext(tokens.length); // don't waste time prefilling if we know we're over the token limit
|
||||
let startPos = 0
|
||||
const prefillToks = tokens.slice(0, -1);
|
||||
|
||||
// Skip the largest possible sequence of tokens already represented at the beginning of the model's kv caches
|
||||
for (let i=0; i <= prefillToks.length; i++) {
|
||||
startPos = i;
|
||||
if (i == prefillToks.length) break;
|
||||
if (i == this.lastSeenToks.length) break;
|
||||
if (prefillToks[i] !== this.lastSeenToks[i]) break;
|
||||
}
|
||||
//this.lastSeenToks = prefillToks;
|
||||
//prefillToks = prefillToks.slice(startPos);
|
||||
const unprocessedPrefillToks = prefillToks.slice(startPos);
|
||||
this.lastSeenToks = prefillToks.slice(0, startPos);
|
||||
|
||||
this.progress = makeProgress(unprocessedPrefillToks.length);
|
||||
this.loadingMessage = (window.BACKEND === "WebGPU") ? "Reading input:" : "Loading (enable WebGPU for speed):";
|
||||
this.progress(0, this.loadingMessage);
|
||||
for (const tok of unprocessedPrefillToks) {
|
||||
if (this.cancelGeneration) {this.loadingMessage=""; return;}
|
||||
if (window.BACKEND === "WebGPU") {await this.nets["transformer"](new Int32Array([tok]), new Int32Array([startPos]));}
|
||||
else {await this.nets["transformer"](tok, startPos);}
|
||||
this.lastSeenToks.push(tok)
|
||||
startPos += 1;
|
||||
this.progress(1);
|
||||
}
|
||||
this.loadingMessage = ""; // hides progress bar
|
||||
|
||||
let lastTok = tokens[tokens.length - 1];
|
||||
while (true) {
|
||||
if (window.BACKEND === "WebGPU") {var tok = await this.nets["transformer"](new Int32Array([lastTok]), new Int32Array([startPos])); tok = tok[0][0];}
|
||||
else {var tok = await this.nets["transformer"](lastTok, startPos);}
|
||||
this.lastSeenToks.push(lastTok); // lets us skip prefilling with these tokens at the next prompt in this chain
|
||||
startPos += 1;
|
||||
lastTok = tok;
|
||||
if (this.tokenizer.stop_tokens.has(lastTok)) break;
|
||||
yield new TextDecoder().decode(this.tokenizer.decode([lastTok]));
|
||||
}
|
||||
},
|
||||
}));
|
||||
});
|
||||
|
||||
const { markedHighlight } = globalThis.markedHighlight;
|
||||
marked.use(markedHighlight({
|
||||
langPrefix: "hljs language-",
|
||||
highlight(code, lang, _info) {
|
||||
const language = hljs.getLanguage(lang) ? lang : "plaintext";
|
||||
return hljs.highlight(code, { language }).value;
|
||||
},
|
||||
}));
|
||||
|
||||
// **** eventsource-parser ****
|
||||
class EventSourceParserStream extends TransformStream {
|
||||
constructor() {
|
||||
let parser;
|
||||
|
||||
super({
|
||||
start(controller) {
|
||||
parser = createParser((event) => {
|
||||
if (event.type === "event") {
|
||||
controller.enqueue(event);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
transform(chunk) {
|
||||
parser.feed(chunk);
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function createParser(onParse) {
|
||||
let isFirstChunk;
|
||||
let buffer;
|
||||
let startingPosition;
|
||||
let startingFieldLength;
|
||||
let eventId;
|
||||
let eventName;
|
||||
let data;
|
||||
reset();
|
||||
return {
|
||||
feed,
|
||||
reset,
|
||||
};
|
||||
function reset() {
|
||||
isFirstChunk = true;
|
||||
buffer = "";
|
||||
startingPosition = 0;
|
||||
startingFieldLength = -1;
|
||||
eventId = void 0;
|
||||
eventName = void 0;
|
||||
data = "";
|
||||
}
|
||||
function feed(chunk) {
|
||||
buffer = buffer ? buffer + chunk : chunk;
|
||||
if (isFirstChunk && hasBom(buffer)) {
|
||||
buffer = buffer.slice(BOM.length);
|
||||
}
|
||||
isFirstChunk = false;
|
||||
const length = buffer.length;
|
||||
let position = 0;
|
||||
let discardTrailingNewline = false;
|
||||
while (position < length) {
|
||||
if (discardTrailingNewline) {
|
||||
if (buffer[position] === "\n") {
|
||||
++position;
|
||||
}
|
||||
discardTrailingNewline = false;
|
||||
}
|
||||
let lineLength = -1;
|
||||
let fieldLength = startingFieldLength;
|
||||
let character;
|
||||
for (
|
||||
let index = startingPosition;
|
||||
lineLength < 0 && index < length;
|
||||
++index
|
||||
) {
|
||||
character = buffer[index];
|
||||
if (character === ":" && fieldLength < 0) {
|
||||
fieldLength = index - position;
|
||||
} else if (character === "\r") {
|
||||
discardTrailingNewline = true;
|
||||
lineLength = index - position;
|
||||
} else if (character === "\n") {
|
||||
lineLength = index - position;
|
||||
}
|
||||
}
|
||||
if (lineLength < 0) {
|
||||
startingPosition = length - position;
|
||||
startingFieldLength = fieldLength;
|
||||
break;
|
||||
} else {
|
||||
startingPosition = 0;
|
||||
startingFieldLength = -1;
|
||||
}
|
||||
parseEventStreamLine(buffer, position, fieldLength, lineLength);
|
||||
position += lineLength + 1;
|
||||
}
|
||||
if (position === length) {
|
||||
buffer = "";
|
||||
} else if (position > 0) {
|
||||
buffer = buffer.slice(position);
|
||||
}
|
||||
}
|
||||
function parseEventStreamLine(lineBuffer, index, fieldLength, lineLength) {
|
||||
if (lineLength === 0) {
|
||||
if (data.length > 0) {
|
||||
onParse({
|
||||
type: "event",
|
||||
id: eventId,
|
||||
event: eventName || void 0,
|
||||
data: data.slice(0, -1),
|
||||
// remove trailing newline
|
||||
});
|
||||
|
||||
data = "";
|
||||
eventId = void 0;
|
||||
}
|
||||
eventName = void 0;
|
||||
return;
|
||||
}
|
||||
const noValue = fieldLength < 0;
|
||||
const field = lineBuffer.slice(
|
||||
index,
|
||||
index + (noValue ? lineLength : fieldLength),
|
||||
);
|
||||
let step = 0;
|
||||
if (noValue) {
|
||||
step = lineLength;
|
||||
} else if (lineBuffer[index + fieldLength + 1] === " ") {
|
||||
step = fieldLength + 2;
|
||||
} else {
|
||||
step = fieldLength + 1;
|
||||
}
|
||||
const position = index + step;
|
||||
const valueLength = lineLength - step;
|
||||
const value = lineBuffer.slice(position, position + valueLength).toString();
|
||||
if (field === "data") {
|
||||
data += value ? "".concat(value, "\n") : "\n";
|
||||
} else if (field === "event") {
|
||||
eventName = value;
|
||||
} else if (field === "id" && !value.includes("\0")) {
|
||||
eventId = value;
|
||||
} else if (field === "retry") {
|
||||
const retry = parseInt(value, 10);
|
||||
if (!Number.isNaN(retry)) {
|
||||
onParse({
|
||||
type: "reconnect-interval",
|
||||
value: retry,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const BOM = [239, 187, 191];
|
||||
function hasBom(buffer) {
|
||||
return BOM.every((charCode, index) => buffer.charCodeAt(index) === charCode);
|
||||
}
|
||||
|
||||
const PAT_STR = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";
|
||||
|
||||
async function createTokenizer(bpeUrl) {
|
||||
const num_base_tokens = 128000;
|
||||
const special_tokens = {
|
||||
"<|begin_of_text|>": 128000,
|
||||
"<|end_of_text|>": 128001,
|
||||
"<|start_header_id|>": 128006,
|
||||
"<|end_header_id|>": 128007,
|
||||
"<|eot_id|>": 128009
|
||||
};
|
||||
const model = await window.tiktokenLoad({
|
||||
"load_tiktoken_bpe": bpeUrl,
|
||||
"special_tokens": special_tokens,
|
||||
"pat_str": PAT_STR
|
||||
});
|
||||
const tokenizer = new window.Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str)
|
||||
|
||||
return {
|
||||
get bos_id() {
|
||||
return special_tokens["<|begin_of_text|>"];
|
||||
},
|
||||
|
||||
get stop_tokens() {
|
||||
return new Set([
|
||||
special_tokens["<|end_of_text|>"],
|
||||
special_tokens["<|eot_id|>"],
|
||||
]);
|
||||
},
|
||||
|
||||
decode(toks) {
|
||||
const filtered = toks.filter((t) => t < num_base_tokens);
|
||||
return tokenizer.decode(filtered);
|
||||
},
|
||||
|
||||
encode(text, allow_special = false) {
|
||||
const allowedSpecial = allow_special ? "all" : new Set();
|
||||
const disallowedSpecial = new Set();
|
||||
return tokenizer.encode(text, allowedSpecial, disallowedSpecial);
|
||||
},
|
||||
|
||||
encodeRole(role) {
|
||||
const tokens = [];
|
||||
tokens.push(special_tokens["<|start_header_id|>"]);
|
||||
tokens.push(...this.encode(role));
|
||||
tokens.push(special_tokens["<|end_header_id|>"]);
|
||||
tokens.push(...this.encode("\n\n"));
|
||||
return tokens;
|
||||
},
|
||||
|
||||
encodeMessage(role, content) {
|
||||
const roleTokens = this.encodeRole(role);
|
||||
const contentTokens = this.encode(content.trim());
|
||||
return [...roleTokens, ...contentTokens, special_tokens["<|eot_id|>"]];
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user