Skip to content

Commit 44767bc

Browse files
ngxsonmglambda
authored andcommitted
server : (webui) revamp Settings dialog, add Pyodide interpreter (ggml-org#11759)
* redo Settings modal UI * add python code interpreter * fix auto scroll * build * fix overflow for long output lines * bring back sticky copy button * adapt layout on mobile view * fix multiple lines output and color scheme * handle python exception * better state management * add webworker * add headers * format code * speed up by loading pyodide on page load * (small tweak) add small animation to make it feels like claude
1 parent edd8185 commit 44767bc

18 files changed

+869
-268
lines changed

examples/server/public/index.html.gz

4.81 KB
Binary file not shown.

examples/server/server.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4378,6 +4378,9 @@ int main(int argc, char ** argv) {
43784378
res.set_content("Error: gzip is not supported by this browser", "text/plain");
43794379
} else {
43804380
res.set_header("Content-Encoding", "gzip");
4381+
// COEP and COOP headers, required by pyodide (python interpreter)
4382+
res.set_header("Cross-Origin-Embedder-Policy", "require-corp");
4383+
res.set_header("Cross-Origin-Opener-Policy", "same-origin");
43814384
res.set_content(reinterpret_cast<const char*>(index_html_gz), index_html_gz_len, "text/html; charset=utf-8");
43824385
}
43834386
return false;

examples/server/webui/package-lock.json

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

examples/server/webui/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"preview": "vite preview"
1212
},
1313
"dependencies": {
14+
"@heroicons/react": "^2.2.0",
1415
"@sec-ant/readable-stream": "^0.6.0",
1516
"@vscode/markdown-it-katex": "^1.1.1",
1617
"autoprefixer": "^10.4.20",

examples/server/webui/src/App.tsx

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import { HashRouter, Outlet, Route, Routes } from 'react-router';
22
import Header from './components/Header';
33
import Sidebar from './components/Sidebar';
4-
import { AppContextProvider } from './utils/app.context';
4+
import { AppContextProvider, useAppContext } from './utils/app.context';
55
import ChatScreen from './components/ChatScreen';
6+
import SettingDialog from './components/SettingDialog';
67

78
function App() {
89
return (
@@ -22,13 +23,23 @@ function App() {
2223
}
2324

2425
function AppLayout() {
26+
const { showSettings, setShowSettings } = useAppContext();
2527
return (
2628
<>
2729
<Sidebar />
28-
<div className="chat-screen drawer-content grow flex flex-col h-screen w-screen mx-auto px-4">
30+
<div
31+
className="drawer-content grow flex flex-col h-screen w-screen mx-auto px-4 overflow-auto"
32+
id="main-scroll"
33+
>
2934
<Header />
3035
<Outlet />
3136
</div>
37+
{
38+
<SettingDialog
39+
show={showSettings}
40+
onClose={() => setShowSettings(false)}
41+
/>
42+
}
3243
</>
3344
);
3445
}

examples/server/webui/src/Config.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export const BASE_URL = new URL('.', document.baseURI).href
1010

1111
export const CONFIG_DEFAULT = {
1212
// Note: in order not to introduce breaking changes, please keep the same data type (number, string, etc) if you want to change the default value. Do not use null or undefined for default value.
13+
// Do not use nested objects, keep it single level. Prefix the key if you need to group them.
1314
apiKey: '',
1415
systemMessage: 'You are a helpful assistant.',
1516
showTokensPerSecond: false,
@@ -36,6 +37,8 @@ export const CONFIG_DEFAULT = {
3637
dry_penalty_last_n: -1,
3738
max_tokens: -1,
3839
custom: '', // custom json-stringified object
40+
// experimental features
41+
pyIntepreterEnabled: false,
3942
};
4043
export const CONFIG_INFO: Record<string, string> = {
4144
apiKey: 'Set the API Key if you are using --api-key option for the server.',
Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
import { useEffect, useState } from 'react';
2+
import { useAppContext } from '../utils/app.context';
3+
import { OpenInNewTab, XCloseButton } from '../utils/common';
4+
import { CanvasType } from '../utils/types';
5+
import { PlayIcon, StopIcon } from '@heroicons/react/24/outline';
6+
import StorageUtils from '../utils/storage';
7+
8+
const canInterrupt = typeof SharedArrayBuffer === 'function';
9+
10+
// adapted from https://pyodide.org/en/stable/usage/webworker.html
11+
const WORKER_CODE = `
12+
importScripts("https://cdn.jsdelivr.net/pyodide/v0.27.2/full/pyodide.js");
13+
14+
let stdOutAndErr = [];
15+
16+
let pyodideReadyPromise = loadPyodide({
17+
stdout: (data) => stdOutAndErr.push(data),
18+
stderr: (data) => stdOutAndErr.push(data),
19+
});
20+
21+
let alreadySetBuff = false;
22+
23+
self.onmessage = async (event) => {
24+
stdOutAndErr = [];
25+
26+
// make sure loading is done
27+
const pyodide = await pyodideReadyPromise;
28+
const { id, python, context, interruptBuffer } = event.data;
29+
30+
if (interruptBuffer && !alreadySetBuff) {
31+
pyodide.setInterruptBuffer(interruptBuffer);
32+
alreadySetBuff = true;
33+
}
34+
35+
// Now load any packages we need, run the code, and send the result back.
36+
await pyodide.loadPackagesFromImports(python);
37+
38+
// make a Python dictionary with the data from content
39+
const dict = pyodide.globals.get("dict");
40+
const globals = dict(Object.entries(context));
41+
try {
42+
self.postMessage({ id, running: true });
43+
// Execute the python code in this context
44+
const result = pyodide.runPython(python, { globals });
45+
self.postMessage({ result, id, stdOutAndErr });
46+
} catch (error) {
47+
self.postMessage({ error: error.message, id });
48+
}
49+
interruptBuffer[0] = 0;
50+
};
51+
`;
52+
53+
let worker: Worker;
54+
const interruptBuffer = canInterrupt
55+
? new Uint8Array(new SharedArrayBuffer(1))
56+
: null;
57+
58+
const startWorker = () => {
59+
if (!worker) {
60+
worker = new Worker(
61+
URL.createObjectURL(new Blob([WORKER_CODE], { type: 'text/javascript' }))
62+
);
63+
}
64+
};
65+
66+
if (StorageUtils.getConfig().pyIntepreterEnabled) {
67+
startWorker();
68+
}
69+
70+
const runCodeInWorker = (
71+
pyCode: string,
72+
callbackRunning: () => void
73+
): {
74+
donePromise: Promise<string>;
75+
interrupt: () => void;
76+
} => {
77+
startWorker();
78+
const id = Math.random() * 1e8;
79+
const context = {};
80+
if (interruptBuffer) {
81+
interruptBuffer[0] = 0;
82+
}
83+
84+
const donePromise = new Promise<string>((resolve) => {
85+
worker.onmessage = (event) => {
86+
const { error, stdOutAndErr, running } = event.data;
87+
if (id !== event.data.id) return;
88+
if (running) {
89+
callbackRunning();
90+
return;
91+
} else if (error) {
92+
resolve(error.toString());
93+
} else {
94+
resolve(stdOutAndErr.join('\n'));
95+
}
96+
};
97+
worker.postMessage({ id, python: pyCode, context, interruptBuffer });
98+
});
99+
100+
const interrupt = () => {
101+
console.log('Interrupting...');
102+
console.trace();
103+
if (interruptBuffer) {
104+
interruptBuffer[0] = 2;
105+
}
106+
};
107+
108+
return { donePromise, interrupt };
109+
};
110+
111+
export default function CanvasPyInterpreter() {
112+
const { canvasData, setCanvasData } = useAppContext();
113+
114+
const [code, setCode] = useState(canvasData?.content ?? ''); // copy to avoid direct mutation
115+
const [running, setRunning] = useState(false);
116+
const [output, setOutput] = useState('');
117+
const [interruptFn, setInterruptFn] = useState<() => void>();
118+
const [showStopBtn, setShowStopBtn] = useState(false);
119+
120+
const runCode = async (pycode: string) => {
121+
interruptFn?.();
122+
setRunning(true);
123+
setOutput('Loading Pyodide...');
124+
const { donePromise, interrupt } = runCodeInWorker(pycode, () => {
125+
setOutput('Running...');
126+
setShowStopBtn(canInterrupt);
127+
});
128+
setInterruptFn(() => interrupt);
129+
const out = await donePromise;
130+
setOutput(out);
131+
setRunning(false);
132+
setShowStopBtn(false);
133+
};
134+
135+
// run code on mount
136+
useEffect(() => {
137+
setCode(canvasData?.content ?? '');
138+
runCode(canvasData?.content ?? '');
139+
// eslint-disable-next-line react-hooks/exhaustive-deps
140+
}, [canvasData?.content]);
141+
142+
if (canvasData?.type !== CanvasType.PY_INTERPRETER) {
143+
return null;
144+
}
145+
146+
return (
147+
<div className="card bg-base-200 w-full h-full shadow-xl">
148+
<div className="card-body">
149+
<div className="flex justify-between items-center mb-4">
150+
<span className="text-lg font-bold">Python Interpreter</span>
151+
<XCloseButton
152+
className="bg-base-100"
153+
onClick={() => setCanvasData(null)}
154+
/>
155+
</div>
156+
<div className="grid grid-rows-3 gap-4 h-full">
157+
<textarea
158+
className="textarea textarea-bordered w-full h-full font-mono"
159+
value={code}
160+
onChange={(e) => setCode(e.target.value)}
161+
></textarea>
162+
<div className="font-mono flex flex-col row-span-2">
163+
<div className="flex items-center mb-2">
164+
<button
165+
className="btn btn-sm bg-base-100"
166+
onClick={() => runCode(code)}
167+
disabled={running}
168+
>
169+
<PlayIcon className="h-6 w-6" /> Run
170+
</button>
171+
{showStopBtn && (
172+
<button
173+
className="btn btn-sm bg-base-100 ml-2"
174+
onClick={() => interruptFn?.()}
175+
>
176+
<StopIcon className="h-6 w-6" /> Stop
177+
</button>
178+
)}
179+
<span className="grow text-right text-xs">
180+
<OpenInNewTab href="https://github.com/ggerganov/llama.cpp/issues/11762">
181+
Report a bug
182+
</OpenInNewTab>
183+
</span>
184+
</div>
185+
<textarea
186+
className="textarea textarea-bordered h-full dark-color"
187+
value={output}
188+
readOnly
189+
></textarea>
190+
</div>
191+
</div>
192+
</div>
193+
</div>
194+
);
195+
}

examples/server/webui/src/components/ChatMessage.tsx

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,11 +149,17 @@ export default function ChatMessage({
149149
)}
150150
</summary>
151151
<div className="collapse-content">
152-
<MarkdownDisplay content={thought} />
152+
<MarkdownDisplay
153+
content={thought}
154+
isGenerating={isPending}
155+
/>
153156
</div>
154157
</details>
155158
)}
156-
<MarkdownDisplay content={content} />
159+
<MarkdownDisplay
160+
content={content}
161+
isGenerating={isPending}
162+
/>
157163
</div>
158164
</>
159165
)}

0 commit comments

Comments
 (0)