enh: code interpreter pyodide fs

This commit is contained in:
Timothy Jaeryang Baek
2026-03-07 19:23:18 -06:00
parent d1975b740b
commit ce0ca894fe
12 changed files with 727 additions and 77 deletions
+23 -12
View File
@@ -2296,20 +2296,31 @@ CODE_INTERPRETER_BLOCKED_MODULES = [
]
DEFAULT_CODE_INTERPRETER_PROMPT = """
#### Tools Available
#### Code Interpreter
1. **Code Interpreter**: `<code_interpreter type="code" lang="python"></code_interpreter>`
- You have access to a Python shell that runs directly in the user's browser, enabling fast execution of code for analysis, calculations, or problem-solving. Use it in this response.
- The Python code you write can incorporate a wide array of libraries, handle data manipulation or visualization, perform API calls for web-related tasks, or tackle virtually any computational challenge. Use this flexibility to **think outside the box, craft elegant solutions, and harness Python's full potential**.
- To use it, **you must enclose your code within `<code_interpreter type="code" lang="python">` XML tags** and stop right away. If you don't, the code won't execute.
- When writing code in the code_interpreter XML tag, Do NOT use the triple backticks code block for markdown formatting, example: ```py # python code ``` will cause an error because it is markdown formatting, it is not python code.
- When coding, **always aim to print meaningful outputs** (e.g., results, tables, summaries, or visuals) to better interpret and verify the findings. Avoid relying on implicit outputs; prioritize explicit and clear print statements so the results are effectively communicated to the user.
- After obtaining the printed output, **always provide a concise analysis, interpretation, or next steps to help the user understand the findings or refine the outcome further.**
- If the results are unclear, unexpected, or require validation, refine the code and execute it again as needed. Always aim to deliver meaningful insights from the results, iterating if necessary.
- **If a link to an image, audio, or any file is provided in markdown format in the output, ALWAYS regurgitate word for word, explicitly display it as part of the response to ensure the user can access it easily, do NOT change the link.**
- All responses should be communicated in the chat's primary language, ensuring seamless understanding. If the chat is multilingual, default to English for clarity.
You have access to a Python code interpreter via: `<code_interpreter type="code" lang="python"></code_interpreter>`
Ensure that the tools are effectively utilized to achieve the highest-quality analysis for the user."""
- The Python shell runs directly in the user's browser for fast execution of analysis, calculations, or problem-solving. Use it in this response.
- You can use a wide array of libraries for data manipulation, visualization, API calls, or any computational task. Think outside the box and harness Python's full potential.
- **You must enclose your code within `<code_interpreter type="code" lang="python">` XML tags** and stop right away. If you don't, the code won't execute.
- Do NOT use triple backticks (```py ... ```) inside the XML tags — that is markdown formatting, not executable Python code.
- **Always print meaningful outputs** (results, tables, summaries, visuals). Avoid implicit outputs; use explicit print statements.
- After obtaining output, **provide a concise analysis, interpretation, or next steps** to help the user understand the findings.
- If results are unclear or unexpected, refine the code and re-execute. Iterate until you deliver meaningful insights.
- **If a link to an image, audio, or any file appears in the output, display it exactly as-is** in your response so the user can access it. Do not modify the link.
- Respond in the chat's primary language. Default to English if multilingual.
Ensure the code interpreter is effectively utilized to achieve the highest-quality analysis for the user."""
# Appended to the code interpreter prompt only when engine is pyodide (not jupyter)
CODE_INTERPRETER_PYODIDE_FS_PROMPT = """
##### Persistent File System
- User-uploaded files are available at `/mnt/uploads/`. When the user asks you to work with their files, read from this directory.
- You can also write output files to `/mnt/uploads/` so the user can access and download them from the file browser.
- The file system persists across code executions within the same session.
- Use `import os; os.listdir('/mnt/uploads')` to discover available files."""
####################################
+1
View File
@@ -2193,6 +2193,7 @@ async def get_app_config(request: Request):
"user_count": user_count,
"code": {
"engine": app.state.config.CODE_EXECUTION_ENGINE,
"interpreter_engine": app.state.config.CODE_INTERPRETER_ENGINE,
},
"audio": {
"tts": {
+26 -6
View File
@@ -1,3 +1,4 @@
import copy
import time
import logging
import sys
@@ -119,6 +120,7 @@ from open_webui.config import (
DEFAULT_VOICE_MODE_PROMPT_TEMPLATE,
DEFAULT_TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
DEFAULT_CODE_INTERPRETER_PROMPT,
CODE_INTERPRETER_PYODIDE_FS_PROMPT,
CODE_INTERPRETER_BLOCKED_MODULES,
)
from open_webui.env import (
@@ -2352,18 +2354,36 @@ async def process_chat_payload(request, form_data, user, metadata, model):
)
if "code_interpreter" in features and features["code_interpreter"]:
engine = getattr(
request.app.state.config, "CODE_INTERPRETER_ENGINE", "pyodide"
)
# Skip XML-tag prompt injection when native FC is enabled —
# execute_code will be injected as a builtin tool instead
if metadata.get("params", {}).get("function_calling") != "native":
prompt = (
request.app.state.config.CODE_INTERPRETER_PROMPT_TEMPLATE
if request.app.state.config.CODE_INTERPRETER_PROMPT_TEMPLATE
!= ""
else DEFAULT_CODE_INTERPRETER_PROMPT
)
# Append filesystem awareness only for pyodide engine
if engine != "jupyter":
prompt += CODE_INTERPRETER_PYODIDE_FS_PROMPT
form_data["messages"] = add_or_update_user_message(
(
request.app.state.config.CODE_INTERPRETER_PROMPT_TEMPLATE
if request.app.state.config.CODE_INTERPRETER_PROMPT_TEMPLATE
!= ""
else DEFAULT_CODE_INTERPRETER_PROMPT
),
prompt,
form_data["messages"],
)
else:
# Native FC: tool docstring can't be dynamic, so inject
# filesystem context into messages for pyodide engine
if engine != "jupyter":
form_data["messages"] = add_or_update_user_message(
CODE_INTERPRETER_PYODIDE_FS_PROMPT,
form_data["messages"],
)
tool_ids = form_data.pop("tool_ids", None)
terminal_id = form_data.pop("terminal_id", None)
@@ -67,7 +67,7 @@
>
<option disabled selected value="">{$i18n.t('Select a engine')}</option>
{#each engines as engine}
<option value={engine}>{engine}</option>
<option value={engine}>{engine}{engine === 'jupyter' ? ' (Legacy)' : ''}</option>
{/each}
</select>
</div>
@@ -193,7 +193,7 @@
>
<option disabled selected value="">{$i18n.t('Select a engine')}</option>
{#each engines as engine}
<option value={engine}>{engine}</option>
<option value={engine}>{engine}{engine === 'jupyter' ? ' (Legacy)' : ''}</option>
{/each}
</select>
</div>
+3
View File
@@ -148,6 +148,8 @@
let webSearchEnabled = false;
let codeInterpreterEnabled = false;
let showCommands = false;
let generating = false;
@@ -2924,6 +2926,7 @@
{stopResponse}
{showMessage}
{eventTarget}
{codeInterpreterEnabled}
/>
</PaneGroup>
</div>
+9 -1
View File
@@ -10,6 +10,7 @@
import { onDestroy, onMount, tick, getContext } from 'svelte';
import {
config,
terminalServers,
mobile,
showControls,
@@ -31,6 +32,7 @@
import Artifacts from './Artifacts.svelte';
import Embeds from './ChatControls/Embeds.svelte';
import FileNav from './FileNav.svelte';
import PyodideFileNav from './PyodideFileNav.svelte';
import Overview from './Overview.svelte';
const i18n = getContext('i18n');
@@ -50,6 +52,8 @@
export let files;
export let modelId;
export let codeInterpreterEnabled = false;
export let pane: Pane | null = null;
let largeScreen = false;
@@ -67,7 +71,7 @@
$: hasMessages = history?.messages && Object.keys(history.messages).length > 0;
$: showControlsTab = $user?.role === 'admin' || ($user?.permissions?.chat?.controls ?? true);
$: showFilesTab = !!$selectedTerminalId;
$: showFilesTab = !!$selectedTerminalId || (codeInterpreterEnabled && $config?.code?.interpreter_engine !== 'jupyter');
$: showOverviewTab = hasMessages;
// Tab fallback: if active tab becomes hidden, switch to next available
@@ -355,6 +359,8 @@
/>
{:else if activeTab === 'files' && $selectedTerminalId}
<FileNav onAttach={handleTerminalAttach} />
{:else if activeTab === 'files' && codeInterpreterEnabled}
<PyodideFileNav />
{:else}
<Controls embed={true} {models} bind:chatFiles bind:params />
{/if}
@@ -504,6 +510,8 @@
/>
{:else if activeTab === 'files' && $selectedTerminalId}
<FileNav onAttach={handleTerminalAttach} overlay={dragged} />
{:else if activeTab === 'files' && codeInterpreterEnabled}
<PyodideFileNav overlay={dragged} />
{:else}
<Controls embed={true} {models} bind:chatFiles bind:params />
{/if}
@@ -12,8 +12,8 @@
export let entry: FileEntry;
export let currentPath: string;
export let terminalUrl: string;
export let terminalKey: string;
export let terminalUrl: string = '';
export let terminalKey: string = '';
export let onOpen: (entry: FileEntry) => void = () => {};
export let onDownload: (path: string) => void = () => {};
@@ -508,11 +508,17 @@
let showCodeInterpreterButton = false;
$: showCodeInterpreterButton =
!$selectedTerminalId &&
(atSelectedModel?.id ? [atSelectedModel.id] : selectedModels).length ===
codeInterpreterCapableModels.length &&
$config?.features?.enable_code_interpreter &&
($_user.role === 'admin' || $_user?.permissions?.features?.code_interpreter);
// Disable code interpreter when terminal is active (mutually exclusive)
$: if ($selectedTerminalId && codeInterpreterEnabled) {
codeInterpreterEnabled = false;
}
const scrollToBottom = () => {
const element = document.getElementById('messages-container');
element.scrollTo({
@@ -0,0 +1,367 @@
<script context="module">
let savedPyodidePath = '/mnt/uploads';
</script>
<script lang="ts">
import { getContext, onMount } from 'svelte';
import { pyodideWorker } from '$lib/stores';
import PyodideWorkerConstructor from '$lib/workers/pyodide.worker?worker';
import type { FileEntry } from '$lib/apis/terminal';
import FileNavToolbar from './FileNav/FileNavToolbar.svelte';
import FileEntryRow from './FileNav/FileEntryRow.svelte';
import FilePreview from './FileNav/FilePreview.svelte';
import ConfirmDialog from '../common/ConfirmDialog.svelte';
import Spinner from '../common/Spinner.svelte';
import Folder from '../icons/Folder.svelte';
const i18n = getContext('i18n');
export let overlay = false;
// ── State ─────────────────────────────────────────────────────────────
let currentPath = savedPyodidePath;
let entries: FileEntry[] = [];
let loading = false;
let error: string | null = null;
let selectedFile: string | null = null;
let fileLoading = false;
let fileContent: string | null = null;
let fileImageUrl: string | null = null;
let isDragOver = false;
let showDeleteConfirm = false;
let deletePath = '';
let deleteName = '';
let _reqId = 0;
const IMAGE_EXTS = new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'ico', 'avif']);
const isImage = (path: string) => IMAGE_EXTS.has(path.split('.').pop()?.toLowerCase() ?? '');
// ── Worker management ─────────────────────────────────────────────────
function ensureWorker(): Worker {
let worker = $pyodideWorker;
if (!worker) {
worker = new PyodideWorkerConstructor();
pyodideWorker.set(worker);
}
return worker;
}
function sendWorkerMessage(msg: any): Promise<any> {
const worker = ensureWorker();
const id = `fs-${++_reqId}`;
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
worker.removeEventListener('message', handler);
reject('Timeout');
}, 30000);
function handler(event: MessageEvent) {
if (event.data?.id !== id) return;
clearTimeout(timeout);
worker.removeEventListener('message', handler);
resolve(event.data);
}
worker.addEventListener('message', handler);
worker.postMessage({ ...msg, id });
});
}
// ── Breadcrumbs ───────────────────────────────────────────────────────
const buildBreadcrumbs = (path: string) => {
const parts = path.split('/').filter(Boolean);
return parts.reduce(
(acc, part) => {
const prev = acc[acc.length - 1];
acc.push({ label: part, path: `${prev.path}${part}/` });
return acc;
},
[{ label: '/', path: '/' }]
);
};
$: breadcrumbs = buildBreadcrumbs(currentPath);
// ── Operations ────────────────────────────────────────────────────────
const loadDir = async (path: string) => {
loading = true;
error = null;
selectedFile = null;
clearPreview();
currentPath = path.endsWith('/') ? path : path + '/';
savedPyodidePath = currentPath;
try {
const res = await sendWorkerMessage({ type: 'fs:list', path: currentPath.replace(/\/$/, '') || '/' });
entries = (res.entries || []).sort((a: FileEntry, b: FileEntry) => {
if (a.type !== b.type) return a.type === 'directory' ? -1 : 1;
return a.name.localeCompare(b.name);
});
} catch {
error = 'Failed to list directory';
entries = [];
}
loading = false;
};
const openEntry = async (entry: FileEntry) => {
if (entry.type === 'directory') {
await loadDir(`${currentPath}${entry.name}/`);
return;
}
const filePath = `${currentPath}${entry.name}`;
selectedFile = filePath;
fileLoading = true;
clearPreview();
try {
const res = await sendWorkerMessage({ type: 'fs:read', path: filePath });
if (res.error) {
fileContent = `Error: ${res.error}`;
} else if (isImage(filePath)) {
const blob = new Blob([res.data]);
fileImageUrl = URL.createObjectURL(blob);
} else {
const decoder = new TextDecoder('utf-8', { fatal: true });
try {
fileContent = decoder.decode(res.data);
} catch {
fileContent = `[Binary file: ${entry.size ?? 0} bytes]`;
}
}
} catch {
fileContent = 'Failed to read file';
}
fileLoading = false;
};
const clearPreview = () => {
fileContent = null;
if (fileImageUrl) {
URL.revokeObjectURL(fileImageUrl);
fileImageUrl = null;
}
};
const downloadFile = async (path: string) => {
try {
const res = await sendWorkerMessage({ type: 'fs:read', path });
if (res.data) {
const blob = new Blob([res.data]);
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = path.split('/').pop() ?? 'file';
a.click();
URL.revokeObjectURL(url);
}
} catch (e) {
console.error('Download failed:', e);
}
};
const confirmDelete = (path: string, name: string) => {
deletePath = path;
deleteName = name;
showDeleteConfirm = true;
};
const doDelete = async () => {
try {
await sendWorkerMessage({ type: 'fs:delete', path: deletePath });
if (selectedFile === deletePath) {
selectedFile = null;
clearPreview();
}
await loadDir(currentPath);
} catch (e) {
console.error('Delete failed:', e);
}
};
const createFolder = async () => {
const name = prompt($i18n.t('Folder name'));
if (!name) return;
const folderPath = `${currentPath}${name}`.replace(/\/$/, '');
try {
await sendWorkerMessage({ type: 'fs:mkdir', path: folderPath });
await loadDir(currentPath);
} catch (e) {
console.error('Failed to create folder:', e);
}
};
const uploadFiles = async (fileList: File[]) => {
const payloads: { name: string; data: ArrayBuffer }[] = [];
for (const file of fileList) {
payloads.push({ name: file.name, data: await file.arrayBuffer() });
}
try {
await sendWorkerMessage({ type: 'fs:upload', files: payloads, dir: currentPath.replace(/\/$/, '') || '/' });
} catch (e) {
console.error('Upload failed:', e);
}
await loadDir(currentPath);
};
// ── Drag and drop ─────────────────────────────────────────────────────
const handleDragOver = (e: DragEvent) => {
if (selectedFile) return;
if (!e.dataTransfer?.types.includes('Files')) return;
e.preventDefault();
e.stopPropagation();
isDragOver = true;
};
const handleDrop = async (e: DragEvent) => {
e.preventDefault();
e.stopPropagation();
isDragOver = false;
if (selectedFile) return;
const droppedFiles = Array.from(e.dataTransfer?.files ?? []);
if (droppedFiles.length) await uploadFiles(droppedFiles);
};
// ── Lifecycle ─────────────────────────────────────────────────────────
onMount(() => {
ensureWorker();
loadDir(currentPath);
});
</script>
<ConfirmDialog
bind:show={showDeleteConfirm}
title={$i18n.t('Delete {{name}}', { name: deleteName })}
message={$i18n.t('Are you sure you want to delete this?')}
on:confirm={doDelete}
/>
<div
class="flex flex-col h-full min-h-0 min-w-0 relative"
on:dragover={handleDragOver}
on:dragleave={() => (isDragOver = false)}
on:drop={handleDrop}
role="region"
aria-label={$i18n.t('Pyodide file browser')}
>
{#if isDragOver}
<div
class="absolute inset-0 z-10 flex flex-col items-center justify-center bg-white/80 dark:bg-gray-850/80 backdrop-blur-sm pointer-events-none gap-1.5"
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.5"
class="size-5 text-gray-400 dark:text-gray-500"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M3 16.5v2.25A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75V16.5m-13.5-9L12 3m0 0 4.5 4.5M12 3v13.5"
/>
</svg>
<span class="text-xs text-gray-400 dark:text-gray-500">{$i18n.t('Drop files here')}</span>
</div>
{/if}
{#if overlay}
<div class="absolute inset-0 z-10 pointer-events-none" />
{/if}
<!-- Toolbar (shared with FileNav) -->
<FileNavToolbar
{breadcrumbs}
{selectedFile}
{loading}
onNavigate={(path) => loadDir(path)}
onRefresh={() => {
if (selectedFile) {
const name = selectedFile.split('/').pop() ?? '';
openEntry({ name, type: 'file', size: 0 });
} else {
loadDir(currentPath);
}
}}
onNewFolder={createFolder}
onNewFile={() => {}}
onUploadFiles={uploadFiles}
onMove={() => {}}
>
<!-- File action buttons when a file is selected (slot content) -->
<button
class="shrink-0 p-1 rounded hover:bg-gray-100 dark:hover:bg-gray-800 transition text-gray-400 dark:text-gray-500 hover:text-gray-600 dark:hover:text-gray-400"
on:click={() => selectedFile && downloadFile(selectedFile)}
aria-label={$i18n.t('Download')}
>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="currentColor" class="size-3.5">
<path d="M10.75 2.75a.75.75 0 0 0-1.5 0v8.614L6.295 8.235a.75.75 0 1 0-1.09 1.03l4.25 4.5a.75.75 0 0 0 1.09 0l4.25-4.5a.75.75 0 0 0-1.09-1.03l-2.955 3.129V2.75Z" />
<path d="M3.5 12.75a.75.75 0 0 0-1.5 0v2.5A2.75 2.75 0 0 0 4.75 18h10.5A2.75 2.75 0 0 0 18 15.25v-2.5a.75.75 0 0 0-1.5 0v2.5c0 .69-.56 1.25-1.25 1.25H4.75c-.69 0-1.25-.56-1.25-1.25v-2.5Z" />
</svg>
</button>
<button
class="shrink-0 p-1 rounded hover:bg-gray-100 dark:hover:bg-gray-800 transition text-gray-400 dark:text-gray-500 hover:text-red-600 dark:hover:text-red-400"
on:click={() => {
if (selectedFile) {
confirmDelete(selectedFile, selectedFile.split('/').pop() ?? '');
}
}}
aria-label={$i18n.t('Delete')}
>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="currentColor" class="size-3.5">
<path fill-rule="evenodd" d="M8.75 1A2.75 2.75 0 0 0 6 3.75v.443c-.795.077-1.584.176-2.365.298a.75.75 0 1 0 .23 1.482l.149-.022 1.005 11.36A2.75 2.75 0 0 0 7.764 20h4.472a2.75 2.75 0 0 0 2.745-2.689l1.005-11.36.149.022a.75.75 0 0 0 .23-1.482A41.03 41.03 0 0 0 14 4.193V3.75A2.75 2.75 0 0 0 11.25 1h-2.5ZM10 4c.84 0 1.673.025 2.5.075V3.75c0-.69-.56-1.25-1.25-1.25h-2.5c-.69 0-1.25.56-1.25 1.25v.325C8.327 4.025 9.16 4 10 4ZM8.58 7.72a.75.75 0 0 1 .7.8l-.3 6a.75.75 0 0 1-1.5-.075l.3-6a.75.75 0 0 1 .8-.725ZM12.2 8.52a.75.75 0 0 0-1.5.075l-.3 6a.75.75 0 0 0 1.5-.075l.3-6Z" clip-rule="evenodd" />
</svg>
</button>
</FileNavToolbar>
<!-- Content area -->
<div class="flex-1 min-h-0 flex flex-col">
{#if selectedFile}
<FilePreview
{selectedFile}
{fileLoading}
{fileImageUrl}
{fileContent}
{overlay}
/>
{:else if loading}
<div class="flex items-center justify-center flex-1 p-6">
<Spinner className="size-4" />
</div>
{:else if error}
<div class="flex items-center justify-center flex-1 p-6">
<div class="text-xs text-red-500">{error}</div>
</div>
{:else if entries.length === 0}
<div class="flex flex-col items-center justify-center flex-1 p-6 text-center gap-2">
<Folder className="size-5 text-gray-300 dark:text-gray-600" />
<div class="text-xs text-gray-400 dark:text-gray-500">
{$i18n.t('No files yet. Upload files or run Python code to create them.')}
</div>
</div>
{:else}
<ul class="overflow-y-auto flex-1 min-h-0">
{#each entries as entry (entry.name)}
<FileEntryRow
{entry}
{currentPath}
onOpen={openEntry}
onDownload={downloadFile}
onDelete={confirmDelete}
/>
{/each}
</ul>
{/if}
</div>
</div>
+3
View File
@@ -72,6 +72,9 @@ export const functions = writable(null);
export const toolServers = writable([]);
export const terminalServers = writable([]);
// Persistent Pyodide worker for code interpreter FS
export const pyodideWorker: Writable<Worker | null> = writable(null);
export const banners: Writable<Banner[]> = writable([]);
export const settings: Writable<Settings> = writable({});
+219 -41
View File
@@ -13,6 +13,12 @@ declare global {
}
}
// ---------------------------------------------------------------------------
// Pyodide bootstrap
// ---------------------------------------------------------------------------
let pyodideReady: Promise<void> | null = null;
async function loadPyodideAndPackages(packages: string[] = []) {
self.stdout = null;
self.stderr = null;
@@ -40,41 +46,148 @@ async function loadPyodideAndPackages(packages: string[] = []) {
packages: ['micropip']
});
const mountDir = '/mnt';
self.pyodide.FS.mkdirTree(mountDir);
// self.pyodide.FS.mount(self.pyodide.FS.filesystems.IDBFS, {}, mountDir);
// Create the upload directory and mount IDBFS for persistence
const uploadDir = '/mnt/uploads';
self.pyodide.FS.mkdirTree(uploadDir);
self.pyodide.FS.mount(self.pyodide.FS.filesystems.IDBFS, {}, '/mnt');
// // Load persisted files from IndexedDB (Initial Sync)
// await new Promise<void>((resolve, reject) => {
// self.pyodide.FS.syncfs(true, (err) => {
// if (err) {
// console.error('Error syncing from IndexedDB:', err);
// reject(err);
// } else {
// console.log('Successfully loaded from IndexedDB.');
// resolve();
// }
// });
// });
// Load persisted files from IndexedDB
await new Promise<void>((resolve) => {
(self.pyodide.FS as any).syncfs(true, (err: Error | null) => {
if (err) {
console.error('Error syncing from IndexedDB:', err);
}
// Always resolve — missing data is fine on first run
resolve();
});
});
// Ensure /mnt/uploads still exists after sync (first-time init)
try {
self.pyodide.FS.stat(uploadDir);
} catch {
self.pyodide.FS.mkdirTree(uploadDir);
}
const micropip = self.pyodide.pyimport('micropip');
// await micropip.set_index_urls('https://pypi.org/pypi/{package_name}/json');
await micropip.install(packages);
}
self.onmessage = async (event) => {
const { id, code, ...context } = event.data;
/**
* Ensure Pyodide is loaded. On the first call, loads and installs packages.
* Subsequent calls reuse the already-loaded instance (persistent worker).
*/
async function ensurePyodide(packages: string[] = []) {
if (!pyodideReady) {
pyodideReady = loadPyodideAndPackages(packages);
}
await pyodideReady;
console.log(event.data);
// Install any additional packages not loaded on init
if (packages.length > 0 && self.pyodide) {
const micropip = self.pyodide.pyimport('micropip');
await micropip.install(packages);
}
}
// The worker copies the context in its own "memory" (an object mapping name to values)
for (const key of Object.keys(context)) {
self[key] = context[key];
/**
* Persist the in-memory FS to IndexedDB (fire-and-forget with logging).
*/
function persistFS() {
if (!self.pyodide) return;
(self.pyodide.FS as any).syncfs(false, (err: Error | null) => {
if (err) {
console.error('Error syncing to IndexedDB:', err);
} else {
console.log('Successfully synced to IndexedDB.');
}
});
}
// ---------------------------------------------------------------------------
// FS operations
// ---------------------------------------------------------------------------
function fsUploadFiles(files: { name: string; data: ArrayBuffer }[], dir = '/mnt/uploads') {
try {
self.pyodide.FS.stat(dir);
} catch {
self.pyodide.FS.mkdirTree(dir);
}
// make sure loading is done
await loadPyodideAndPackages(self.packages);
for (const file of files) {
self.pyodide.FS.writeFile(`${dir}/${file.name}`, new Uint8Array(file.data));
}
}
function fsList(path: string) {
const entries: { name: string; type: 'file' | 'directory'; size: number }[] = [];
try {
const items = self.pyodide.FS.readdir(path).filter(
(n: string) => n !== '.' && n !== '..'
);
for (const name of items) {
try {
const stat = self.pyodide.FS.stat(`${path}/${name}`);
const isDir = self.pyodide.FS.isDir(stat.mode);
entries.push({
name,
type: isDir ? 'directory' : 'file',
size: isDir ? 0 : stat.size
});
} catch {
// skip inaccessible entries
}
}
} catch {
// directory doesn't exist
}
return entries;
}
function fsRead(path: string): ArrayBuffer {
const data: Uint8Array = (self.pyodide.FS as any).readFile(path) as Uint8Array;
return data.buffer as ArrayBuffer;
}
function fsDelete(path: string) {
try {
const stat = self.pyodide.FS.stat(path);
if (self.pyodide.FS.isDir(stat.mode)) {
// Recursively delete directory contents
const items = self.pyodide.FS.readdir(path).filter(
(n: string) => n !== '.' && n !== '..'
);
for (const item of items) {
fsDelete(`${path}/${item}`);
}
self.pyodide.FS.rmdir(path);
} else {
self.pyodide.FS.unlink(path);
}
} catch {
// already gone
}
}
function fsMkdir(path: string) {
self.pyodide.FS.mkdirTree(path);
}
// ---------------------------------------------------------------------------
// Code execution
// ---------------------------------------------------------------------------
async function executeCode(id: string, code: string, files?: { name: string; data: ArrayBuffer }[]) {
self.stdout = null;
self.stderr = null;
self.result = null;
// Upload any accompanying files before execution
if (files && files.length > 0) {
fsUploadFiles(files);
persistFS();
}
try {
// check if matplotlib is imported in the code
@@ -113,25 +226,90 @@ matplotlib.pyplot.show = show`);
console.log('Python result:', self.result);
// Persist any changes to IndexedDB
// await new Promise<void>((resolve, reject) => {
// self.pyodide.FS.syncfs(false, (err) => {
// if (err) {
// console.error('Error syncing to IndexedDB:', err);
// reject(err);
// } else {
// console.log('Successfully synced to IndexedDB.');
// resolve();
// }
// });
// });
} catch (error) {
self.stderr = error.toString();
// Persist any files the code may have written
persistFS();
} catch (error: unknown) {
self.stderr = error instanceof Error ? error.message : String(error);
}
self.postMessage({ id, result: self.result, stdout: self.stdout, stderr: self.stderr });
}
// ---------------------------------------------------------------------------
// Message handler
// ---------------------------------------------------------------------------
self.onmessage = async (event) => {
const data = event.data;
const { id, type } = data;
// Backward compatibility: messages without a `type` field are execute requests
if (!type || type === 'execute') {
const { code, files, ...context } = data;
// Copy context keys (packages, etc.) into worker scope
for (const key of Object.keys(context)) {
if (key !== 'id' && key !== 'type') {
self[key] = context[key];
}
}
await ensurePyodide(self.packages);
await executeCode(id, code, files);
return;
}
// FS operations require Pyodide to be loaded
await ensurePyodide();
switch (type) {
case 'fs:upload': {
const { files, dir } = data;
fsUploadFiles(files, dir);
persistFS();
self.postMessage({ id, type: 'fs:upload', success: true });
break;
}
case 'fs:list': {
const entries = fsList(data.path);
self.postMessage({ id, type: 'fs:list', entries });
break;
}
case 'fs:read': {
try {
const buffer = fsRead(data.path);
self.postMessage({ id, type: 'fs:read', data: buffer }, { transfer: [buffer] });
} catch (err: unknown) {
self.postMessage({ id, type: 'fs:read', error: err instanceof Error ? err.message : String(err) });
}
break;
}
case 'fs:delete': {
fsDelete(data.path);
persistFS();
self.postMessage({ id, type: 'fs:delete', success: true });
break;
}
case 'fs:mkdir': {
fsMkdir(data.path);
persistFS();
self.postMessage({ id, type: 'fs:mkdir', success: true });
break;
}
default:
console.warn('Unknown message type:', type);
}
};
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function processResult(result: any): any {
// Catch and always return JSON-safe string representations
try {
@@ -167,9 +345,9 @@ function processResult(result: any): any {
}
// Stringify anything that's left (e.g., Proxy objects that cannot be directly processed)
return JSON.stringify(result);
} catch (err) {
} catch (err: unknown) {
// In case something unexpected happens, we return a stringified fallback
return `[processResult error]: ${err.message || err.toString()}`;
return `[processResult error]: ${err instanceof Error ? err.message : String(err)}`;
}
}
+66 -13
View File
@@ -34,8 +34,10 @@
terminalServers,
showControls,
showFileNavPath,
showFileNavDir
showFileNavDir,
pyodideWorker
} from '$lib/stores';
import { getFileContentById } from '$lib/apis/files';
import { goto } from '$app/navigation';
import { page } from '$app/stores';
import { beforeNavigate } from '$app/navigation';
@@ -184,7 +186,20 @@
});
};
const executePythonAsWorker = async (id, code, cb) => {
/**
* Get or create the persistent Pyodide worker.
* The worker persists across executions so the virtual FS (IDBFS) is preserved.
*/
const getOrCreateWorker = () => {
let worker = $pyodideWorker;
if (!worker) {
worker = new PyodideWorker();
pyodideWorker.set(worker);
}
return worker;
};
const executePythonAsWorker = async (id, code, cb, files = []) => {
let result = null;
let stdout = null;
let stderr = null;
@@ -206,19 +221,44 @@
/\bimport\s+pytz\b|\bfrom\s+pytz\b/.test(code) ? 'pytz' : null
].filter(Boolean);
const pyodideWorker = new PyodideWorker();
const worker = getOrCreateWorker();
pyodideWorker.postMessage({
// Fetch file content from the server and prepare for the worker
let filePayloads = [];
if (files && files.length > 0) {
for (const file of files) {
try {
const fileId = file?.id;
const fileName = file?.filename || file?.name || 'file';
if (fileId) {
const content = await getFileContentById(fileId);
if (content) {
filePayloads.push({ name: fileName, data: content });
}
}
} catch (e) {
console.error('Failed to fetch file for Pyodide:', e);
}
}
}
worker.postMessage({
type: 'execute',
id: id,
code: code,
packages: packages
packages: packages,
files: filePayloads.length > 0 ? filePayloads : undefined
});
setTimeout(() => {
// Timeout for this specific execution (not the worker itself)
let timeoutId = setTimeout(() => {
if (executing) {
executing = false;
stderr = 'Execution Time Limit Exceeded';
pyodideWorker.terminate();
// Terminate and recreate the worker on timeout
worker.terminate();
pyodideWorker.set(null);
if (cb) {
cb(
@@ -237,11 +277,18 @@
}
}, 60000);
pyodideWorker.onmessage = (event) => {
console.log('pyodideWorker.onmessage', event);
const { id, ...data } = event.data;
// Use addEventListener so multiple concurrent executions don't clobber each other
const onMessage = (event) => {
const { id: eventId, ...data } = event.data;
// Only handle responses for this execution ID
if (eventId !== id) return;
// Ignore FS responses (they use a type field)
if (data.type && data.type.startsWith('fs:')) return;
console.log(id, data);
console.log('pyodideWorker.onmessage', event);
clearTimeout(timeoutId);
worker.removeEventListener('message', onMessage);
worker.removeEventListener('error', onError);
data['stdout'] && (stdout = data['stdout']);
data['stderr'] && (stderr = data['stderr']);
@@ -265,8 +312,11 @@
executing = false;
};
pyodideWorker.onerror = (event) => {
const onError = (event) => {
console.log('pyodideWorker.onerror', event);
clearTimeout(timeoutId);
worker.removeEventListener('message', onMessage);
worker.removeEventListener('error', onError);
if (cb) {
cb(
@@ -284,6 +334,9 @@
}
executing = false;
};
worker.addEventListener('message', onMessage);
worker.addEventListener('error', onError);
};
const resolveToolServer = (serverUrl) => {
@@ -423,7 +476,7 @@
} else if (data?.session_id === $socket.id) {
if (type === 'execute:python') {
console.log('execute:python', data);
executePythonAsWorker(data.id, data.code, cb);
executePythonAsWorker(data.id, data.code, cb, data.files || []);
} else if (type === 'execute:tool') {
console.log('execute:tool', data);
executeTool(data, cb);