enh: code interpreter pyodide fs
This commit is contained in:
Timothy Jaeryang Baek
@@ -67,7 +67,7 @@
|
||||
>
|
||||
<option disabled selected value="">{$i18n.t('Select a engine')}</option>
|
||||
{#each engines as engine}
|
||||
<option value={engine}>{engine}</option>
|
||||
<option value={engine}>{engine}{engine === 'jupyter' ? ' (Legacy)' : ''}</option>
|
||||
{/each}
|
||||
</select>
|
||||
</div>
|
||||
@@ -193,7 +193,7 @@
|
||||
>
|
||||
<option disabled selected value="">{$i18n.t('Select a engine')}</option>
|
||||
{#each engines as engine}
|
||||
<option value={engine}>{engine}</option>
|
||||
<option value={engine}>{engine}{engine === 'jupyter' ? ' (Legacy)' : ''}</option>
|
||||
{/each}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
@@ -148,6 +148,8 @@
|
||||
let webSearchEnabled = false;
|
||||
let codeInterpreterEnabled = false;
|
||||
|
||||
|
||||
|
||||
let showCommands = false;
|
||||
|
||||
let generating = false;
|
||||
@@ -2924,6 +2926,7 @@
|
||||
{stopResponse}
|
||||
{showMessage}
|
||||
{eventTarget}
|
||||
{codeInterpreterEnabled}
|
||||
/>
|
||||
</PaneGroup>
|
||||
</div>
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
|
||||
import { onDestroy, onMount, tick, getContext } from 'svelte';
|
||||
import {
|
||||
config,
|
||||
terminalServers,
|
||||
mobile,
|
||||
showControls,
|
||||
@@ -31,6 +32,7 @@
|
||||
import Artifacts from './Artifacts.svelte';
|
||||
import Embeds from './ChatControls/Embeds.svelte';
|
||||
import FileNav from './FileNav.svelte';
|
||||
import PyodideFileNav from './PyodideFileNav.svelte';
|
||||
import Overview from './Overview.svelte';
|
||||
|
||||
const i18n = getContext('i18n');
|
||||
@@ -50,6 +52,8 @@
|
||||
export let files;
|
||||
export let modelId;
|
||||
|
||||
export let codeInterpreterEnabled = false;
|
||||
|
||||
export let pane: Pane | null = null;
|
||||
|
||||
let largeScreen = false;
|
||||
@@ -67,7 +71,7 @@
|
||||
$: hasMessages = history?.messages && Object.keys(history.messages).length > 0;
|
||||
|
||||
$: showControlsTab = $user?.role === 'admin' || ($user?.permissions?.chat?.controls ?? true);
|
||||
$: showFilesTab = !!$selectedTerminalId;
|
||||
$: showFilesTab = !!$selectedTerminalId || (codeInterpreterEnabled && $config?.code?.interpreter_engine !== 'jupyter');
|
||||
$: showOverviewTab = hasMessages;
|
||||
|
||||
// Tab fallback: if active tab becomes hidden, switch to next available
|
||||
@@ -355,6 +359,8 @@
|
||||
/>
|
||||
{:else if activeTab === 'files' && $selectedTerminalId}
|
||||
<FileNav onAttach={handleTerminalAttach} />
|
||||
{:else if activeTab === 'files' && codeInterpreterEnabled}
|
||||
<PyodideFileNav />
|
||||
{:else}
|
||||
<Controls embed={true} {models} bind:chatFiles bind:params />
|
||||
{/if}
|
||||
@@ -504,6 +510,8 @@
|
||||
/>
|
||||
{:else if activeTab === 'files' && $selectedTerminalId}
|
||||
<FileNav onAttach={handleTerminalAttach} overlay={dragged} />
|
||||
{:else if activeTab === 'files' && codeInterpreterEnabled}
|
||||
<PyodideFileNav overlay={dragged} />
|
||||
{:else}
|
||||
<Controls embed={true} {models} bind:chatFiles bind:params />
|
||||
{/if}
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
|
||||
export let entry: FileEntry;
|
||||
export let currentPath: string;
|
||||
export let terminalUrl: string;
|
||||
export let terminalKey: string;
|
||||
export let terminalUrl: string = '';
|
||||
export let terminalKey: string = '';
|
||||
|
||||
export let onOpen: (entry: FileEntry) => void = () => {};
|
||||
export let onDownload: (path: string) => void = () => {};
|
||||
|
||||
@@ -508,11 +508,17 @@
|
||||
|
||||
let showCodeInterpreterButton = false;
|
||||
$: showCodeInterpreterButton =
|
||||
!$selectedTerminalId &&
|
||||
(atSelectedModel?.id ? [atSelectedModel.id] : selectedModels).length ===
|
||||
codeInterpreterCapableModels.length &&
|
||||
$config?.features?.enable_code_interpreter &&
|
||||
($_user.role === 'admin' || $_user?.permissions?.features?.code_interpreter);
|
||||
|
||||
// Disable code interpreter when terminal is active (mutually exclusive)
|
||||
$: if ($selectedTerminalId && codeInterpreterEnabled) {
|
||||
codeInterpreterEnabled = false;
|
||||
}
|
||||
|
||||
const scrollToBottom = () => {
|
||||
const element = document.getElementById('messages-container');
|
||||
element.scrollTo({
|
||||
|
||||
@@ -0,0 +1,367 @@
|
||||
<script context="module">
|
||||
let savedPyodidePath = '/mnt/uploads';
|
||||
</script>
|
||||
|
||||
<script lang="ts">
|
||||
import { getContext, onMount } from 'svelte';
|
||||
import { pyodideWorker } from '$lib/stores';
|
||||
import PyodideWorkerConstructor from '$lib/workers/pyodide.worker?worker';
|
||||
import type { FileEntry } from '$lib/apis/terminal';
|
||||
|
||||
import FileNavToolbar from './FileNav/FileNavToolbar.svelte';
|
||||
import FileEntryRow from './FileNav/FileEntryRow.svelte';
|
||||
import FilePreview from './FileNav/FilePreview.svelte';
|
||||
import ConfirmDialog from '../common/ConfirmDialog.svelte';
|
||||
import Spinner from '../common/Spinner.svelte';
|
||||
import Folder from '../icons/Folder.svelte';
|
||||
|
||||
const i18n = getContext('i18n');
|
||||
|
||||
export let overlay = false;
|
||||
|
||||
// ── State ─────────────────────────────────────────────────────────────
|
||||
let currentPath = savedPyodidePath;
|
||||
let entries: FileEntry[] = [];
|
||||
let loading = false;
|
||||
let error: string | null = null;
|
||||
|
||||
let selectedFile: string | null = null;
|
||||
let fileLoading = false;
|
||||
let fileContent: string | null = null;
|
||||
let fileImageUrl: string | null = null;
|
||||
|
||||
let isDragOver = false;
|
||||
let showDeleteConfirm = false;
|
||||
let deletePath = '';
|
||||
let deleteName = '';
|
||||
|
||||
let _reqId = 0;
|
||||
|
||||
const IMAGE_EXTS = new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'ico', 'avif']);
|
||||
const isImage = (path: string) => IMAGE_EXTS.has(path.split('.').pop()?.toLowerCase() ?? '');
|
||||
|
||||
// ── Worker management ─────────────────────────────────────────────────
|
||||
|
||||
function ensureWorker(): Worker {
|
||||
let worker = $pyodideWorker;
|
||||
if (!worker) {
|
||||
worker = new PyodideWorkerConstructor();
|
||||
pyodideWorker.set(worker);
|
||||
}
|
||||
return worker;
|
||||
}
|
||||
|
||||
function sendWorkerMessage(msg: any): Promise<any> {
|
||||
const worker = ensureWorker();
|
||||
const id = `fs-${++_reqId}`;
|
||||
return new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
worker.removeEventListener('message', handler);
|
||||
reject('Timeout');
|
||||
}, 30000);
|
||||
|
||||
function handler(event: MessageEvent) {
|
||||
if (event.data?.id !== id) return;
|
||||
clearTimeout(timeout);
|
||||
worker.removeEventListener('message', handler);
|
||||
resolve(event.data);
|
||||
}
|
||||
|
||||
worker.addEventListener('message', handler);
|
||||
worker.postMessage({ ...msg, id });
|
||||
});
|
||||
}
|
||||
|
||||
// ── Breadcrumbs ───────────────────────────────────────────────────────
|
||||
|
||||
const buildBreadcrumbs = (path: string) => {
|
||||
const parts = path.split('/').filter(Boolean);
|
||||
return parts.reduce(
|
||||
(acc, part) => {
|
||||
const prev = acc[acc.length - 1];
|
||||
acc.push({ label: part, path: `${prev.path}${part}/` });
|
||||
return acc;
|
||||
},
|
||||
[{ label: '/', path: '/' }]
|
||||
);
|
||||
};
|
||||
|
||||
$: breadcrumbs = buildBreadcrumbs(currentPath);
|
||||
|
||||
// ── Operations ────────────────────────────────────────────────────────
|
||||
|
||||
const loadDir = async (path: string) => {
|
||||
loading = true;
|
||||
error = null;
|
||||
selectedFile = null;
|
||||
clearPreview();
|
||||
currentPath = path.endsWith('/') ? path : path + '/';
|
||||
savedPyodidePath = currentPath;
|
||||
|
||||
try {
|
||||
const res = await sendWorkerMessage({ type: 'fs:list', path: currentPath.replace(/\/$/, '') || '/' });
|
||||
entries = (res.entries || []).sort((a: FileEntry, b: FileEntry) => {
|
||||
if (a.type !== b.type) return a.type === 'directory' ? -1 : 1;
|
||||
return a.name.localeCompare(b.name);
|
||||
});
|
||||
} catch {
|
||||
error = 'Failed to list directory';
|
||||
entries = [];
|
||||
}
|
||||
loading = false;
|
||||
};
|
||||
|
||||
const openEntry = async (entry: FileEntry) => {
|
||||
if (entry.type === 'directory') {
|
||||
await loadDir(`${currentPath}${entry.name}/`);
|
||||
return;
|
||||
}
|
||||
|
||||
const filePath = `${currentPath}${entry.name}`;
|
||||
selectedFile = filePath;
|
||||
fileLoading = true;
|
||||
clearPreview();
|
||||
|
||||
try {
|
||||
const res = await sendWorkerMessage({ type: 'fs:read', path: filePath });
|
||||
if (res.error) {
|
||||
fileContent = `Error: ${res.error}`;
|
||||
} else if (isImage(filePath)) {
|
||||
const blob = new Blob([res.data]);
|
||||
fileImageUrl = URL.createObjectURL(blob);
|
||||
} else {
|
||||
const decoder = new TextDecoder('utf-8', { fatal: true });
|
||||
try {
|
||||
fileContent = decoder.decode(res.data);
|
||||
} catch {
|
||||
fileContent = `[Binary file: ${entry.size ?? 0} bytes]`;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
fileContent = 'Failed to read file';
|
||||
}
|
||||
fileLoading = false;
|
||||
};
|
||||
|
||||
const clearPreview = () => {
|
||||
fileContent = null;
|
||||
if (fileImageUrl) {
|
||||
URL.revokeObjectURL(fileImageUrl);
|
||||
fileImageUrl = null;
|
||||
}
|
||||
};
|
||||
|
||||
const downloadFile = async (path: string) => {
|
||||
try {
|
||||
const res = await sendWorkerMessage({ type: 'fs:read', path });
|
||||
if (res.data) {
|
||||
const blob = new Blob([res.data]);
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = path.split('/').pop() ?? 'file';
|
||||
a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Download failed:', e);
|
||||
}
|
||||
};
|
||||
|
||||
const confirmDelete = (path: string, name: string) => {
|
||||
deletePath = path;
|
||||
deleteName = name;
|
||||
showDeleteConfirm = true;
|
||||
};
|
||||
|
||||
const doDelete = async () => {
|
||||
try {
|
||||
await sendWorkerMessage({ type: 'fs:delete', path: deletePath });
|
||||
if (selectedFile === deletePath) {
|
||||
selectedFile = null;
|
||||
clearPreview();
|
||||
}
|
||||
await loadDir(currentPath);
|
||||
} catch (e) {
|
||||
console.error('Delete failed:', e);
|
||||
}
|
||||
};
|
||||
|
||||
const createFolder = async () => {
|
||||
const name = prompt($i18n.t('Folder name'));
|
||||
if (!name) return;
|
||||
const folderPath = `${currentPath}${name}`.replace(/\/$/, '');
|
||||
try {
|
||||
await sendWorkerMessage({ type: 'fs:mkdir', path: folderPath });
|
||||
await loadDir(currentPath);
|
||||
} catch (e) {
|
||||
console.error('Failed to create folder:', e);
|
||||
}
|
||||
};
|
||||
|
||||
const uploadFiles = async (fileList: File[]) => {
|
||||
const payloads: { name: string; data: ArrayBuffer }[] = [];
|
||||
for (const file of fileList) {
|
||||
payloads.push({ name: file.name, data: await file.arrayBuffer() });
|
||||
}
|
||||
try {
|
||||
await sendWorkerMessage({ type: 'fs:upload', files: payloads, dir: currentPath.replace(/\/$/, '') || '/' });
|
||||
} catch (e) {
|
||||
console.error('Upload failed:', e);
|
||||
}
|
||||
await loadDir(currentPath);
|
||||
};
|
||||
|
||||
// ── Drag and drop ─────────────────────────────────────────────────────
|
||||
|
||||
const handleDragOver = (e: DragEvent) => {
|
||||
if (selectedFile) return;
|
||||
if (!e.dataTransfer?.types.includes('Files')) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
isDragOver = true;
|
||||
};
|
||||
|
||||
const handleDrop = async (e: DragEvent) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
isDragOver = false;
|
||||
if (selectedFile) return;
|
||||
const droppedFiles = Array.from(e.dataTransfer?.files ?? []);
|
||||
if (droppedFiles.length) await uploadFiles(droppedFiles);
|
||||
};
|
||||
|
||||
// ── Lifecycle ─────────────────────────────────────────────────────────
|
||||
|
||||
onMount(() => {
|
||||
ensureWorker();
|
||||
loadDir(currentPath);
|
||||
});
|
||||
</script>
|
||||
|
||||
<ConfirmDialog
|
||||
bind:show={showDeleteConfirm}
|
||||
title={$i18n.t('Delete {{name}}', { name: deleteName })}
|
||||
message={$i18n.t('Are you sure you want to delete this?')}
|
||||
on:confirm={doDelete}
|
||||
/>
|
||||
|
||||
<div
|
||||
class="flex flex-col h-full min-h-0 min-w-0 relative"
|
||||
on:dragover={handleDragOver}
|
||||
on:dragleave={() => (isDragOver = false)}
|
||||
on:drop={handleDrop}
|
||||
role="region"
|
||||
aria-label={$i18n.t('Pyodide file browser')}
|
||||
>
|
||||
{#if isDragOver}
|
||||
<div
|
||||
class="absolute inset-0 z-10 flex flex-col items-center justify-center bg-white/80 dark:bg-gray-850/80 backdrop-blur-sm pointer-events-none gap-1.5"
|
||||
>
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="1.5"
|
||||
class="size-5 text-gray-400 dark:text-gray-500"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M3 16.5v2.25A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75V16.5m-13.5-9L12 3m0 0 4.5 4.5M12 3v13.5"
|
||||
/>
|
||||
</svg>
|
||||
<span class="text-xs text-gray-400 dark:text-gray-500">{$i18n.t('Drop files here')}</span>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if overlay}
|
||||
<div class="absolute inset-0 z-10 pointer-events-none" />
|
||||
{/if}
|
||||
|
||||
<!-- Toolbar (shared with FileNav) -->
|
||||
<FileNavToolbar
|
||||
{breadcrumbs}
|
||||
{selectedFile}
|
||||
{loading}
|
||||
onNavigate={(path) => loadDir(path)}
|
||||
onRefresh={() => {
|
||||
if (selectedFile) {
|
||||
const name = selectedFile.split('/').pop() ?? '';
|
||||
openEntry({ name, type: 'file', size: 0 });
|
||||
} else {
|
||||
loadDir(currentPath);
|
||||
}
|
||||
}}
|
||||
onNewFolder={createFolder}
|
||||
onNewFile={() => {}}
|
||||
onUploadFiles={uploadFiles}
|
||||
onMove={() => {}}
|
||||
>
|
||||
<!-- File action buttons when a file is selected (slot content) -->
|
||||
<button
|
||||
class="shrink-0 p-1 rounded hover:bg-gray-100 dark:hover:bg-gray-800 transition text-gray-400 dark:text-gray-500 hover:text-gray-600 dark:hover:text-gray-400"
|
||||
on:click={() => selectedFile && downloadFile(selectedFile)}
|
||||
aria-label={$i18n.t('Download')}
|
||||
>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="currentColor" class="size-3.5">
|
||||
<path d="M10.75 2.75a.75.75 0 0 0-1.5 0v8.614L6.295 8.235a.75.75 0 1 0-1.09 1.03l4.25 4.5a.75.75 0 0 0 1.09 0l4.25-4.5a.75.75 0 0 0-1.09-1.03l-2.955 3.129V2.75Z" />
|
||||
<path d="M3.5 12.75a.75.75 0 0 0-1.5 0v2.5A2.75 2.75 0 0 0 4.75 18h10.5A2.75 2.75 0 0 0 18 15.25v-2.5a.75.75 0 0 0-1.5 0v2.5c0 .69-.56 1.25-1.25 1.25H4.75c-.69 0-1.25-.56-1.25-1.25v-2.5Z" />
|
||||
</svg>
|
||||
</button>
|
||||
<button
|
||||
class="shrink-0 p-1 rounded hover:bg-gray-100 dark:hover:bg-gray-800 transition text-gray-400 dark:text-gray-500 hover:text-red-600 dark:hover:text-red-400"
|
||||
on:click={() => {
|
||||
if (selectedFile) {
|
||||
confirmDelete(selectedFile, selectedFile.split('/').pop() ?? '');
|
||||
}
|
||||
}}
|
||||
aria-label={$i18n.t('Delete')}
|
||||
>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="currentColor" class="size-3.5">
|
||||
<path fill-rule="evenodd" d="M8.75 1A2.75 2.75 0 0 0 6 3.75v.443c-.795.077-1.584.176-2.365.298a.75.75 0 1 0 .23 1.482l.149-.022 1.005 11.36A2.75 2.75 0 0 0 7.764 20h4.472a2.75 2.75 0 0 0 2.745-2.689l1.005-11.36.149.022a.75.75 0 0 0 .23-1.482A41.03 41.03 0 0 0 14 4.193V3.75A2.75 2.75 0 0 0 11.25 1h-2.5ZM10 4c.84 0 1.673.025 2.5.075V3.75c0-.69-.56-1.25-1.25-1.25h-2.5c-.69 0-1.25.56-1.25 1.25v.325C8.327 4.025 9.16 4 10 4ZM8.58 7.72a.75.75 0 0 1 .7.8l-.3 6a.75.75 0 0 1-1.5-.075l.3-6a.75.75 0 0 1 .8-.725ZM12.2 8.52a.75.75 0 0 0-1.5.075l-.3 6a.75.75 0 0 0 1.5-.075l.3-6Z" clip-rule="evenodd" />
|
||||
</svg>
|
||||
</button>
|
||||
</FileNavToolbar>
|
||||
|
||||
<!-- Content area -->
|
||||
<div class="flex-1 min-h-0 flex flex-col">
|
||||
{#if selectedFile}
|
||||
<FilePreview
|
||||
{selectedFile}
|
||||
{fileLoading}
|
||||
{fileImageUrl}
|
||||
{fileContent}
|
||||
{overlay}
|
||||
/>
|
||||
{:else if loading}
|
||||
<div class="flex items-center justify-center flex-1 p-6">
|
||||
<Spinner className="size-4" />
|
||||
</div>
|
||||
{:else if error}
|
||||
<div class="flex items-center justify-center flex-1 p-6">
|
||||
<div class="text-xs text-red-500">{error}</div>
|
||||
</div>
|
||||
{:else if entries.length === 0}
|
||||
<div class="flex flex-col items-center justify-center flex-1 p-6 text-center gap-2">
|
||||
<Folder className="size-5 text-gray-300 dark:text-gray-600" />
|
||||
<div class="text-xs text-gray-400 dark:text-gray-500">
|
||||
{$i18n.t('No files yet. Upload files or run Python code to create them.')}
|
||||
</div>
|
||||
</div>
|
||||
{:else}
|
||||
<ul class="overflow-y-auto flex-1 min-h-0">
|
||||
{#each entries as entry (entry.name)}
|
||||
<FileEntryRow
|
||||
{entry}
|
||||
{currentPath}
|
||||
onOpen={openEntry}
|
||||
onDownload={downloadFile}
|
||||
onDelete={confirmDelete}
|
||||
/>
|
||||
{/each}
|
||||
</ul>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
@@ -72,6 +72,9 @@ export const functions = writable(null);
|
||||
export const toolServers = writable([]);
|
||||
export const terminalServers = writable([]);
|
||||
|
||||
// Persistent Pyodide worker for code interpreter FS
|
||||
export const pyodideWorker: Writable<Worker | null> = writable(null);
|
||||
|
||||
export const banners: Writable<Banner[]> = writable([]);
|
||||
|
||||
export const settings: Writable<Settings> = writable({});
|
||||
|
||||
@@ -13,6 +13,12 @@ declare global {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pyodide bootstrap
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let pyodideReady: Promise<void> | null = null;
|
||||
|
||||
async function loadPyodideAndPackages(packages: string[] = []) {
|
||||
self.stdout = null;
|
||||
self.stderr = null;
|
||||
@@ -40,41 +46,148 @@ async function loadPyodideAndPackages(packages: string[] = []) {
|
||||
packages: ['micropip']
|
||||
});
|
||||
|
||||
const mountDir = '/mnt';
|
||||
self.pyodide.FS.mkdirTree(mountDir);
|
||||
// self.pyodide.FS.mount(self.pyodide.FS.filesystems.IDBFS, {}, mountDir);
|
||||
// Create the upload directory and mount IDBFS for persistence
|
||||
const uploadDir = '/mnt/uploads';
|
||||
self.pyodide.FS.mkdirTree(uploadDir);
|
||||
self.pyodide.FS.mount(self.pyodide.FS.filesystems.IDBFS, {}, '/mnt');
|
||||
|
||||
// // Load persisted files from IndexedDB (Initial Sync)
|
||||
// await new Promise<void>((resolve, reject) => {
|
||||
// self.pyodide.FS.syncfs(true, (err) => {
|
||||
// if (err) {
|
||||
// console.error('Error syncing from IndexedDB:', err);
|
||||
// reject(err);
|
||||
// } else {
|
||||
// console.log('Successfully loaded from IndexedDB.');
|
||||
// resolve();
|
||||
// }
|
||||
// });
|
||||
// });
|
||||
// Load persisted files from IndexedDB
|
||||
await new Promise<void>((resolve) => {
|
||||
(self.pyodide.FS as any).syncfs(true, (err: Error | null) => {
|
||||
if (err) {
|
||||
console.error('Error syncing from IndexedDB:', err);
|
||||
}
|
||||
// Always resolve — missing data is fine on first run
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// Ensure /mnt/uploads still exists after sync (first-time init)
|
||||
try {
|
||||
self.pyodide.FS.stat(uploadDir);
|
||||
} catch {
|
||||
self.pyodide.FS.mkdirTree(uploadDir);
|
||||
}
|
||||
|
||||
const micropip = self.pyodide.pyimport('micropip');
|
||||
|
||||
// await micropip.set_index_urls('https://pypi.org/pypi/{package_name}/json');
|
||||
await micropip.install(packages);
|
||||
}
|
||||
|
||||
self.onmessage = async (event) => {
|
||||
const { id, code, ...context } = event.data;
|
||||
/**
|
||||
* Ensure Pyodide is loaded. On the first call, loads and installs packages.
|
||||
* Subsequent calls reuse the already-loaded instance (persistent worker).
|
||||
*/
|
||||
async function ensurePyodide(packages: string[] = []) {
|
||||
if (!pyodideReady) {
|
||||
pyodideReady = loadPyodideAndPackages(packages);
|
||||
}
|
||||
await pyodideReady;
|
||||
|
||||
console.log(event.data);
|
||||
// Install any additional packages not loaded on init
|
||||
if (packages.length > 0 && self.pyodide) {
|
||||
const micropip = self.pyodide.pyimport('micropip');
|
||||
await micropip.install(packages);
|
||||
}
|
||||
}
|
||||
|
||||
// The worker copies the context in its own "memory" (an object mapping name to values)
|
||||
for (const key of Object.keys(context)) {
|
||||
self[key] = context[key];
|
||||
/**
|
||||
* Persist the in-memory FS to IndexedDB (fire-and-forget with logging).
|
||||
*/
|
||||
function persistFS() {
|
||||
if (!self.pyodide) return;
|
||||
(self.pyodide.FS as any).syncfs(false, (err: Error | null) => {
|
||||
if (err) {
|
||||
console.error('Error syncing to IndexedDB:', err);
|
||||
} else {
|
||||
console.log('Successfully synced to IndexedDB.');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FS operations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function fsUploadFiles(files: { name: string; data: ArrayBuffer }[], dir = '/mnt/uploads') {
|
||||
try {
|
||||
self.pyodide.FS.stat(dir);
|
||||
} catch {
|
||||
self.pyodide.FS.mkdirTree(dir);
|
||||
}
|
||||
|
||||
// make sure loading is done
|
||||
await loadPyodideAndPackages(self.packages);
|
||||
for (const file of files) {
|
||||
self.pyodide.FS.writeFile(`${dir}/${file.name}`, new Uint8Array(file.data));
|
||||
}
|
||||
}
|
||||
|
||||
function fsList(path: string) {
|
||||
const entries: { name: string; type: 'file' | 'directory'; size: number }[] = [];
|
||||
try {
|
||||
const items = self.pyodide.FS.readdir(path).filter(
|
||||
(n: string) => n !== '.' && n !== '..'
|
||||
);
|
||||
for (const name of items) {
|
||||
try {
|
||||
const stat = self.pyodide.FS.stat(`${path}/${name}`);
|
||||
const isDir = self.pyodide.FS.isDir(stat.mode);
|
||||
entries.push({
|
||||
name,
|
||||
type: isDir ? 'directory' : 'file',
|
||||
size: isDir ? 0 : stat.size
|
||||
});
|
||||
} catch {
|
||||
// skip inaccessible entries
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// directory doesn't exist
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function fsRead(path: string): ArrayBuffer {
|
||||
const data: Uint8Array = (self.pyodide.FS as any).readFile(path) as Uint8Array;
|
||||
return data.buffer as ArrayBuffer;
|
||||
}
|
||||
|
||||
function fsDelete(path: string) {
|
||||
try {
|
||||
const stat = self.pyodide.FS.stat(path);
|
||||
if (self.pyodide.FS.isDir(stat.mode)) {
|
||||
// Recursively delete directory contents
|
||||
const items = self.pyodide.FS.readdir(path).filter(
|
||||
(n: string) => n !== '.' && n !== '..'
|
||||
);
|
||||
for (const item of items) {
|
||||
fsDelete(`${path}/${item}`);
|
||||
}
|
||||
self.pyodide.FS.rmdir(path);
|
||||
} else {
|
||||
self.pyodide.FS.unlink(path);
|
||||
}
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
}
|
||||
|
||||
function fsMkdir(path: string) {
|
||||
self.pyodide.FS.mkdirTree(path);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Code execution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function executeCode(id: string, code: string, files?: { name: string; data: ArrayBuffer }[]) {
|
||||
self.stdout = null;
|
||||
self.stderr = null;
|
||||
self.result = null;
|
||||
|
||||
// Upload any accompanying files before execution
|
||||
if (files && files.length > 0) {
|
||||
fsUploadFiles(files);
|
||||
persistFS();
|
||||
}
|
||||
|
||||
try {
|
||||
// check if matplotlib is imported in the code
|
||||
@@ -113,25 +226,90 @@ matplotlib.pyplot.show = show`);
|
||||
|
||||
console.log('Python result:', self.result);
|
||||
|
||||
// Persist any changes to IndexedDB
|
||||
// await new Promise<void>((resolve, reject) => {
|
||||
// self.pyodide.FS.syncfs(false, (err) => {
|
||||
// if (err) {
|
||||
// console.error('Error syncing to IndexedDB:', err);
|
||||
// reject(err);
|
||||
// } else {
|
||||
// console.log('Successfully synced to IndexedDB.');
|
||||
// resolve();
|
||||
// }
|
||||
// });
|
||||
// });
|
||||
} catch (error) {
|
||||
self.stderr = error.toString();
|
||||
// Persist any files the code may have written
|
||||
persistFS();
|
||||
} catch (error: unknown) {
|
||||
self.stderr = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
self.postMessage({ id, result: self.result, stdout: self.stdout, stderr: self.stderr });
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Message handler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
self.onmessage = async (event) => {
|
||||
const data = event.data;
|
||||
const { id, type } = data;
|
||||
|
||||
// Backward compatibility: messages without a `type` field are execute requests
|
||||
if (!type || type === 'execute') {
|
||||
const { code, files, ...context } = data;
|
||||
|
||||
// Copy context keys (packages, etc.) into worker scope
|
||||
for (const key of Object.keys(context)) {
|
||||
if (key !== 'id' && key !== 'type') {
|
||||
self[key] = context[key];
|
||||
}
|
||||
}
|
||||
|
||||
await ensurePyodide(self.packages);
|
||||
await executeCode(id, code, files);
|
||||
return;
|
||||
}
|
||||
|
||||
// FS operations require Pyodide to be loaded
|
||||
await ensurePyodide();
|
||||
|
||||
switch (type) {
|
||||
case 'fs:upload': {
|
||||
const { files, dir } = data;
|
||||
fsUploadFiles(files, dir);
|
||||
persistFS();
|
||||
self.postMessage({ id, type: 'fs:upload', success: true });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'fs:list': {
|
||||
const entries = fsList(data.path);
|
||||
self.postMessage({ id, type: 'fs:list', entries });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'fs:read': {
|
||||
try {
|
||||
const buffer = fsRead(data.path);
|
||||
self.postMessage({ id, type: 'fs:read', data: buffer }, { transfer: [buffer] });
|
||||
} catch (err: unknown) {
|
||||
self.postMessage({ id, type: 'fs:read', error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'fs:delete': {
|
||||
fsDelete(data.path);
|
||||
persistFS();
|
||||
self.postMessage({ id, type: 'fs:delete', success: true });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'fs:mkdir': {
|
||||
fsMkdir(data.path);
|
||||
persistFS();
|
||||
self.postMessage({ id, type: 'fs:mkdir', success: true });
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
console.warn('Unknown message type:', type);
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function processResult(result: any): any {
|
||||
// Catch and always return JSON-safe string representations
|
||||
try {
|
||||
@@ -167,9 +345,9 @@ function processResult(result: any): any {
|
||||
}
|
||||
// Stringify anything that's left (e.g., Proxy objects that cannot be directly processed)
|
||||
return JSON.stringify(result);
|
||||
} catch (err) {
|
||||
} catch (err: unknown) {
|
||||
// In case something unexpected happens, we return a stringified fallback
|
||||
return `[processResult error]: ${err.message || err.toString()}`;
|
||||
return `[processResult error]: ${err instanceof Error ? err.message : String(err)}`;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+66
-13
@@ -34,8 +34,10 @@
|
||||
terminalServers,
|
||||
showControls,
|
||||
showFileNavPath,
|
||||
showFileNavDir
|
||||
showFileNavDir,
|
||||
pyodideWorker
|
||||
} from '$lib/stores';
|
||||
import { getFileContentById } from '$lib/apis/files';
|
||||
import { goto } from '$app/navigation';
|
||||
import { page } from '$app/stores';
|
||||
import { beforeNavigate } from '$app/navigation';
|
||||
@@ -184,7 +186,20 @@
|
||||
});
|
||||
};
|
||||
|
||||
const executePythonAsWorker = async (id, code, cb) => {
|
||||
/**
|
||||
* Get or create the persistent Pyodide worker.
|
||||
* The worker persists across executions so the virtual FS (IDBFS) is preserved.
|
||||
*/
|
||||
const getOrCreateWorker = () => {
|
||||
let worker = $pyodideWorker;
|
||||
if (!worker) {
|
||||
worker = new PyodideWorker();
|
||||
pyodideWorker.set(worker);
|
||||
}
|
||||
return worker;
|
||||
};
|
||||
|
||||
const executePythonAsWorker = async (id, code, cb, files = []) => {
|
||||
let result = null;
|
||||
let stdout = null;
|
||||
let stderr = null;
|
||||
@@ -206,19 +221,44 @@
|
||||
/\bimport\s+pytz\b|\bfrom\s+pytz\b/.test(code) ? 'pytz' : null
|
||||
].filter(Boolean);
|
||||
|
||||
const pyodideWorker = new PyodideWorker();
|
||||
const worker = getOrCreateWorker();
|
||||
|
||||
pyodideWorker.postMessage({
|
||||
// Fetch file content from the server and prepare for the worker
|
||||
let filePayloads = [];
|
||||
if (files && files.length > 0) {
|
||||
for (const file of files) {
|
||||
try {
|
||||
const fileId = file?.id;
|
||||
const fileName = file?.filename || file?.name || 'file';
|
||||
if (fileId) {
|
||||
const content = await getFileContentById(fileId);
|
||||
if (content) {
|
||||
filePayloads.push({ name: fileName, data: content });
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to fetch file for Pyodide:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
worker.postMessage({
|
||||
type: 'execute',
|
||||
id: id,
|
||||
code: code,
|
||||
packages: packages
|
||||
packages: packages,
|
||||
files: filePayloads.length > 0 ? filePayloads : undefined
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
// Timeout for this specific execution (not the worker itself)
|
||||
let timeoutId = setTimeout(() => {
|
||||
if (executing) {
|
||||
executing = false;
|
||||
stderr = 'Execution Time Limit Exceeded';
|
||||
pyodideWorker.terminate();
|
||||
|
||||
// Terminate and recreate the worker on timeout
|
||||
worker.terminate();
|
||||
pyodideWorker.set(null);
|
||||
|
||||
if (cb) {
|
||||
cb(
|
||||
@@ -237,11 +277,18 @@
|
||||
}
|
||||
}, 60000);
|
||||
|
||||
pyodideWorker.onmessage = (event) => {
|
||||
console.log('pyodideWorker.onmessage', event);
|
||||
const { id, ...data } = event.data;
|
||||
// Use addEventListener so multiple concurrent executions don't clobber each other
|
||||
const onMessage = (event) => {
|
||||
const { id: eventId, ...data } = event.data;
|
||||
// Only handle responses for this execution ID
|
||||
if (eventId !== id) return;
|
||||
// Ignore FS responses (they use a type field)
|
||||
if (data.type && data.type.startsWith('fs:')) return;
|
||||
|
||||
console.log(id, data);
|
||||
console.log('pyodideWorker.onmessage', event);
|
||||
clearTimeout(timeoutId);
|
||||
worker.removeEventListener('message', onMessage);
|
||||
worker.removeEventListener('error', onError);
|
||||
|
||||
data['stdout'] && (stdout = data['stdout']);
|
||||
data['stderr'] && (stderr = data['stderr']);
|
||||
@@ -265,8 +312,11 @@
|
||||
executing = false;
|
||||
};
|
||||
|
||||
pyodideWorker.onerror = (event) => {
|
||||
const onError = (event) => {
|
||||
console.log('pyodideWorker.onerror', event);
|
||||
clearTimeout(timeoutId);
|
||||
worker.removeEventListener('message', onMessage);
|
||||
worker.removeEventListener('error', onError);
|
||||
|
||||
if (cb) {
|
||||
cb(
|
||||
@@ -284,6 +334,9 @@
|
||||
}
|
||||
executing = false;
|
||||
};
|
||||
|
||||
worker.addEventListener('message', onMessage);
|
||||
worker.addEventListener('error', onError);
|
||||
};
|
||||
|
||||
const resolveToolServer = (serverUrl) => {
|
||||
@@ -423,7 +476,7 @@
|
||||
} else if (data?.session_id === $socket.id) {
|
||||
if (type === 'execute:python') {
|
||||
console.log('execute:python', data);
|
||||
executePythonAsWorker(data.id, data.code, cb);
|
||||
executePythonAsWorker(data.id, data.code, cb, data.files || []);
|
||||
} else if (type === 'execute:tool') {
|
||||
console.log('execute:tool', data);
|
||||
executeTool(data, cb);
|
||||
|
||||
Reference in New Issue
Block a user