From a0d1d4f114c1b6cc3e70634cea34cdbf4a80381c Mon Sep 17 00:00:00 2001 From: vikingowl Date: Wed, 7 Jan 2026 15:08:29 +0100 Subject: [PATCH] feat: add embedding model selector and non-blocking file upload - Add embedding model dropdown to project file upload - Create addDocumentAsync that stores immediately, embeds in background - Add embeddingStatus field to track pending/processing/ready/failed - Show status indicator and text for each document - Upload no longer blocks the UI - files appear immediately - Background embedding shows toast notifications on completion/error --- frontend/src/lib/memory/index.ts | 4 +- frontend/src/lib/memory/vector-store.ts | 114 +++++++++++++++++- frontend/src/lib/storage/db.ts | 2 + .../src/routes/projects/[id]/+page.svelte | 81 +++++++++---- 4 files changed, 174 insertions(+), 27 deletions(-) diff --git a/frontend/src/lib/memory/index.ts b/frontend/src/lib/memory/index.ts index a837390..304b0c3 100644 --- a/frontend/src/lib/memory/index.ts +++ b/frontend/src/lib/memory/index.ts @@ -69,6 +69,7 @@ export { // Vector store export { addDocument, + addDocumentAsync, searchSimilar, listDocuments, getDocument, @@ -77,5 +78,6 @@ export { getKnowledgeBaseStats, formatResultsAsContext, type SearchResult, - type AddDocumentOptions + type AddDocumentOptions, + type AddDocumentAsyncOptions } from './vector-store.js'; diff --git a/frontend/src/lib/memory/vector-store.ts b/frontend/src/lib/memory/vector-store.ts index 6e19a23..bd5ee44 100644 --- a/frontend/src/lib/memory/vector-store.ts +++ b/frontend/src/lib/memory/vector-store.ts @@ -92,7 +92,8 @@ export async function addDocument( updatedAt: now, chunkCount: storedChunks.length, embeddingModel, - projectId: projectId ?? null + projectId: projectId ?? null, + embeddingStatus: 'ready' }; // Store in database @@ -104,6 +105,117 @@ export async function addDocument( return document; } +/** Options for async document upload */ +export interface AddDocumentAsyncOptions extends AddDocumentOptions { + /** Callback when embedding generation completes */ + onComplete?: (doc: StoredDocument) => void; + /** Callback when embedding generation fails */ + onError?: (error: Error) => void; +} + +/** + * Add a document asynchronously - stores immediately, generates embeddings in background + * Returns immediately with the document in 'pending' state + */ +export async function addDocumentAsync( + name: string, + content: string, + mimeType: string, + options: AddDocumentAsyncOptions = {} +): Promise { + const { + chunkOptions, + embeddingModel = DEFAULT_EMBEDDING_MODEL, + onProgress, + onComplete, + onError, + projectId + } = options; + + const documentId = crypto.randomUUID(); + const now = Date.now(); + + // Chunk the content + const textChunks = chunkText(content, documentId, chunkOptions); + + if (textChunks.length === 0) { + throw new Error('Document produced no chunks'); + } + + // Create document record immediately (without embeddings) + const document: StoredDocument = { + id: documentId, + name, + mimeType, + size: content.length, + createdAt: now, + updatedAt: now, + chunkCount: textChunks.length, + embeddingModel, + projectId: projectId ?? null, + embeddingStatus: 'pending' + }; + + // Store document immediately + await db.documents.add(document); + + // Generate embeddings in background (non-blocking) + setTimeout(async () => { + try { + // Update status to processing + await db.documents.update(documentId, { embeddingStatus: 'processing' }); + + const chunkContents = textChunks.map(c => c.content); + const embeddings: number[][] = []; + + // Process in batches with progress + const BATCH_SIZE = 5; + for (let i = 0; i < chunkContents.length; i += BATCH_SIZE) { + const batch = chunkContents.slice(i, i + BATCH_SIZE); + const batchEmbeddings = await generateEmbeddings(batch, embeddingModel); + embeddings.push(...batchEmbeddings); + + if (onProgress) { + onProgress(Math.min(i + BATCH_SIZE, chunkContents.length), chunkContents.length); + } + } + + // Create stored chunks with embeddings + const storedChunks: StoredChunk[] = textChunks.map((chunk, index) => ({ + id: chunk.id, + documentId, + content: chunk.content, + embedding: embeddings[index], + startIndex: chunk.startIndex, + endIndex: chunk.endIndex, + tokenCount: estimateChunkTokens(chunk.content) + })); + + // Store chunks and update document status + await db.transaction('rw', [db.documents, db.chunks], async () => { + await db.chunks.bulkAdd(storedChunks); + await db.documents.update(documentId, { + embeddingStatus: 'ready', + updatedAt: Date.now() + }); + }); + + const updatedDoc = await db.documents.get(documentId); + if (updatedDoc && onComplete) { + onComplete(updatedDoc); + } + } catch (error) { + console.error('Failed to generate embeddings:', error); + await db.documents.update(documentId, { embeddingStatus: 'failed' }); + if (onError) { + onError(error instanceof Error ? error : new Error(String(error))); + } + } + }, 0); + + return document; +} + /** * Search for similar chunks across all documents */ diff --git a/frontend/src/lib/storage/db.ts b/frontend/src/lib/storage/db.ts index a39e385..6eba555 100644 --- a/frontend/src/lib/storage/db.ts +++ b/frontend/src/lib/storage/db.ts @@ -157,6 +157,8 @@ export interface StoredDocument { embeddingModel: string; /** Optional project ID - if set, document is project-scoped */ projectId?: string | null; + /** Embedding generation status: 'pending' | 'processing' | 'ready' | 'failed' */ + embeddingStatus?: 'pending' | 'processing' | 'ready' | 'failed'; } /** diff --git a/frontend/src/routes/projects/[id]/+page.svelte b/frontend/src/routes/projects/[id]/+page.svelte index 91e11d7..da1dcad 100644 --- a/frontend/src/routes/projects/[id]/+page.svelte +++ b/frontend/src/routes/projects/[id]/+page.svelte @@ -11,9 +11,10 @@ import { getProjectStats, getProjectLinks, type ProjectLink } from '$lib/storage/projects.js'; import { listDocuments, - addDocument, + addDocumentAsync, deleteDocument, - DEFAULT_EMBEDDING_MODEL + DEFAULT_EMBEDDING_MODEL, + EMBEDDING_MODELS } from '$lib/memory'; import type { StoredDocument } from '$lib/storage/db'; import ProjectModal from '$lib/components/projects/ProjectModal.svelte'; @@ -40,7 +41,7 @@ let links = $state([]); let documents = $state([]); let isLoadingDocs = $state(false); - let isUploading = $state(false); + let selectedEmbeddingModel = $state(DEFAULT_EMBEDDING_MODEL); let activeTab = $state<'chats' | 'files' | 'links'>('chats'); let fileInput: HTMLInputElement; let dragOver = $state(false); @@ -186,8 +187,6 @@ return; } - isUploading = true; - for (const file of files) { try { const content = await file.text(); @@ -196,13 +195,21 @@ continue; } - // Add document with projectId - await addDocument(file.name, content, file.type || 'text/plain', { - embeddingModel: DEFAULT_EMBEDDING_MODEL, - projectId: projectId + // Add document async - stores immediately, embeds in background + await addDocumentAsync(file.name, content, file.type || 'text/plain', { + embeddingModel: selectedEmbeddingModel, + projectId: projectId, + onComplete: (doc) => { + toastState.success(`Embeddings ready for "${doc.name}"`); + loadProjectData(); // Refresh to show updated status + }, + onError: (error) => { + toastState.error(`Embedding failed for "${file.name}": ${error.message}`); + loadProjectData(); // Refresh to show failed status + } }); - toastState.success(`Added "${file.name}" to project`); + toastState.info(`Added "${file.name}" - generating embeddings...`); } catch (error) { console.error(`Failed to process ${file.name}:`, error); const message = error instanceof Error ? error.message : 'Unknown error'; @@ -210,12 +217,8 @@ } } - try { - await loadProjectData(); - } catch (err) { - console.error('Failed to reload project data:', err); - } - isUploading = false; + // Refresh immediately to show pending documents + await loadProjectData(); } async function handleDeleteDocument(doc: StoredDocument) { @@ -426,6 +429,20 @@ {/if} {:else if activeTab === 'files'} + +
+

Embedding Model

+ +
+

- {#if isUploading} - Uploading files... - {:else} - Drag & drop files here, or - - {/if} + Drag & drop files here, or +

Text files, code, markdown, JSON, etc. @@ -467,13 +480,31 @@ {#each documents as doc (doc.id)}

- - - + + {#if doc.embeddingStatus === 'pending' || doc.embeddingStatus === 'processing'} + + + + {:else if doc.embeddingStatus === 'failed'} + + + + {:else} + + + + {/if}

{doc.name}

{formatSize(doc.size)} + {#if doc.embeddingStatus === 'pending'} + • Queued + {:else if doc.embeddingStatus === 'processing'} + • Generating embeddings... + {:else if doc.embeddingStatus === 'failed'} + • Embedding failed + {/if}