From fae74ade95216054a1bbdbb68ede40daa8432a53 Mon Sep 17 00:00:00 2001 From: Florian Egger Date: Tue, 19 May 2026 23:00:04 +0200 Subject: [PATCH] feat: add semantic/RAG vault indexing with automatic background sync - Add VaultVectorStore backed by ChromaDB for vector-based vault search - Integrate existing ContentVectorizer/IndexingPipeline for embeddings - Update VaultIndexer to prefer semantic search with keyword fallback - Background indexing on plugin load + incremental sync via vault events - Add vault index settings, commands, and UI controls - Add tests for VaultVectorStore - Update README with RAG setup instructions --- README.md | 29 +- main.js | 673 ++++++++++++++++++++++++- src/chat-view.ts | 11 +- src/constants.ts | 7 + src/main.ts | 242 ++++++++- src/types.ts | 10 + src/vault-indexer.ts | 106 +++- src/vault-vector-store.ts | 242 +++++++++ src/workflow-engine/workflow-engine.ts | 6 +- tests/chat-view.test.ts | 7 + tests/vault-vector-store.test.ts | 222 ++++++++ 11 files changed, 1528 insertions(+), 27 deletions(-) create mode 100644 src/vault-vector-store.ts create mode 100644 tests/vault-vector-store.test.ts diff --git a/README.md b/README.md index 89de3d0..12ea593 100755 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ A plugin that integrates [Ollama](https://ollama.ai) with Obsidian, allowing you - Chat with Ollama models directly in Obsidian - Vault context search — the assistant can reference your notes +- **Semantic/RAG vault indexing** — automatically index your vault into a vector database for intelligent retrieval - Tool integration — create files based on chat responses - Streaming responses - Semantic response cache — repeated or similar queries are answered instantly without hitting the model (requires ChromaDB) @@ -17,6 +18,24 @@ A plugin that integrates [Ollama](https://ollama.ai) with Obsidian, allowing you 2. **Start Ollama**: `ollama serve` 3. **Pull a chat model**: `ollama pull llama3` (or any other model you prefer) +### Optional — Vault Semantic Index (RAG) + +The vault semantic index automatically indexes your Obsidian notes into a local [ChromaDB](https://www.trychroma.com) vector database. When you ask a question, the plugin performs semantic search against your notes and includes the most relevant passages as context for the AI. + +1. **Install ChromaDB**: + ```bash + pip install chromadb + ``` +2. **Start ChromaDB**: + ```bash + chroma run --host localhost --port 8000 + ``` +3. **Pull an embedding model**: + ```bash + ollama pull nomic-embed-text + ``` +4. Enable the vault semantic index in the plugin settings and configure the ChromaDB URL. + ### Optional — Semantic Cache The semantic cache stores responses in a local [ChromaDB](https://www.trychroma.com) vector database. When you ask a question that is semantically similar to one already cached, the stored answer is returned immediately instead of calling the model. @@ -89,6 +108,12 @@ Open **Settings → Ollama Settings** to configure the plugin. | Model | `llama3` | Model used for chat responses | | Vault Search Limit | `3` | Maximum number of vault entries to include in context | | Max Message History | `50` | Maximum number of messages kept in conversation history | +| **Enable Vault Semantic Index** | Off | Index vault notes into a vector DB for semantic/RAG search | +| Vault Index ChromaDB URL | `http://localhost:8000` | URL of your ChromaDB instance for the vault index | +| Vault Index Embedding Model | `nomic-embed-text` | Ollama model used to generate vault embeddings | +| Vault Index Similarity Threshold | `0.75` | Minimum cosine similarity (0–1) for a vault search hit | +| Rebuild Vault Index | — | Button to rebuild the entire vault semantic index | +| Clear Vault Index | — | Button to delete all indexed vault notes | | Enable Semantic Cache | Off | Cache responses for fast repeated queries | | ChromaDB URL | `http://localhost:8000` | URL of your running ChromaDB instance | | Cache Embedding Model | `nomic-embed-text` | Ollama model used to generate cache embeddings | @@ -112,7 +137,7 @@ Open **Settings → Ollama Settings** to configure the plugin. ## Vault Context -When you send a message, the plugin automatically searches your vault for relevant notes and includes them as context. The search uses weighted scoring: +When you send a message, the plugin searches your vault for relevant notes and includes them as context. If the **Vault Semantic Index** is enabled, search is performed via semantic/RAG retrieval using vector embeddings. Otherwise, it falls back to a weighted keyword search: - **Headings** — 5x weight - **Frontmatter title** — 3x weight @@ -120,6 +145,8 @@ When you send a message, the plugin automatically searches your vault for releva - **First paragraph** — 1.5x weight - **General content** — 1x weight +The plugin automatically watches your vault for changes (create, modify, delete, rename) and updates the semantic index in real time when enabled. + ## Tools The plugin exposes a `create_file` tool that allows the AI to create new markdown files in your vault. Paths are validated for safety (no `.obsidian`/`.git` access, no path traversal). diff --git a/main.js b/main.js index a3333ce..d33e6bc 100644 --- a/main.js +++ b/main.js @@ -8382,7 +8382,7 @@ var STOP_WORDS = /* @__PURE__ */ new Set([ ]); var CONTENT_PREVIEW_LENGTH = 500; var VaultIndexer = class { - constructor(vault, cache) { + constructor(vault, cache, vectorStore) { this.SCORING_WEIGHTS = { TITLE: 5, FRONTMATTER_TITLE: 4, @@ -8392,6 +8392,10 @@ var VaultIndexer = class { }; this.vault = vault; this.cache = cache; + this.vectorStore = vectorStore; + } + setVectorStore(vectorStore) { + this.vectorStore = vectorStore; } tokenize(text) { return text.toLowerCase().replace(/[^\w\s]/g, "").split(/\s+/).filter((token) => token.length > 1 && !STOP_WORDS.has(token)); @@ -8461,6 +8465,20 @@ var VaultIndexer = class { if (!query || !query.trim()) { return []; } + if (this.vectorStore) { + try { + const semanticResults = await this.vectorStore.search(query, limit); + if (semanticResults.length > 0) { + return semanticResults; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn( + `Semantic search failed, falling back to keyword search: ${errorMessage}`, + "vault-indexer" + ); + } + } const cacheKey = `query:${query.trim()}:limit:${limit}`; if (this.cache) { let cachedResults = null; @@ -8497,7 +8515,13 @@ var VaultIndexer = class { return { ...entry, score }; }).filter((e) => e.score > 0); scored.sort((a, b) => b.score - a.score); - const results = scored.slice(0, limit); + const results = scored.slice(0, limit).map((e) => ({ + path: e.file.path, + title: e.title, + content: e.content, + score: e.score, + tags: e.frontmatter?.tags + })); if (this.cache) { try { await this.cache.put(cacheKey, JSON.stringify(results)); @@ -8933,7 +8957,7 @@ var ErrorHandler = class { // src/chat-view.ts var ChatView = class extends import_obsidian3.ItemView { - constructor(leaf, settings) { + constructor(leaf, settings, vectorStore) { super(leaf); // State this.messages = []; @@ -8969,7 +8993,7 @@ var ChatView = class extends import_obsidian3.ItemView { void 0, settings.cacheConfig ); - this.vaultIndexer = new VaultIndexer(this.app.vault); + this.vaultIndexer = new VaultIndexer(this.app.vault, void 0, vectorStore); this.toolExecutor = new ToolExecutor(this.app.vault, this.app); this.conversationStateManager = new ConversationStateManager(); } @@ -8997,6 +9021,10 @@ var ChatView = class extends import_obsidian3.ItemView { ); }); } + setVectorStore(vectorStore) { + this.vectorStore = vectorStore; + this.vaultIndexer.setVectorStore(vectorStore); + } async clearCache() { await this.ollamaClient.clearCache(); } @@ -9402,6 +9430,465 @@ var DEFAULT_SETTINGS = { collectionName: "ollama_semantic_cache", embeddingModel: "nomic-embed-text", chromaURL: "http://localhost:8000" + }, + vaultIndexConfig: { + enabled: false, + similarityThreshold: 0.75, + collectionName: "ollama_vault_index", + embeddingModel: "nomic-embed-text", + chromaURL: "http://localhost:8000" + } +}; + +// src/indexing-pipeline/vectorization.ts +var ContentVectorizer = class { + constructor(config, fetchFn) { + this.model = config.model; + this.ollamaUrl = config.ollamaUrl; + this.fetchFn = fetchFn ?? ((url, init) => fetch(url, init)); + } + /** + * Generates embeddings for a content chunk + */ + async vectorize(chunk) { + try { + const prompt = this.createPrompt(chunk); + const response = await this.fetchFn(`${this.ollamaUrl}/api/embeddings`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: this.model, + prompt + }) + }); + if (!response.ok) { + throw new Error(`Embedding failed with status ${response.status}`); + } + const data = await response.json(); + if (!this.isEmbeddingResponse(data)) { + throw new Error("Invalid embedding response"); + } + return data.embedding; + } catch (error) { + Logger.warn(`Failed to generate embedding: ${String(error)}`, "indexing-pipeline"); + return []; + } + } + isEmbeddingResponse(data) { + return typeof data === "object" && data !== null && Array.isArray(data.embedding) && data.embedding.every((value) => typeof value === "number"); + } + /** + * Creates a prompt from content chunk for embedding + */ + createPrompt(chunk) { + const parts = [ + chunk.title, + chunk.firstParagraph, + chunk.content.substring(0, 1e3), + // Limit content to avoid long prompts + chunk.headings.join(" "), + JSON.stringify(chunk.frontmatter) + ].filter(Boolean); + return parts.join("\n\n"); + } +}; + +// src/indexing-pipeline/extraction.ts +var ContentExtractor = class { + extractFromFile(file, content) { + const frontmatter = {}; + const headings = []; + const embeddedCodeBlocks = []; + let firstParagraph; + const frontmatterMatch = content.match(/^---(.*?)---/s); + if (frontmatterMatch) { + try { + const frontmatterContent = frontmatterMatch[1]; + const lines = frontmatterContent.trim().split("\n"); + for (const line of lines) { + const [key, ...valueParts] = line.split(":"); + if (!key) continue; + const value = valueParts.join(":").trim(); + if (key.trim() === "title") { + if (value) { + frontmatter.title = value; + } + } else if (key.trim() === "tags") { + if (value) { + frontmatter.tags = value; + } + } else { + frontmatter[key.trim()] = value; + } + } + } catch { + } + } + const headingMatches = content.match(/^#{1,6} (.*?)$/gm); + if (headingMatches) { + headings.push(...headingMatches.map((h) => h.replace(/^#{1,6} /, ""))); + } + const codeBlockMatches = content.match(/```([\s\S]*?)```/g); + if (codeBlockMatches) { + embeddedCodeBlocks.push(...codeBlockMatches); + } + const paragraphMatch = content.match(/^([^#]*?)(?=#|\n\n|$)/s); + if (paragraphMatch) { + firstParagraph = paragraphMatch[1].trim(); + } + return { + basename: file.basename, + path: file.path, + content, + frontmatter, + headings, + embeddedCodeBlocks, + firstParagraph + }; + } + /** + * Extracts just the raw text content without headers, frontmatter, etc. + */ + extractRawText(content) { + return content.replace(/^---.*?---/s, "").replace(/^#.*?$/gm, "").replace(/```.*?```/gs, "").replace(/`.*?`/g, "").replace(/\[(.*?)\]\(.*?\)/g, "$1").replace(/\*\*(.*?)\*\*/g, "$1").replace(/\*(.*?)\*/g, "$1").trim(); + } +}; + +// src/indexing-pipeline/normalization.ts +var ContentNormalizer = class { + /** + * Normalizes content by: + * - Standardizing dates to ISO 8601 + * - Converting to lowercase for tokenization + * - Extracting tokens + * - Adding metadata + */ + normalize(extractedContent) { + const { basename, path, content, frontmatter, headings, firstParagraph } = extractedContent; + const title = basename.replace(/\.md$/, ""); + const tokens = this.tokenize(content); + const normalizedFrontmatter = this.normalizeFrontmatter(frontmatter); + const wordCount = content.split(/\s+/).filter(Boolean).length; + return { + path, + title, + content, + tokens, + headings, + frontmatter: normalizedFrontmatter, + firstParagraph, + wordCount, + // Add timestamps if available in frontmatter + createdAt: this.extractDate(frontmatter, "created") || this.extractDate(frontmatter, "date"), + updatedAt: this.extractDate(frontmatter, "updated") + }; + } + /** + * Tokenizes text content by splitting on whitespace and removing stop words + */ + tokenize(text) { + const stopWords = /* @__PURE__ */ new Set([ + "the", + "a", + "an", + "and", + "or", + "but", + "is", + "are", + "was", + "were", + "in", + "on", + "at", + "to", + "of", + "for", + "with", + "as", + "by", + "it", + "its", + "that", + "this", + "these", + "those", + "from", + "up", + "out", + "off", + "over", + "under", + "again", + "further", + "then", + "once", + "here", + "there", + "when", + "where", + "why", + "how", + "all", + "any", + "both", + "each", + "few", + "more", + "most", + "other", + "some", + "such", + "no", + "nor", + "not", + "only", + "own", + "same", + "so", + "than", + "too", + "very", + "just", + "now" + ]); + return text.toLowerCase().split(/\W+/).filter((token) => token.length > 1 && !stopWords.has(token)); + } + /** + * Normalizes frontmatter by standardizing data types and formats + */ + normalizeFrontmatter(frontmatter) { + const normalized = {}; + for (const [key, value] of Object.entries(frontmatter)) { + if (key === "tags" && typeof value === "string") { + normalized.tags = value.split(",").map((tag) => tag.trim()); + } else if (key === "date" || key === "created" || key === "updated") { + if (typeof value === "string") { + const date = new Date(value); + if (!isNaN(date.getTime())) { + normalized[key] = date.toISOString(); + } else { + normalized[key] = value; + } + } else { + normalized[key] = value; + } + } else { + normalized[key] = value; + } + } + return normalized; + } + /** + * Extracts a date from frontmatter + */ + extractDate(frontmatter, key) { + const value = frontmatter[key]; + if (typeof value === "string") { + const date = new Date(value); + if (!isNaN(date.getTime())) { + return date.toISOString(); + } + } + return void 0; + } +}; + +// src/vault-vector-store.ts +var VaultVectorStore = class { + constructor(ollamaURL, config) { + this.client = null; + this.collection = null; + this.isInitialized = false; + this.ollamaURL = ollamaURL.replace(/\/+$/, ""); + this.config = config; + this.vectorizer = new ContentVectorizer({ + model: config.embeddingModel, + ollamaUrl: this.ollamaURL + }); + this.extractor = new ContentExtractor(); + this.normalizer = new ContentNormalizer(); + } + async initialize() { + if (!this.config.enabled || this.isInitialized) return; + try { + const rawURL = this.config.chromaURL?.trim() || "http://localhost:8000"; + const chromaURL = rawURL.includes("://") ? rawURL : "http://localhost:8000"; + this.client = new ChromaClient({ path: chromaURL }); + this.collection = await this.client.getOrCreateCollection({ + name: this.config.collectionName, + metadata: { "hnsw:space": "cosine" } + }); + this.isInitialized = true; + Logger.info( + `Vault vector store initialized: ${this.config.collectionName}`, + "vault-vector-store" + ); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.error( + `Failed to initialize vault vector store: ${errorMessage}`, + "vault-vector-store" + ); + throw error; + } + } + /** + * Index a single vault file by generating an embedding and storing it in ChromaDB. + */ + async indexFile(file, content) { + if (!this.collection || !this.config.enabled) return; + if (!content.trim()) { + await this.deleteFile(file.path); + return; + } + try { + const extracted = this.extractor.extractFromFile( + { basename: file.basename, path: file.path }, + content + ); + const normalized = this.normalizer.normalize(extracted); + const chunk = { + id: file.path, + path: file.path, + title: normalized.title, + content: normalized.content, + tokens: normalized.tokens, + headings: normalized.headings, + frontmatter: normalized.frontmatter, + firstParagraph: normalized.firstParagraph, + wordCount: normalized.wordCount, + chunkIndex: 0, + chunkSize: 1 + }; + const embedding = await this.vectorizer.vectorize(chunk); + if (embedding.length === 0) { + Logger.warn(`Empty embedding for ${file.path}, skipping index`, "vault-vector-store"); + return; + } + await this.collection.upsert({ + ids: [file.path], + documents: [this.createDocumentText(normalized)], + embeddings: [embedding], + metadatas: [ + { + path: file.path, + title: normalized.title, + tags: typeof normalized.frontmatter.tags === "string" ? normalized.frontmatter.tags : "" + } + ] + }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn(`Failed to index file ${file.path}: ${errorMessage}`, "vault-vector-store"); + } + } + /** + * Remove a file from the vector index. + */ + async deleteFile(path) { + if (!this.collection || !this.config.enabled) return; + try { + await this.collection.delete({ ids: [path] }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn( + `Failed to delete file ${path} from index: ${errorMessage}`, + "vault-vector-store" + ); + } + } + /** + * Perform semantic search against the vault index. + */ + async search(query, limit = 3) { + if (!this.collection || !this.config.enabled) { + return []; + } + try { + const queryEmbedding = await this.vectorizer.vectorize({ + id: "query", + path: "query", + title: query, + content: query, + tokens: query.toLowerCase().split(/\s+/).filter((t) => t.length > 1), + headings: [query], + frontmatter: {}, + firstParagraph: query, + wordCount: query.split(/\s+/).length, + chunkIndex: 0, + chunkSize: 1 + }); + if (queryEmbedding.length === 0) { + Logger.warn("Empty query embedding, skipping semantic search", "vault-vector-store"); + return []; + } + const results = await this.collection.query({ + queryEmbeddings: [queryEmbedding], + nResults: limit + }); + if (!results.ids[0] || results.ids[0].length === 0) { + return []; + } + const entries = []; + for (let i = 0; i < results.ids[0].length; i++) { + const distance = results.distances?.[0]?.[i] ?? 0; + const score = 1 - distance; + if (score < this.config.similarityThreshold) { + continue; + } + const metadata = results.metadatas?.[0]?.[i]; + const document = results.documents?.[0]?.[i]; + entries.push({ + path: metadata?.path || String(results.ids[0][i]), + title: metadata?.title || "Untitled", + content: document || "", + score, + tags: metadata?.tags + }); + } + entries.sort((a, b) => b.score - a.score); + return entries.slice(0, limit); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn(`Semantic search failed: ${errorMessage}`, "vault-vector-store"); + return []; + } + } + /** + * Clear the entire vault index. + */ + async clearIndex() { + if (!this.client || !this.config.enabled) return; + try { + await this.client.deleteCollection({ name: this.config.collectionName }); + this.collection = null; + this.isInitialized = false; + Logger.info("Vault index cleared", "vault-vector-store"); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.error(`Failed to clear vault index: ${errorMessage}`, "vault-vector-store"); + } + } + /** + * Get the number of indexed documents. + */ + async getIndexedCount() { + if (!this.collection || !this.config.enabled) return 0; + try { + const result = await this.collection.count(); + return result; + } catch { + return 0; + } + } + createDocumentText(normalized) { + const parts = []; + if (normalized.title) parts.push(`Title: ${normalized.title}`); + if (normalized.firstParagraph) parts.push(normalized.firstParagraph); + if (normalized.headings.length > 0) parts.push(normalized.headings.join("\n")); + parts.push(normalized.content.substring(0, 1e3)); + return parts.join("\n\n"); } }; @@ -9413,9 +9900,12 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { } async onload() { await this.loadSettings(); + if (this.settings.vaultIndexConfig.enabled) { + await this.initializeVaultVectorStore(); + } this.registerView( "ollama-chat-view", - (leaf) => new ChatView(leaf, this.settings) + (leaf) => new ChatView(leaf, this.settings, this.vaultVectorStore) ); this.addRibbonIcon("bot", "Open Ollama Chat", async () => { await this.activateChatView(); @@ -9435,6 +9925,23 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { new import_obsidian4.Notice("Semantic cache cleared."); } }); + this.addCommand({ + id: "clear-vault-index", + name: "Clear Vault Index", + callback: async () => { + await this.clearVaultIndex(); + new import_obsidian4.Notice("Vault index cleared."); + } + }); + this.addCommand({ + id: "rebuild-vault-index", + name: "Rebuild Vault Index", + callback: async () => { + new import_obsidian4.Notice("Rebuilding vault index..."); + await this.rebuildVaultIndex(); + new import_obsidian4.Notice("Vault index rebuilt."); + } + }); this.addSettingTab(new OllamaSettingTab(this.app, this)); if (this.settings.cacheConfig) { this.semanticCache = new SemanticCacheService( @@ -9447,6 +9954,7 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { new import_obsidian4.Notice("Semantic cache initialization failed. Check console for details."); } } + this.registerVaultEventListeners(); } // eslint-disable-next-line @typescript-eslint/no-misused-promises onunload() { @@ -9461,6 +9969,89 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { async saveSettings() { await this.saveData(this.settings); } + async initializeVaultVectorStore() { + this.vaultVectorStore = new VaultVectorStore( + this.settings.ollamaUrl, + this.settings.vaultIndexConfig + ); + try { + await this.vaultVectorStore.initialize(); + void this.backgroundIndexVault().catch((err) => { + Logger.warn(`Background vault indexing failed: ${String(err)}`, "main"); + }); + } catch { + new import_obsidian4.Notice("Vault vector store initialization failed. Check console for details."); + } + } + async backgroundIndexVault() { + if (!this.vaultVectorStore) return; + const files = this.app.vault.getMarkdownFiles(); + Logger.info(`Starting background vault indexing for ${files.length} files...`, "main"); + let indexed = 0; + for (const file of files) { + try { + const content = await this.app.vault.read(file); + await this.vaultVectorStore.indexFile(file, content); + indexed++; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main"); + } + } + this.settings.lastIndexTime = Date.now(); + await this.saveSettings(); + Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, "main"); + new import_obsidian4.Notice(`Vault index updated: ${indexed} files indexed.`); + } + async rebuildVaultIndex() { + if (this.vaultVectorStore) { + await this.vaultVectorStore.clearIndex(); + await this.vaultVectorStore.initialize(); + } + await this.backgroundIndexVault(); + } + async clearVaultIndex() { + if (this.vaultVectorStore) { + await this.vaultVectorStore.clearIndex(); + } + } + registerVaultEventListeners() { + this.registerEvent( + this.app.vault.on("create", (file) => { + if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) { + void this.app.vault.read(file).then((content) => { + void this.vaultVectorStore?.indexFile(file, content); + }); + } + }) + ); + this.registerEvent( + this.app.vault.on("modify", (file) => { + if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) { + void this.app.vault.read(file).then((content) => { + void this.vaultVectorStore?.indexFile(file, content); + }); + } + }) + ); + this.registerEvent( + this.app.vault.on("delete", (file) => { + if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) { + void this.vaultVectorStore.deleteFile(file.path); + } + }) + ); + this.registerEvent( + this.app.vault.on("rename", (file, oldPath) => { + if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) { + void this.vaultVectorStore.deleteFile(oldPath); + void this.app.vault.read(file).then((content) => { + void this.vaultVectorStore?.indexFile(file, content); + }); + } + }) + ); + } async activateChatView() { const existing = this.app.workspace.getLeavesOfType("ollama-chat-view"); if (existing.length > 0) { @@ -9485,6 +10076,7 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { leaves.forEach((leaf) => { if (leaf.view instanceof ChatView) { leaf.view.updateSettings(this.settings); + leaf.view.setVectorStore(this.vaultVectorStore); } }); } @@ -9532,6 +10124,77 @@ var OllamaSettingTab = class extends import_obsidian4.PluginSettingTab { } }) ); + containerEl.createEl("h3", { text: "Vault Semantic Index" }); + new import_obsidian4.Setting(containerEl).setName("Enable Vault Semantic Index").setDesc( + "Automatically index vault notes into a vector database for semantic/RAG search. Requires ChromaDB and an embedding model." + ).addToggle( + (toggle) => toggle.setValue(this.plugin.settings.vaultIndexConfig.enabled).onChange(async (value) => { + this.plugin.settings.vaultIndexConfig.enabled = value; + await this.plugin.saveSettings(); + this.plugin.notifyChatViews(); + if (value) { + new import_obsidian4.Notice("Vault semantic index enabled. Rebuilding index..."); + await this.plugin.initializeVaultVectorStore(); + await this.plugin.rebuildVaultIndex(); + } else { + await this.plugin.clearVaultIndex(); + this.plugin.vaultVectorStore = void 0; + this.plugin.notifyChatViews(); + } + }) + ); + new import_obsidian4.Setting(containerEl).setName("Vault Index ChromaDB URL").setDesc( + "URL for your ChromaDB instance used for the vault index (default: http://localhost:8000)" + ).addText( + (text) => text.setValue(this.plugin.settings.vaultIndexConfig.chromaURL || "http://localhost:8000").onChange(async (value) => { + const trimmed = value.trim(); + this.plugin.settings.vaultIndexConfig.chromaURL = trimmed && trimmed.includes("://") ? trimmed : "http://localhost:8000"; + await this.plugin.saveSettings(); + }) + ); + new import_obsidian4.Setting(containerEl).setName("Vault Index Embedding Model").setDesc( + "Ollama model used to generate embeddings for vault notes (default: nomic-embed-text)" + ).addText( + (text) => text.setValue(this.plugin.settings.vaultIndexConfig.embeddingModel).onChange(async (value) => { + this.plugin.settings.vaultIndexConfig.embeddingModel = value; + await this.plugin.saveSettings(); + }) + ); + new import_obsidian4.Setting(containerEl).setName("Vault Index Similarity Threshold").setDesc( + "Minimum cosine similarity (0\u20131) for a vault search hit. Higher values require closer matches (default: 0.75)." + ).addText( + (text) => text.setValue(String(this.plugin.settings.vaultIndexConfig.similarityThreshold)).onChange(async (value) => { + const parsed = parseFloat(value); + if (!isNaN(parsed) && parsed >= 0 && parsed <= 1) { + this.plugin.settings.vaultIndexConfig.similarityThreshold = parsed; + await this.plugin.saveSettings(); + } else { + new import_obsidian4.Notice("Similarity threshold must be a number between 0 and 1."); + } + }) + ); + new import_obsidian4.Setting(containerEl).setName("Rebuild Vault Index").setDesc("Delete and rebuild the entire vault semantic index").addButton( + (button) => button.setButtonText("Rebuild Index").onClick(async () => { + try { + new import_obsidian4.Notice("Rebuilding vault index..."); + await this.plugin.rebuildVaultIndex(); + new import_obsidian4.Notice("Vault index rebuilt."); + } catch { + new import_obsidian4.Notice("Failed to rebuild vault index. Is ChromaDB running?"); + } + }) + ); + new import_obsidian4.Setting(containerEl).setName("Clear Vault Index").setDesc("Delete all indexed vault notes from ChromaDB").addButton( + (button) => button.setButtonText("Clear Index").onClick(async () => { + try { + await this.plugin.clearVaultIndex(); + new import_obsidian4.Notice("Vault index cleared."); + } catch { + new import_obsidian4.Notice("Failed to clear vault index. Is ChromaDB running?"); + } + }) + ); + containerEl.createEl("h3", { text: "Semantic Cache" }); new import_obsidian4.Setting(containerEl).setName("Enable Semantic Cache").setDesc("Use semantic cache to store and retrieve previous responses").addToggle( (toggle) => toggle.setValue(this.plugin.settings.cacheConfig.enabled).onChange(async (value) => { this.plugin.settings.cacheConfig.enabled = value; diff --git a/src/chat-view.ts b/src/chat-view.ts index 4bbff71..f06c159 100755 --- a/src/chat-view.ts +++ b/src/chat-view.ts @@ -1,6 +1,7 @@ import { ItemView, Notice, WorkspaceLeaf } from 'obsidian'; import { OllamaClient } from './ollama-client'; import { VaultIndexer } from './vault-indexer'; +import { VaultVectorStore } from './vault-vector-store'; import { ToolExecutor } from './tool-executor'; import { PluginSettings, OllamaMessage, OllamaTool, OllamaToolCall, ChatMessage } from './types'; import { ConversationStateManager } from './conversation-state'; @@ -22,7 +23,7 @@ export class ChatView extends ItemView { return this.newChatButtonClickHandler; } - constructor(leaf: WorkspaceLeaf, settings: PluginSettings) { + constructor(leaf: WorkspaceLeaf, settings: PluginSettings, vectorStore?: VaultVectorStore) { super(leaf); this.messages = []; this.lastMessageEl = null; @@ -44,7 +45,7 @@ export class ChatView extends ItemView { undefined, settings.cacheConfig ); - this.vaultIndexer = new VaultIndexer(this.app.vault); + this.vaultIndexer = new VaultIndexer(this.app.vault, undefined, vectorStore); this.toolExecutor = new ToolExecutor(this.app.vault, this.app); this.conversationStateManager = new ConversationStateManager(); } @@ -64,6 +65,11 @@ export class ChatView extends ItemView { }); } + setVectorStore(vectorStore: VaultVectorStore | undefined) { + this.vectorStore = vectorStore; + this.vaultIndexer.setVectorStore(vectorStore); + } + async clearCache(): Promise { await this.ollamaClient.clearCache(); } @@ -567,6 +573,7 @@ export class ChatView extends ItemView { private vaultIndexer: VaultIndexer; private toolExecutor: ToolExecutor; private conversationStateManager: ConversationStateManager; + private vectorStore?: VaultVectorStore; } const MAX_TOOL_CALLS = 5; diff --git a/src/constants.ts b/src/constants.ts index 132d55f..22f5fa0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -11,4 +11,11 @@ export const DEFAULT_SETTINGS = { embeddingModel: 'nomic-embed-text', chromaURL: 'http://localhost:8000', }, + vaultIndexConfig: { + enabled: false, + similarityThreshold: 0.75, + collectionName: 'ollama_vault_index', + embeddingModel: 'nomic-embed-text', + chromaURL: 'http://localhost:8000', + }, }; diff --git a/src/main.ts b/src/main.ts index d2d08cd..c038a15 100755 --- a/src/main.ts +++ b/src/main.ts @@ -1,20 +1,28 @@ -import { Plugin, WorkspaceLeaf, App, Setting, Notice, PluginSettingTab } from 'obsidian'; +import { Plugin, WorkspaceLeaf, App, Setting, Notice, PluginSettingTab, TFile } from 'obsidian'; import { ChatView } from './chat-view'; import { DEFAULT_SETTINGS } from './constants'; import { SemanticCacheService } from './semantic-cache'; +import { VaultVectorStore } from './vault-vector-store'; import { PluginSettings } from './types'; +import { Logger } from './utils'; export default class OllamaPlugin extends Plugin { settings: PluginSettings = DEFAULT_SETTINGS; semanticCache?: SemanticCacheService; + vaultVectorStore?: VaultVectorStore; async onload() { await this.loadSettings(); + // Initialize vault vector store if enabled + if (this.settings.vaultIndexConfig.enabled) { + await this.initializeVaultVectorStore(); + } + // Register the chat view this.registerView( 'ollama-chat-view', - (leaf: WorkspaceLeaf) => new ChatView(leaf, this.settings) + (leaf: WorkspaceLeaf) => new ChatView(leaf, this.settings, this.vaultVectorStore) ); // Add a ribbon icon in the left sidebar @@ -41,6 +49,27 @@ export default class OllamaPlugin extends Plugin { }, }); + // Add a command to clear the vault index + this.addCommand({ + id: 'clear-vault-index', + name: 'Clear Vault Index', + callback: async () => { + await this.clearVaultIndex(); + new Notice('Vault index cleared.'); + }, + }); + + // Add a command to rebuild the vault index + this.addCommand({ + id: 'rebuild-vault-index', + name: 'Rebuild Vault Index', + callback: async () => { + new Notice('Rebuilding vault index...'); + await this.rebuildVaultIndex(); + new Notice('Vault index rebuilt.'); + }, + }); + // Add a settings tab this.addSettingTab(new OllamaSettingTab(this.app, this)); @@ -56,6 +85,9 @@ export default class OllamaPlugin extends Plugin { new Notice('Semantic cache initialization failed. Check console for details.'); } } + + // Set up vault event listeners for incremental indexing + this.registerVaultEventListeners(); } // eslint-disable-next-line @typescript-eslint/no-misused-promises @@ -77,6 +109,105 @@ export default class OllamaPlugin extends Plugin { await this.saveData(this.settings); } + async initializeVaultVectorStore(): Promise { + this.vaultVectorStore = new VaultVectorStore( + this.settings.ollamaUrl, + this.settings.vaultIndexConfig + ); + try { + await this.vaultVectorStore.initialize(); + // Run background indexing + void this.backgroundIndexVault().catch((err) => { + Logger.warn(`Background vault indexing failed: ${String(err)}`, 'main'); + }); + } catch { + new Notice('Vault vector store initialization failed. Check console for details.'); + } + } + + async backgroundIndexVault(): Promise { + if (!this.vaultVectorStore) return; + + const files = this.app.vault.getMarkdownFiles(); + Logger.info(`Starting background vault indexing for ${files.length} files...`, 'main'); + + let indexed = 0; + for (const file of files) { + try { + const content = await this.app.vault.read(file); + await this.vaultVectorStore.indexFile(file, content); + indexed++; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, 'main'); + } + } + + this.settings.lastIndexTime = Date.now(); + await this.saveSettings(); + Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, 'main'); + new Notice(`Vault index updated: ${indexed} files indexed.`); + } + + async rebuildVaultIndex(): Promise { + if (this.vaultVectorStore) { + await this.vaultVectorStore.clearIndex(); + await this.vaultVectorStore.initialize(); + } + await this.backgroundIndexVault(); + } + + async clearVaultIndex(): Promise { + if (this.vaultVectorStore) { + await this.vaultVectorStore.clearIndex(); + } + } + + registerVaultEventListeners(): void { + // Listen for file creation + this.registerEvent( + this.app.vault.on('create', (file) => { + if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) { + void this.app.vault.read(file).then((content) => { + void this.vaultVectorStore?.indexFile(file, content); + }); + } + }) + ); + + // Listen for file modification + this.registerEvent( + this.app.vault.on('modify', (file) => { + if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) { + void this.app.vault.read(file).then((content) => { + void this.vaultVectorStore?.indexFile(file, content); + }); + } + }) + ); + + // Listen for file deletion + this.registerEvent( + this.app.vault.on('delete', (file) => { + if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) { + void this.vaultVectorStore.deleteFile(file.path); + } + }) + ); + + // Listen for file renames + this.registerEvent( + this.app.vault.on('rename', (file, oldPath) => { + if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) { + void this.vaultVectorStore.deleteFile(oldPath); + void this.app.vault.read(file).then((content) => { + void this.vaultVectorStore?.indexFile(file, content); + }); + } + }) + ); + } + async activateChatView() { const existing = this.app.workspace.getLeavesOfType('ollama-chat-view'); if (existing.length > 0) { @@ -103,6 +234,7 @@ export default class OllamaPlugin extends Plugin { leaves.forEach((leaf) => { if (leaf.view instanceof ChatView) { leaf.view.updateSettings(this.settings); + leaf.view.setVectorStore(this.vaultVectorStore); } }); } @@ -171,6 +303,112 @@ class OllamaSettingTab extends PluginSettingTab { }) ); + // Vault Index Settings + containerEl.createEl('h3', { text: 'Vault Semantic Index' }); + + new Setting(containerEl) + .setName('Enable Vault Semantic Index') + .setDesc( + 'Automatically index vault notes into a vector database for semantic/RAG search. Requires ChromaDB and an embedding model.' + ) + .addToggle((toggle) => + toggle.setValue(this.plugin.settings.vaultIndexConfig.enabled).onChange(async (value) => { + this.plugin.settings.vaultIndexConfig.enabled = value; + await this.plugin.saveSettings(); + this.plugin.notifyChatViews(); + if (value) { + new Notice('Vault semantic index enabled. Rebuilding index...'); + await this.plugin.initializeVaultVectorStore(); + await this.plugin.rebuildVaultIndex(); + } else { + await this.plugin.clearVaultIndex(); + this.plugin.vaultVectorStore = undefined; + this.plugin.notifyChatViews(); + } + }) + ); + + new Setting(containerEl) + .setName('Vault Index ChromaDB URL') + .setDesc( + 'URL for your ChromaDB instance used for the vault index (default: http://localhost:8000)' + ) + .addText((text) => + text + .setValue(this.plugin.settings.vaultIndexConfig.chromaURL || 'http://localhost:8000') + .onChange(async (value) => { + const trimmed = value.trim(); + this.plugin.settings.vaultIndexConfig.chromaURL = + trimmed && trimmed.includes('://') ? trimmed : 'http://localhost:8000'; + await this.plugin.saveSettings(); + }) + ); + + new Setting(containerEl) + .setName('Vault Index Embedding Model') + .setDesc( + 'Ollama model used to generate embeddings for vault notes (default: nomic-embed-text)' + ) + .addText((text) => + text + .setValue(this.plugin.settings.vaultIndexConfig.embeddingModel) + .onChange(async (value) => { + this.plugin.settings.vaultIndexConfig.embeddingModel = value; + await this.plugin.saveSettings(); + }) + ); + + new Setting(containerEl) + .setName('Vault Index Similarity Threshold') + .setDesc( + 'Minimum cosine similarity (0–1) for a vault search hit. Higher values require closer matches (default: 0.75).' + ) + .addText((text) => + text + .setValue(String(this.plugin.settings.vaultIndexConfig.similarityThreshold)) + .onChange(async (value) => { + const parsed = parseFloat(value); + if (!isNaN(parsed) && parsed >= 0 && parsed <= 1) { + this.plugin.settings.vaultIndexConfig.similarityThreshold = parsed; + await this.plugin.saveSettings(); + } else { + new Notice('Similarity threshold must be a number between 0 and 1.'); + } + }) + ); + + new Setting(containerEl) + .setName('Rebuild Vault Index') + .setDesc('Delete and rebuild the entire vault semantic index') + .addButton((button) => + button.setButtonText('Rebuild Index').onClick(async () => { + try { + new Notice('Rebuilding vault index...'); + await this.plugin.rebuildVaultIndex(); + new Notice('Vault index rebuilt.'); + } catch { + new Notice('Failed to rebuild vault index. Is ChromaDB running?'); + } + }) + ); + + new Setting(containerEl) + .setName('Clear Vault Index') + .setDesc('Delete all indexed vault notes from ChromaDB') + .addButton((button) => + button.setButtonText('Clear Index').onClick(async () => { + try { + await this.plugin.clearVaultIndex(); + new Notice('Vault index cleared.'); + } catch { + new Notice('Failed to clear vault index. Is ChromaDB running?'); + } + }) + ); + + // Semantic Cache Settings + containerEl.createEl('h3', { text: 'Semantic Cache' }); + new Setting(containerEl) .setName('Enable Semantic Cache') .setDesc('Use semantic cache to store and retrieve previous responses') diff --git a/src/types.ts b/src/types.ts index db620ff..33f7abc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -133,6 +133,7 @@ export interface VaultIndexEntry { title: string; content: string; score: number; + tags?: string; } export interface ChatMessage { @@ -176,6 +177,14 @@ export interface CacheConfig { chromaURL?: string; } +export interface VaultIndexConfig { + enabled: boolean; + collectionName: string; + embeddingModel: string; + chromaURL?: string; + similarityThreshold: number; +} + export interface PluginSettings { ollamaUrl: string; model: string; @@ -183,6 +192,7 @@ export interface PluginSettings { maxMessageHistory: number; lastIndexTime: number; cacheConfig: CacheConfig; + vaultIndexConfig: VaultIndexConfig; } // ============================================================ diff --git a/src/vault-indexer.ts b/src/vault-indexer.ts index 4771fab..fa77218 100644 --- a/src/vault-indexer.ts +++ b/src/vault-indexer.ts @@ -3,6 +3,8 @@ import { Vault, TFile } from 'obsidian'; import { Logger } from './utils'; import { Cache } from './cache'; +import { VaultVectorStore } from './vault-vector-store'; +import { VaultIndexEntry } from './types'; interface ParsedFrontmatter { title?: string; @@ -43,12 +45,63 @@ export class InMemoryCache implements Cache { } const STOP_WORDS = new Set([ - 'a', 'an', 'the', 'is', 'it', 'in', 'on', 'at', 'to', 'for', 'of', - 'and', 'or', 'but', 'with', 'by', 'from', 'up', 'about', 'into', - 'this', 'that', 'these', 'those', 'be', 'been', 'being', 'have', - 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', - 'may', 'might', 'can', 'are', 'was', 'were', 'as', 'so', 'if', 'not', - 'no', 'my', 'your', 'our', 'its', 'we', 'you', 'he', 'she', 'they', + 'a', + 'an', + 'the', + 'is', + 'it', + 'in', + 'on', + 'at', + 'to', + 'for', + 'of', + 'and', + 'or', + 'but', + 'with', + 'by', + 'from', + 'up', + 'about', + 'into', + 'this', + 'that', + 'these', + 'those', + 'be', + 'been', + 'being', + 'have', + 'has', + 'had', + 'do', + 'does', + 'did', + 'will', + 'would', + 'could', + 'should', + 'may', + 'might', + 'can', + 'are', + 'was', + 'were', + 'as', + 'so', + 'if', + 'not', + 'no', + 'my', + 'your', + 'our', + 'its', + 'we', + 'you', + 'he', + 'she', + 'they', ]); const CONTENT_PREVIEW_LENGTH = 500; @@ -56,6 +109,7 @@ const CONTENT_PREVIEW_LENGTH = 500; export class VaultIndexer { private vault: Vault; private cache?: Cache; + private vectorStore?: VaultVectorStore; private readonly SCORING_WEIGHTS = { TITLE: 5, FRONTMATTER_TITLE: 4, @@ -64,9 +118,14 @@ export class VaultIndexer { CONTENT: 1, }; - constructor(vault: Vault, cache?: Cache) { + constructor(vault: Vault, cache?: Cache, vectorStore?: VaultVectorStore) { this.vault = vault; this.cache = cache; + this.vectorStore = vectorStore; + } + + setVectorStore(vectorStore: VaultVectorStore | undefined): void { + this.vectorStore = vectorStore; } tokenize(text: string): string[] { @@ -95,10 +154,7 @@ export class VaultIndexer { }; } - calculateWeightedScore( - tokenized: TokenizedContent, - queryTokens: string[] - ): { score: number } { + calculateWeightedScore(tokenized: TokenizedContent, queryTokens: string[]): { score: number } { let score = 0; for (const token of queryTokens) { if (tokenized.frontmatter?.title && this.exactMatch(tokenized.frontmatter.title, token)) { @@ -150,11 +206,27 @@ export class VaultIndexer { return entries; } - async searchVault(query: string, limit = 3): Promise { + async searchVault(query: string, limit = 3): Promise { if (!query || !query.trim()) { return []; } + // Try semantic search first if vector store is available + if (this.vectorStore) { + try { + const semanticResults = await this.vectorStore.search(query, limit); + if (semanticResults.length > 0) { + return semanticResults; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn( + `Semantic search failed, falling back to keyword search: ${errorMessage}`, + 'vault-indexer' + ); + } + } + const cacheKey = `query:${query.trim()}:limit:${limit}`; if (this.cache) { let cachedResults: string | null = null; @@ -166,7 +238,7 @@ export class VaultIndexer { if (cachedResults) { try { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const parsedResults: VaultEntry[] = JSON.parse(cachedResults); + const parsedResults: VaultIndexEntry[] = JSON.parse(cachedResults); return parsedResults.slice(0, limit); } catch { // ignore parse errors @@ -198,7 +270,13 @@ export class VaultIndexer { .filter((e) => e.score > 0); scored.sort((a, b) => b.score - a.score); - const results = scored.slice(0, limit); + const results: VaultIndexEntry[] = scored.slice(0, limit).map((e) => ({ + path: e.file.path, + title: e.title, + content: e.content, + score: e.score, + tags: e.frontmatter?.tags, + })); if (this.cache) { try { diff --git a/src/vault-vector-store.ts b/src/vault-vector-store.ts new file mode 100644 index 0000000..76de1ce --- /dev/null +++ b/src/vault-vector-store.ts @@ -0,0 +1,242 @@ +// src/vault-vector-store.ts + +import { ChromaClient, Collection } from 'chromadb'; +import { TFile } from 'obsidian'; +import { VaultIndexConfig, VaultIndexEntry } from './types'; +import { Logger } from './utils'; +import { ContentVectorizer } from './indexing-pipeline/vectorization'; +import { ContentExtractor } from './indexing-pipeline/extraction'; +import { ContentNormalizer } from './indexing-pipeline/normalization'; + +export class VaultVectorStore { + private client: ChromaClient | null = null; + private collection: Collection | null = null; + private config: VaultIndexConfig; + private ollamaURL: string; + private vectorizer: ContentVectorizer; + private extractor: ContentExtractor; + private normalizer: ContentNormalizer; + private isInitialized = false; + + constructor(ollamaURL: string, config: VaultIndexConfig) { + this.ollamaURL = ollamaURL.replace(/\/+$/, ''); + this.config = config; + this.vectorizer = new ContentVectorizer({ + model: config.embeddingModel, + ollamaUrl: this.ollamaURL, + }); + this.extractor = new ContentExtractor(); + this.normalizer = new ContentNormalizer(); + } + + async initialize(): Promise { + if (!this.config.enabled || this.isInitialized) return; + + try { + const rawURL = this.config.chromaURL?.trim() || 'http://localhost:8000'; + const chromaURL = rawURL.includes('://') ? rawURL : 'http://localhost:8000'; + this.client = new ChromaClient({ path: chromaURL }); + this.collection = await this.client.getOrCreateCollection({ + name: this.config.collectionName, + metadata: { 'hnsw:space': 'cosine' }, + }); + this.isInitialized = true; + Logger.info( + `Vault vector store initialized: ${this.config.collectionName}`, + 'vault-vector-store' + ); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.error( + `Failed to initialize vault vector store: ${errorMessage}`, + 'vault-vector-store' + ); + throw error; + } + } + + /** + * Index a single vault file by generating an embedding and storing it in ChromaDB. + */ + async indexFile(file: TFile, content: string): Promise { + if (!this.collection || !this.config.enabled) return; + if (!content.trim()) { + // Remove empty files from index if they exist + await this.deleteFile(file.path); + return; + } + + try { + const extracted = this.extractor.extractFromFile( + { basename: file.basename, path: file.path }, + content + ); + const normalized = this.normalizer.normalize(extracted); + + const chunk = { + id: file.path, + path: file.path, + title: normalized.title, + content: normalized.content, + tokens: normalized.tokens, + headings: normalized.headings, + frontmatter: normalized.frontmatter, + firstParagraph: normalized.firstParagraph, + wordCount: normalized.wordCount, + chunkIndex: 0, + chunkSize: 1, + }; + + const embedding = await this.vectorizer.vectorize(chunk); + if (embedding.length === 0) { + Logger.warn(`Empty embedding for ${file.path}, skipping index`, 'vault-vector-store'); + return; + } + + // Upsert by path so re-indexing updates rather than duplicates + await this.collection.upsert({ + ids: [file.path], + documents: [this.createDocumentText(normalized)], + embeddings: [embedding], + metadatas: [ + { + path: file.path, + title: normalized.title, + tags: + typeof normalized.frontmatter.tags === 'string' ? normalized.frontmatter.tags : '', + }, + ], + }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn(`Failed to index file ${file.path}: ${errorMessage}`, 'vault-vector-store'); + } + } + + /** + * Remove a file from the vector index. + */ + async deleteFile(path: string): Promise { + if (!this.collection || !this.config.enabled) return; + + try { + await this.collection.delete({ ids: [path] }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn( + `Failed to delete file ${path} from index: ${errorMessage}`, + 'vault-vector-store' + ); + } + } + + /** + * Perform semantic search against the vault index. + */ + async search(query: string, limit = 3): Promise { + if (!this.collection || !this.config.enabled) { + return []; + } + + try { + const queryEmbedding = await this.vectorizer.vectorize({ + id: 'query', + path: 'query', + title: query, + content: query, + tokens: query + .toLowerCase() + .split(/\s+/) + .filter((t) => t.length > 1), + headings: [query], + frontmatter: {}, + firstParagraph: query, + wordCount: query.split(/\s+/).length, + chunkIndex: 0, + chunkSize: 1, + }); + + if (queryEmbedding.length === 0) { + Logger.warn('Empty query embedding, skipping semantic search', 'vault-vector-store'); + return []; + } + + const results = await this.collection.query({ + queryEmbeddings: [queryEmbedding], + nResults: limit, + }); + + if (!results.ids[0] || results.ids[0].length === 0) { + return []; + } + + const entries: VaultIndexEntry[] = []; + for (let i = 0; i < results.ids[0].length; i++) { + const distance = results.distances?.[0]?.[i] ?? 0; + // Cosine distance to score: closer to 1 is better + const score = 1 - distance; + if (score < this.config.similarityThreshold) { + continue; + } + + const metadata = results.metadatas?.[0]?.[i] as Record | undefined; + const document = results.documents?.[0]?.[i] as string | undefined; + + entries.push({ + path: metadata?.path || String(results.ids[0][i]), + title: metadata?.title || 'Untitled', + content: document || '', + score, + tags: metadata?.tags, + }); + } + + // Sort by score descending + entries.sort((a, b) => b.score - a.score); + return entries.slice(0, limit); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn(`Semantic search failed: ${errorMessage}`, 'vault-vector-store'); + return []; + } + } + + /** + * Clear the entire vault index. + */ + async clearIndex(): Promise { + if (!this.client || !this.config.enabled) return; + + try { + await this.client.deleteCollection({ name: this.config.collectionName }); + this.collection = null; + this.isInitialized = false; + Logger.info('Vault index cleared', 'vault-vector-store'); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.error(`Failed to clear vault index: ${errorMessage}`, 'vault-vector-store'); + } + } + + /** + * Get the number of indexed documents. + */ + async getIndexedCount(): Promise { + if (!this.collection || !this.config.enabled) return 0; + try { + const result = await this.collection.count(); + return result; + } catch { + return 0; + } + } + + private createDocumentText(normalized: ReturnType): string { + const parts: string[] = []; + if (normalized.title) parts.push(`Title: ${normalized.title}`); + if (normalized.firstParagraph) parts.push(normalized.firstParagraph); + if (normalized.headings.length > 0) parts.push(normalized.headings.join('\n')); + parts.push(normalized.content.substring(0, 1000)); + return parts.join('\n\n'); + } +} diff --git a/src/workflow-engine/workflow-engine.ts b/src/workflow-engine/workflow-engine.ts index b06b953..836fe8f 100644 --- a/src/workflow-engine/workflow-engine.ts +++ b/src/workflow-engine/workflow-engine.ts @@ -422,17 +422,17 @@ Rules: // Apply tag filter if specified const filtered = config.tagFilter ? entries.filter((entry) => { - const tags = entry.frontmatter?.tags ?? ''; + const tags = entry.tags ?? ''; return tags.toLowerCase().includes(config.tagFilter!.toLowerCase()); }) : entries; return filtered.map((entry) => ({ - path: entry.file.path, + path: entry.path, title: entry.title, content: entry.content, score: entry.score, - tags: entry.frontmatter?.tags, + tags: entry.tags, })); } diff --git a/tests/chat-view.test.ts b/tests/chat-view.test.ts index fbe670c..0b75da0 100755 --- a/tests/chat-view.test.ts +++ b/tests/chat-view.test.ts @@ -39,6 +39,13 @@ const mockSettings: PluginSettings = { embeddingModel: 'nomic-embed-text', chromaURL: 'http://localhost:8000', }, + vaultIndexConfig: { + enabled: false, + similarityThreshold: 0.75, + collectionName: 'test-vault-index', + embeddingModel: 'nomic-embed-text', + chromaURL: 'http://localhost:8000', + }, }; describe('ChatView', () => { diff --git a/tests/vault-vector-store.test.ts b/tests/vault-vector-store.test.ts new file mode 100644 index 0000000..9160d45 --- /dev/null +++ b/tests/vault-vector-store.test.ts @@ -0,0 +1,222 @@ +import { ChromaClient } from 'chromadb'; +import { VaultIndexConfig } from '../src/types'; + +// Mock ChromaDB module +jest.mock('chromadb', () => ({ + ChromaClient: jest.fn().mockImplementation(() => { + return { + getOrCreateCollection: jest.fn().mockResolvedValue({ + query: jest.fn(), + upsert: jest.fn(), + delete: jest.fn(), + count: jest.fn().mockResolvedValue(5), + }), + deleteCollection: jest.fn(), + }; + }), +})); + +import { VaultVectorStore } from '../src/vault-vector-store'; + +describe('VaultVectorStore', () => { + const mockOllamaUrl = 'http://localhost:11434'; + const mockConfig: VaultIndexConfig = { + enabled: true, + similarityThreshold: 0.75, + collectionName: 'test_vault_index', + embeddingModel: 'nomic-embed-text', + chromaURL: 'http://localhost:8000', + }; + + let store: VaultVectorStore; + let mockChromaClient: any; + let mockCollection: any; + + beforeEach(async () => { + jest.clearAllMocks(); + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ embedding: [0.1, 0.2, 0.3] }), + }); + + store = new VaultVectorStore(mockOllamaUrl, mockConfig); + await store.initialize(); + + mockChromaClient = (ChromaClient as jest.Mock).mock.results[0].value; + mockCollection = await mockChromaClient.getOrCreateCollection.mock.results[0].value; + }); + + describe('constructor', () => { + it('should initialize with provided config', () => { + expect(store).toBeInstanceOf(VaultVectorStore); + }); + }); + + describe('initialize', () => { + it('should initialize the collection', async () => { + expect(mockChromaClient.getOrCreateCollection).toHaveBeenCalledWith({ + name: mockConfig.collectionName, + metadata: { 'hnsw:space': 'cosine' }, + }); + }); + + it('should not initialize when disabled', async () => { + jest.clearAllMocks(); + const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false }; + const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig); + await disabledStore.initialize(); + expect(mockChromaClient.getOrCreateCollection).not.toHaveBeenCalled(); + }); + }); + + describe('indexFile', () => { + it('should upsert a file into the collection', async () => { + const mockFile = { + basename: 'test.md', + path: 'test.md', + extension: 'md', + } as any; + + await store.indexFile(mockFile, '# Test\n\nThis is test content.'); + + expect(mockCollection.upsert).toHaveBeenCalled(); + const upsertCall = mockCollection.upsert.mock.calls[0][0]; + expect(upsertCall.ids).toContain('test.md'); + expect(upsertCall.metadatas[0].title).toBe('test'); + }); + + it('should delete file from index when content is empty', async () => { + const mockFile = { + basename: 'empty.md', + path: 'empty.md', + extension: 'md', + } as any; + + await store.indexFile(mockFile, ' '); + expect(mockCollection.delete).toHaveBeenCalledWith({ ids: ['empty.md'] }); + }); + + it('should not index when collection is null', async () => { + jest.clearAllMocks(); + const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false }; + const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig); + await disabledStore.initialize(); + + const mockFile = { basename: 'test.md', path: 'test.md' } as any; + await disabledStore.indexFile(mockFile, 'content'); + expect(mockCollection.upsert).not.toHaveBeenCalled(); + }); + }); + + describe('deleteFile', () => { + it('should delete a file from the collection', async () => { + await store.deleteFile('test.md'); + expect(mockCollection.delete).toHaveBeenCalledWith({ ids: ['test.md'] }); + }); + }); + + describe('search', () => { + it('should return empty array when disabled', async () => { + jest.clearAllMocks(); + const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false }; + const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig); + await disabledStore.initialize(); + + const results = await disabledStore.search('test query', 3); + expect(results).toEqual([]); + expect(mockCollection.query).not.toHaveBeenCalled(); + }); + + it('should return semantic search results', async () => { + mockCollection.query.mockResolvedValue({ + ids: [['file1.md', 'file2.md']], + documents: [['Doc 1 content', 'Doc 2 content']], + distances: [[0.1, 0.15]], + metadatas: [ + [ + { path: 'file1.md', title: 'File 1' }, + { path: 'file2.md', title: 'File 2' }, + ], + ], + }); + + const results = await store.search('test query', 2); + + expect(results).toHaveLength(2); + expect(results[0].path).toBe('file1.md'); + expect(results[0].title).toBe('File 1'); + expect(results[0].score).toBe(0.9); // 1 - 0.1 + expect(results[1].score).toBe(0.85); // 1 - 0.15 + }); + + it('should filter results below similarity threshold', async () => { + mockCollection.query.mockResolvedValue({ + ids: [['file1.md', 'file2.md']], + documents: [['Doc 1', 'Doc 2']], + distances: [[0.1, 0.5]], // scores: 0.9 and 0.5; threshold is 0.75 + metadatas: [ + [ + { path: 'file1.md', title: 'File 1' }, + { path: 'file2.md', title: 'File 2' }, + ], + ], + }); + + const results = await store.search('test', 2); + expect(results).toHaveLength(1); + expect(results[0].path).toBe('file1.md'); + }); + + it('should return empty array when no results', async () => { + mockCollection.query.mockResolvedValue({ + ids: [[]], + documents: [[]], + distances: [[]], + metadatas: [[]], + }); + + const results = await store.search('test', 3); + expect(results).toEqual([]); + }); + + it('should return empty array on query failure', async () => { + mockCollection.query.mockRejectedValue(new Error('Query failed')); + + const results = await store.search('test', 3); + expect(results).toEqual([]); + }); + }); + + describe('clearIndex', () => { + it('should delete the collection', async () => { + await store.clearIndex(); + expect(mockChromaClient.deleteCollection).toHaveBeenCalledWith({ + name: mockConfig.collectionName, + }); + }); + + it('should not clear when disabled', async () => { + jest.clearAllMocks(); + const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false }; + const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig); + await disabledStore.clearIndex(); + expect(mockChromaClient.deleteCollection).not.toHaveBeenCalled(); + }); + }); + + describe('getIndexedCount', () => { + it('should return the collection count', async () => { + const count = await store.getIndexedCount(); + expect(count).toBe(5); + expect(mockCollection.count).toHaveBeenCalled(); + }); + + it('should return 0 when disabled', async () => { + jest.clearAllMocks(); + const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false }; + const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig); + const count = await disabledStore.getIndexedCount(); + expect(count).toBe(0); + }); + }); +});