feat: add semantic/RAG vault indexing with automatic background sync
- Add VaultVectorStore backed by ChromaDB for vector-based vault search - Integrate existing ContentVectorizer/IndexingPipeline for embeddings - Update VaultIndexer to prefer semantic search with keyword fallback - Background indexing on plugin load + incremental sync via vault events - Add vault index settings, commands, and UI controls - Add tests for VaultVectorStore - Update README with RAG setup instructions
This commit is contained in:
@@ -6,6 +6,7 @@ A plugin that integrates [Ollama](https://ollama.ai) with Obsidian, allowing you
|
||||
|
||||
- Chat with Ollama models directly in Obsidian
|
||||
- Vault context search — the assistant can reference your notes
|
||||
- **Semantic/RAG vault indexing** — automatically index your vault into a vector database for intelligent retrieval
|
||||
- Tool integration — create files based on chat responses
|
||||
- Streaming responses
|
||||
- Semantic response cache — repeated or similar queries are answered instantly without hitting the model (requires ChromaDB)
|
||||
@@ -17,6 +18,24 @@ A plugin that integrates [Ollama](https://ollama.ai) with Obsidian, allowing you
|
||||
2. **Start Ollama**: `ollama serve`
|
||||
3. **Pull a chat model**: `ollama pull llama3` (or any other model you prefer)
|
||||
|
||||
### Optional — Vault Semantic Index (RAG)
|
||||
|
||||
The vault semantic index automatically indexes your Obsidian notes into a local [ChromaDB](https://www.trychroma.com) vector database. When you ask a question, the plugin performs semantic search against your notes and includes the most relevant passages as context for the AI.
|
||||
|
||||
1. **Install ChromaDB**:
|
||||
```bash
|
||||
pip install chromadb
|
||||
```
|
||||
2. **Start ChromaDB**:
|
||||
```bash
|
||||
chroma run --host localhost --port 8000
|
||||
```
|
||||
3. **Pull an embedding model**:
|
||||
```bash
|
||||
ollama pull nomic-embed-text
|
||||
```
|
||||
4. Enable the vault semantic index in the plugin settings and configure the ChromaDB URL.
|
||||
|
||||
### Optional — Semantic Cache
|
||||
|
||||
The semantic cache stores responses in a local [ChromaDB](https://www.trychroma.com) vector database. When you ask a question that is semantically similar to one already cached, the stored answer is returned immediately instead of calling the model.
|
||||
@@ -89,6 +108,12 @@ Open **Settings → Ollama Settings** to configure the plugin.
|
||||
| Model | `llama3` | Model used for chat responses |
|
||||
| Vault Search Limit | `3` | Maximum number of vault entries to include in context |
|
||||
| Max Message History | `50` | Maximum number of messages kept in conversation history |
|
||||
| **Enable Vault Semantic Index** | Off | Index vault notes into a vector DB for semantic/RAG search |
|
||||
| Vault Index ChromaDB URL | `http://localhost:8000` | URL of your ChromaDB instance for the vault index |
|
||||
| Vault Index Embedding Model | `nomic-embed-text` | Ollama model used to generate vault embeddings |
|
||||
| Vault Index Similarity Threshold | `0.75` | Minimum cosine similarity (0–1) for a vault search hit |
|
||||
| Rebuild Vault Index | — | Button to rebuild the entire vault semantic index |
|
||||
| Clear Vault Index | — | Button to delete all indexed vault notes |
|
||||
| Enable Semantic Cache | Off | Cache responses for fast repeated queries |
|
||||
| ChromaDB URL | `http://localhost:8000` | URL of your running ChromaDB instance |
|
||||
| Cache Embedding Model | `nomic-embed-text` | Ollama model used to generate cache embeddings |
|
||||
@@ -112,7 +137,7 @@ Open **Settings → Ollama Settings** to configure the plugin.
|
||||
|
||||
## Vault Context
|
||||
|
||||
When you send a message, the plugin automatically searches your vault for relevant notes and includes them as context. The search uses weighted scoring:
|
||||
When you send a message, the plugin searches your vault for relevant notes and includes them as context. If the **Vault Semantic Index** is enabled, search is performed via semantic/RAG retrieval using vector embeddings. Otherwise, it falls back to a weighted keyword search:
|
||||
|
||||
- **Headings** — 5x weight
|
||||
- **Frontmatter title** — 3x weight
|
||||
@@ -120,6 +145,8 @@ When you send a message, the plugin automatically searches your vault for releva
|
||||
- **First paragraph** — 1.5x weight
|
||||
- **General content** — 1x weight
|
||||
|
||||
The plugin automatically watches your vault for changes (create, modify, delete, rename) and updates the semantic index in real time when enabled.
|
||||
|
||||
## Tools
|
||||
|
||||
The plugin exposes a `create_file` tool that allows the AI to create new markdown files in your vault. Paths are validated for safety (no `.obsidian`/`.git` access, no path traversal).
|
||||
|
||||
@@ -8382,7 +8382,7 @@ var STOP_WORDS = /* @__PURE__ */ new Set([
|
||||
]);
|
||||
var CONTENT_PREVIEW_LENGTH = 500;
|
||||
var VaultIndexer = class {
|
||||
constructor(vault, cache) {
|
||||
constructor(vault, cache, vectorStore) {
|
||||
this.SCORING_WEIGHTS = {
|
||||
TITLE: 5,
|
||||
FRONTMATTER_TITLE: 4,
|
||||
@@ -8392,6 +8392,10 @@ var VaultIndexer = class {
|
||||
};
|
||||
this.vault = vault;
|
||||
this.cache = cache;
|
||||
this.vectorStore = vectorStore;
|
||||
}
|
||||
setVectorStore(vectorStore) {
|
||||
this.vectorStore = vectorStore;
|
||||
}
|
||||
tokenize(text) {
|
||||
return text.toLowerCase().replace(/[^\w\s]/g, "").split(/\s+/).filter((token) => token.length > 1 && !STOP_WORDS.has(token));
|
||||
@@ -8461,6 +8465,20 @@ var VaultIndexer = class {
|
||||
if (!query || !query.trim()) {
|
||||
return [];
|
||||
}
|
||||
if (this.vectorStore) {
|
||||
try {
|
||||
const semanticResults = await this.vectorStore.search(query, limit);
|
||||
if (semanticResults.length > 0) {
|
||||
return semanticResults;
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(
|
||||
`Semantic search failed, falling back to keyword search: ${errorMessage}`,
|
||||
"vault-indexer"
|
||||
);
|
||||
}
|
||||
}
|
||||
const cacheKey = `query:${query.trim()}:limit:${limit}`;
|
||||
if (this.cache) {
|
||||
let cachedResults = null;
|
||||
@@ -8497,7 +8515,13 @@ var VaultIndexer = class {
|
||||
return { ...entry, score };
|
||||
}).filter((e) => e.score > 0);
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
const results = scored.slice(0, limit);
|
||||
const results = scored.slice(0, limit).map((e) => ({
|
||||
path: e.file.path,
|
||||
title: e.title,
|
||||
content: e.content,
|
||||
score: e.score,
|
||||
tags: e.frontmatter?.tags
|
||||
}));
|
||||
if (this.cache) {
|
||||
try {
|
||||
await this.cache.put(cacheKey, JSON.stringify(results));
|
||||
@@ -8933,7 +8957,7 @@ var ErrorHandler = class {
|
||||
|
||||
// src/chat-view.ts
|
||||
var ChatView = class extends import_obsidian3.ItemView {
|
||||
constructor(leaf, settings) {
|
||||
constructor(leaf, settings, vectorStore) {
|
||||
super(leaf);
|
||||
// State
|
||||
this.messages = [];
|
||||
@@ -8969,7 +8993,7 @@ var ChatView = class extends import_obsidian3.ItemView {
|
||||
void 0,
|
||||
settings.cacheConfig
|
||||
);
|
||||
this.vaultIndexer = new VaultIndexer(this.app.vault);
|
||||
this.vaultIndexer = new VaultIndexer(this.app.vault, void 0, vectorStore);
|
||||
this.toolExecutor = new ToolExecutor(this.app.vault, this.app);
|
||||
this.conversationStateManager = new ConversationStateManager();
|
||||
}
|
||||
@@ -8997,6 +9021,10 @@ var ChatView = class extends import_obsidian3.ItemView {
|
||||
);
|
||||
});
|
||||
}
|
||||
setVectorStore(vectorStore) {
|
||||
this.vectorStore = vectorStore;
|
||||
this.vaultIndexer.setVectorStore(vectorStore);
|
||||
}
|
||||
async clearCache() {
|
||||
await this.ollamaClient.clearCache();
|
||||
}
|
||||
@@ -9402,6 +9430,465 @@ var DEFAULT_SETTINGS = {
|
||||
collectionName: "ollama_semantic_cache",
|
||||
embeddingModel: "nomic-embed-text",
|
||||
chromaURL: "http://localhost:8000"
|
||||
},
|
||||
vaultIndexConfig: {
|
||||
enabled: false,
|
||||
similarityThreshold: 0.75,
|
||||
collectionName: "ollama_vault_index",
|
||||
embeddingModel: "nomic-embed-text",
|
||||
chromaURL: "http://localhost:8000"
|
||||
}
|
||||
};
|
||||
|
||||
// src/indexing-pipeline/vectorization.ts
|
||||
var ContentVectorizer = class {
|
||||
constructor(config, fetchFn) {
|
||||
this.model = config.model;
|
||||
this.ollamaUrl = config.ollamaUrl;
|
||||
this.fetchFn = fetchFn ?? ((url, init) => fetch(url, init));
|
||||
}
|
||||
/**
|
||||
* Generates embeddings for a content chunk
|
||||
*/
|
||||
async vectorize(chunk) {
|
||||
try {
|
||||
const prompt = this.createPrompt(chunk);
|
||||
const response = await this.fetchFn(`${this.ollamaUrl}/api/embeddings`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: this.model,
|
||||
prompt
|
||||
})
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Embedding failed with status ${response.status}`);
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!this.isEmbeddingResponse(data)) {
|
||||
throw new Error("Invalid embedding response");
|
||||
}
|
||||
return data.embedding;
|
||||
} catch (error) {
|
||||
Logger.warn(`Failed to generate embedding: ${String(error)}`, "indexing-pipeline");
|
||||
return [];
|
||||
}
|
||||
}
|
||||
isEmbeddingResponse(data) {
|
||||
return typeof data === "object" && data !== null && Array.isArray(data.embedding) && data.embedding.every((value) => typeof value === "number");
|
||||
}
|
||||
/**
|
||||
* Creates a prompt from content chunk for embedding
|
||||
*/
|
||||
createPrompt(chunk) {
|
||||
const parts = [
|
||||
chunk.title,
|
||||
chunk.firstParagraph,
|
||||
chunk.content.substring(0, 1e3),
|
||||
// Limit content to avoid long prompts
|
||||
chunk.headings.join(" "),
|
||||
JSON.stringify(chunk.frontmatter)
|
||||
].filter(Boolean);
|
||||
return parts.join("\n\n");
|
||||
}
|
||||
};
|
||||
|
||||
// src/indexing-pipeline/extraction.ts
|
||||
var ContentExtractor = class {
|
||||
extractFromFile(file, content) {
|
||||
const frontmatter = {};
|
||||
const headings = [];
|
||||
const embeddedCodeBlocks = [];
|
||||
let firstParagraph;
|
||||
const frontmatterMatch = content.match(/^---(.*?)---/s);
|
||||
if (frontmatterMatch) {
|
||||
try {
|
||||
const frontmatterContent = frontmatterMatch[1];
|
||||
const lines = frontmatterContent.trim().split("\n");
|
||||
for (const line of lines) {
|
||||
const [key, ...valueParts] = line.split(":");
|
||||
if (!key) continue;
|
||||
const value = valueParts.join(":").trim();
|
||||
if (key.trim() === "title") {
|
||||
if (value) {
|
||||
frontmatter.title = value;
|
||||
}
|
||||
} else if (key.trim() === "tags") {
|
||||
if (value) {
|
||||
frontmatter.tags = value;
|
||||
}
|
||||
} else {
|
||||
frontmatter[key.trim()] = value;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
const headingMatches = content.match(/^#{1,6} (.*?)$/gm);
|
||||
if (headingMatches) {
|
||||
headings.push(...headingMatches.map((h) => h.replace(/^#{1,6} /, "")));
|
||||
}
|
||||
const codeBlockMatches = content.match(/```([\s\S]*?)```/g);
|
||||
if (codeBlockMatches) {
|
||||
embeddedCodeBlocks.push(...codeBlockMatches);
|
||||
}
|
||||
const paragraphMatch = content.match(/^([^#]*?)(?=#|\n\n|$)/s);
|
||||
if (paragraphMatch) {
|
||||
firstParagraph = paragraphMatch[1].trim();
|
||||
}
|
||||
return {
|
||||
basename: file.basename,
|
||||
path: file.path,
|
||||
content,
|
||||
frontmatter,
|
||||
headings,
|
||||
embeddedCodeBlocks,
|
||||
firstParagraph
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Extracts just the raw text content without headers, frontmatter, etc.
|
||||
*/
|
||||
extractRawText(content) {
|
||||
return content.replace(/^---.*?---/s, "").replace(/^#.*?$/gm, "").replace(/```.*?```/gs, "").replace(/`.*?`/g, "").replace(/\[(.*?)\]\(.*?\)/g, "$1").replace(/\*\*(.*?)\*\*/g, "$1").replace(/\*(.*?)\*/g, "$1").trim();
|
||||
}
|
||||
};
|
||||
|
||||
// src/indexing-pipeline/normalization.ts
|
||||
var ContentNormalizer = class {
|
||||
/**
|
||||
* Normalizes content by:
|
||||
* - Standardizing dates to ISO 8601
|
||||
* - Converting to lowercase for tokenization
|
||||
* - Extracting tokens
|
||||
* - Adding metadata
|
||||
*/
|
||||
normalize(extractedContent) {
|
||||
const { basename, path, content, frontmatter, headings, firstParagraph } = extractedContent;
|
||||
const title = basename.replace(/\.md$/, "");
|
||||
const tokens = this.tokenize(content);
|
||||
const normalizedFrontmatter = this.normalizeFrontmatter(frontmatter);
|
||||
const wordCount = content.split(/\s+/).filter(Boolean).length;
|
||||
return {
|
||||
path,
|
||||
title,
|
||||
content,
|
||||
tokens,
|
||||
headings,
|
||||
frontmatter: normalizedFrontmatter,
|
||||
firstParagraph,
|
||||
wordCount,
|
||||
// Add timestamps if available in frontmatter
|
||||
createdAt: this.extractDate(frontmatter, "created") || this.extractDate(frontmatter, "date"),
|
||||
updatedAt: this.extractDate(frontmatter, "updated")
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Tokenizes text content by splitting on whitespace and removing stop words
|
||||
*/
|
||||
tokenize(text) {
|
||||
const stopWords = /* @__PURE__ */ new Set([
|
||||
"the",
|
||||
"a",
|
||||
"an",
|
||||
"and",
|
||||
"or",
|
||||
"but",
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"in",
|
||||
"on",
|
||||
"at",
|
||||
"to",
|
||||
"of",
|
||||
"for",
|
||||
"with",
|
||||
"as",
|
||||
"by",
|
||||
"it",
|
||||
"its",
|
||||
"that",
|
||||
"this",
|
||||
"these",
|
||||
"those",
|
||||
"from",
|
||||
"up",
|
||||
"out",
|
||||
"off",
|
||||
"over",
|
||||
"under",
|
||||
"again",
|
||||
"further",
|
||||
"then",
|
||||
"once",
|
||||
"here",
|
||||
"there",
|
||||
"when",
|
||||
"where",
|
||||
"why",
|
||||
"how",
|
||||
"all",
|
||||
"any",
|
||||
"both",
|
||||
"each",
|
||||
"few",
|
||||
"more",
|
||||
"most",
|
||||
"other",
|
||||
"some",
|
||||
"such",
|
||||
"no",
|
||||
"nor",
|
||||
"not",
|
||||
"only",
|
||||
"own",
|
||||
"same",
|
||||
"so",
|
||||
"than",
|
||||
"too",
|
||||
"very",
|
||||
"just",
|
||||
"now"
|
||||
]);
|
||||
return text.toLowerCase().split(/\W+/).filter((token) => token.length > 1 && !stopWords.has(token));
|
||||
}
|
||||
/**
|
||||
* Normalizes frontmatter by standardizing data types and formats
|
||||
*/
|
||||
normalizeFrontmatter(frontmatter) {
|
||||
const normalized = {};
|
||||
for (const [key, value] of Object.entries(frontmatter)) {
|
||||
if (key === "tags" && typeof value === "string") {
|
||||
normalized.tags = value.split(",").map((tag) => tag.trim());
|
||||
} else if (key === "date" || key === "created" || key === "updated") {
|
||||
if (typeof value === "string") {
|
||||
const date = new Date(value);
|
||||
if (!isNaN(date.getTime())) {
|
||||
normalized[key] = date.toISOString();
|
||||
} else {
|
||||
normalized[key] = value;
|
||||
}
|
||||
} else {
|
||||
normalized[key] = value;
|
||||
}
|
||||
} else {
|
||||
normalized[key] = value;
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
/**
|
||||
* Extracts a date from frontmatter
|
||||
*/
|
||||
extractDate(frontmatter, key) {
|
||||
const value = frontmatter[key];
|
||||
if (typeof value === "string") {
|
||||
const date = new Date(value);
|
||||
if (!isNaN(date.getTime())) {
|
||||
return date.toISOString();
|
||||
}
|
||||
}
|
||||
return void 0;
|
||||
}
|
||||
};
|
||||
|
||||
// src/vault-vector-store.ts
|
||||
var VaultVectorStore = class {
|
||||
constructor(ollamaURL, config) {
|
||||
this.client = null;
|
||||
this.collection = null;
|
||||
this.isInitialized = false;
|
||||
this.ollamaURL = ollamaURL.replace(/\/+$/, "");
|
||||
this.config = config;
|
||||
this.vectorizer = new ContentVectorizer({
|
||||
model: config.embeddingModel,
|
||||
ollamaUrl: this.ollamaURL
|
||||
});
|
||||
this.extractor = new ContentExtractor();
|
||||
this.normalizer = new ContentNormalizer();
|
||||
}
|
||||
async initialize() {
|
||||
if (!this.config.enabled || this.isInitialized) return;
|
||||
try {
|
||||
const rawURL = this.config.chromaURL?.trim() || "http://localhost:8000";
|
||||
const chromaURL = rawURL.includes("://") ? rawURL : "http://localhost:8000";
|
||||
this.client = new ChromaClient({ path: chromaURL });
|
||||
this.collection = await this.client.getOrCreateCollection({
|
||||
name: this.config.collectionName,
|
||||
metadata: { "hnsw:space": "cosine" }
|
||||
});
|
||||
this.isInitialized = true;
|
||||
Logger.info(
|
||||
`Vault vector store initialized: ${this.config.collectionName}`,
|
||||
"vault-vector-store"
|
||||
);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.error(
|
||||
`Failed to initialize vault vector store: ${errorMessage}`,
|
||||
"vault-vector-store"
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Index a single vault file by generating an embedding and storing it in ChromaDB.
|
||||
*/
|
||||
async indexFile(file, content) {
|
||||
if (!this.collection || !this.config.enabled) return;
|
||||
if (!content.trim()) {
|
||||
await this.deleteFile(file.path);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const extracted = this.extractor.extractFromFile(
|
||||
{ basename: file.basename, path: file.path },
|
||||
content
|
||||
);
|
||||
const normalized = this.normalizer.normalize(extracted);
|
||||
const chunk = {
|
||||
id: file.path,
|
||||
path: file.path,
|
||||
title: normalized.title,
|
||||
content: normalized.content,
|
||||
tokens: normalized.tokens,
|
||||
headings: normalized.headings,
|
||||
frontmatter: normalized.frontmatter,
|
||||
firstParagraph: normalized.firstParagraph,
|
||||
wordCount: normalized.wordCount,
|
||||
chunkIndex: 0,
|
||||
chunkSize: 1
|
||||
};
|
||||
const embedding = await this.vectorizer.vectorize(chunk);
|
||||
if (embedding.length === 0) {
|
||||
Logger.warn(`Empty embedding for ${file.path}, skipping index`, "vault-vector-store");
|
||||
return;
|
||||
}
|
||||
await this.collection.upsert({
|
||||
ids: [file.path],
|
||||
documents: [this.createDocumentText(normalized)],
|
||||
embeddings: [embedding],
|
||||
metadatas: [
|
||||
{
|
||||
path: file.path,
|
||||
title: normalized.title,
|
||||
tags: typeof normalized.frontmatter.tags === "string" ? normalized.frontmatter.tags : ""
|
||||
}
|
||||
]
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(`Failed to index file ${file.path}: ${errorMessage}`, "vault-vector-store");
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Remove a file from the vector index.
|
||||
*/
|
||||
async deleteFile(path) {
|
||||
if (!this.collection || !this.config.enabled) return;
|
||||
try {
|
||||
await this.collection.delete({ ids: [path] });
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(
|
||||
`Failed to delete file ${path} from index: ${errorMessage}`,
|
||||
"vault-vector-store"
|
||||
);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Perform semantic search against the vault index.
|
||||
*/
|
||||
async search(query, limit = 3) {
|
||||
if (!this.collection || !this.config.enabled) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const queryEmbedding = await this.vectorizer.vectorize({
|
||||
id: "query",
|
||||
path: "query",
|
||||
title: query,
|
||||
content: query,
|
||||
tokens: query.toLowerCase().split(/\s+/).filter((t) => t.length > 1),
|
||||
headings: [query],
|
||||
frontmatter: {},
|
||||
firstParagraph: query,
|
||||
wordCount: query.split(/\s+/).length,
|
||||
chunkIndex: 0,
|
||||
chunkSize: 1
|
||||
});
|
||||
if (queryEmbedding.length === 0) {
|
||||
Logger.warn("Empty query embedding, skipping semantic search", "vault-vector-store");
|
||||
return [];
|
||||
}
|
||||
const results = await this.collection.query({
|
||||
queryEmbeddings: [queryEmbedding],
|
||||
nResults: limit
|
||||
});
|
||||
if (!results.ids[0] || results.ids[0].length === 0) {
|
||||
return [];
|
||||
}
|
||||
const entries = [];
|
||||
for (let i = 0; i < results.ids[0].length; i++) {
|
||||
const distance = results.distances?.[0]?.[i] ?? 0;
|
||||
const score = 1 - distance;
|
||||
if (score < this.config.similarityThreshold) {
|
||||
continue;
|
||||
}
|
||||
const metadata = results.metadatas?.[0]?.[i];
|
||||
const document = results.documents?.[0]?.[i];
|
||||
entries.push({
|
||||
path: metadata?.path || String(results.ids[0][i]),
|
||||
title: metadata?.title || "Untitled",
|
||||
content: document || "",
|
||||
score,
|
||||
tags: metadata?.tags
|
||||
});
|
||||
}
|
||||
entries.sort((a, b) => b.score - a.score);
|
||||
return entries.slice(0, limit);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(`Semantic search failed: ${errorMessage}`, "vault-vector-store");
|
||||
return [];
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Clear the entire vault index.
|
||||
*/
|
||||
async clearIndex() {
|
||||
if (!this.client || !this.config.enabled) return;
|
||||
try {
|
||||
await this.client.deleteCollection({ name: this.config.collectionName });
|
||||
this.collection = null;
|
||||
this.isInitialized = false;
|
||||
Logger.info("Vault index cleared", "vault-vector-store");
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.error(`Failed to clear vault index: ${errorMessage}`, "vault-vector-store");
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get the number of indexed documents.
|
||||
*/
|
||||
async getIndexedCount() {
|
||||
if (!this.collection || !this.config.enabled) return 0;
|
||||
try {
|
||||
const result = await this.collection.count();
|
||||
return result;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
createDocumentText(normalized) {
|
||||
const parts = [];
|
||||
if (normalized.title) parts.push(`Title: ${normalized.title}`);
|
||||
if (normalized.firstParagraph) parts.push(normalized.firstParagraph);
|
||||
if (normalized.headings.length > 0) parts.push(normalized.headings.join("\n"));
|
||||
parts.push(normalized.content.substring(0, 1e3));
|
||||
return parts.join("\n\n");
|
||||
}
|
||||
};
|
||||
|
||||
@@ -9413,9 +9900,12 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
|
||||
}
|
||||
async onload() {
|
||||
await this.loadSettings();
|
||||
if (this.settings.vaultIndexConfig.enabled) {
|
||||
await this.initializeVaultVectorStore();
|
||||
}
|
||||
this.registerView(
|
||||
"ollama-chat-view",
|
||||
(leaf) => new ChatView(leaf, this.settings)
|
||||
(leaf) => new ChatView(leaf, this.settings, this.vaultVectorStore)
|
||||
);
|
||||
this.addRibbonIcon("bot", "Open Ollama Chat", async () => {
|
||||
await this.activateChatView();
|
||||
@@ -9435,6 +9925,23 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
|
||||
new import_obsidian4.Notice("Semantic cache cleared.");
|
||||
}
|
||||
});
|
||||
this.addCommand({
|
||||
id: "clear-vault-index",
|
||||
name: "Clear Vault Index",
|
||||
callback: async () => {
|
||||
await this.clearVaultIndex();
|
||||
new import_obsidian4.Notice("Vault index cleared.");
|
||||
}
|
||||
});
|
||||
this.addCommand({
|
||||
id: "rebuild-vault-index",
|
||||
name: "Rebuild Vault Index",
|
||||
callback: async () => {
|
||||
new import_obsidian4.Notice("Rebuilding vault index...");
|
||||
await this.rebuildVaultIndex();
|
||||
new import_obsidian4.Notice("Vault index rebuilt.");
|
||||
}
|
||||
});
|
||||
this.addSettingTab(new OllamaSettingTab(this.app, this));
|
||||
if (this.settings.cacheConfig) {
|
||||
this.semanticCache = new SemanticCacheService(
|
||||
@@ -9447,6 +9954,7 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
|
||||
new import_obsidian4.Notice("Semantic cache initialization failed. Check console for details.");
|
||||
}
|
||||
}
|
||||
this.registerVaultEventListeners();
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
onunload() {
|
||||
@@ -9461,6 +9969,89 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
|
||||
async saveSettings() {
|
||||
await this.saveData(this.settings);
|
||||
}
|
||||
async initializeVaultVectorStore() {
|
||||
this.vaultVectorStore = new VaultVectorStore(
|
||||
this.settings.ollamaUrl,
|
||||
this.settings.vaultIndexConfig
|
||||
);
|
||||
try {
|
||||
await this.vaultVectorStore.initialize();
|
||||
void this.backgroundIndexVault().catch((err) => {
|
||||
Logger.warn(`Background vault indexing failed: ${String(err)}`, "main");
|
||||
});
|
||||
} catch {
|
||||
new import_obsidian4.Notice("Vault vector store initialization failed. Check console for details.");
|
||||
}
|
||||
}
|
||||
async backgroundIndexVault() {
|
||||
if (!this.vaultVectorStore) return;
|
||||
const files = this.app.vault.getMarkdownFiles();
|
||||
Logger.info(`Starting background vault indexing for ${files.length} files...`, "main");
|
||||
let indexed = 0;
|
||||
for (const file of files) {
|
||||
try {
|
||||
const content = await this.app.vault.read(file);
|
||||
await this.vaultVectorStore.indexFile(file, content);
|
||||
indexed++;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main");
|
||||
}
|
||||
}
|
||||
this.settings.lastIndexTime = Date.now();
|
||||
await this.saveSettings();
|
||||
Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, "main");
|
||||
new import_obsidian4.Notice(`Vault index updated: ${indexed} files indexed.`);
|
||||
}
|
||||
async rebuildVaultIndex() {
|
||||
if (this.vaultVectorStore) {
|
||||
await this.vaultVectorStore.clearIndex();
|
||||
await this.vaultVectorStore.initialize();
|
||||
}
|
||||
await this.backgroundIndexVault();
|
||||
}
|
||||
async clearVaultIndex() {
|
||||
if (this.vaultVectorStore) {
|
||||
await this.vaultVectorStore.clearIndex();
|
||||
}
|
||||
}
|
||||
registerVaultEventListeners() {
|
||||
this.registerEvent(
|
||||
this.app.vault.on("create", (file) => {
|
||||
if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) {
|
||||
void this.app.vault.read(file).then((content) => {
|
||||
void this.vaultVectorStore?.indexFile(file, content);
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
this.registerEvent(
|
||||
this.app.vault.on("modify", (file) => {
|
||||
if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) {
|
||||
void this.app.vault.read(file).then((content) => {
|
||||
void this.vaultVectorStore?.indexFile(file, content);
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
this.registerEvent(
|
||||
this.app.vault.on("delete", (file) => {
|
||||
if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) {
|
||||
void this.vaultVectorStore.deleteFile(file.path);
|
||||
}
|
||||
})
|
||||
);
|
||||
this.registerEvent(
|
||||
this.app.vault.on("rename", (file, oldPath) => {
|
||||
if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) {
|
||||
void this.vaultVectorStore.deleteFile(oldPath);
|
||||
void this.app.vault.read(file).then((content) => {
|
||||
void this.vaultVectorStore?.indexFile(file, content);
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
async activateChatView() {
|
||||
const existing = this.app.workspace.getLeavesOfType("ollama-chat-view");
|
||||
if (existing.length > 0) {
|
||||
@@ -9485,6 +10076,7 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
|
||||
leaves.forEach((leaf) => {
|
||||
if (leaf.view instanceof ChatView) {
|
||||
leaf.view.updateSettings(this.settings);
|
||||
leaf.view.setVectorStore(this.vaultVectorStore);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -9532,6 +10124,77 @@ var OllamaSettingTab = class extends import_obsidian4.PluginSettingTab {
|
||||
}
|
||||
})
|
||||
);
|
||||
containerEl.createEl("h3", { text: "Vault Semantic Index" });
|
||||
new import_obsidian4.Setting(containerEl).setName("Enable Vault Semantic Index").setDesc(
|
||||
"Automatically index vault notes into a vector database for semantic/RAG search. Requires ChromaDB and an embedding model."
|
||||
).addToggle(
|
||||
(toggle) => toggle.setValue(this.plugin.settings.vaultIndexConfig.enabled).onChange(async (value) => {
|
||||
this.plugin.settings.vaultIndexConfig.enabled = value;
|
||||
await this.plugin.saveSettings();
|
||||
this.plugin.notifyChatViews();
|
||||
if (value) {
|
||||
new import_obsidian4.Notice("Vault semantic index enabled. Rebuilding index...");
|
||||
await this.plugin.initializeVaultVectorStore();
|
||||
await this.plugin.rebuildVaultIndex();
|
||||
} else {
|
||||
await this.plugin.clearVaultIndex();
|
||||
this.plugin.vaultVectorStore = void 0;
|
||||
this.plugin.notifyChatViews();
|
||||
}
|
||||
})
|
||||
);
|
||||
new import_obsidian4.Setting(containerEl).setName("Vault Index ChromaDB URL").setDesc(
|
||||
"URL for your ChromaDB instance used for the vault index (default: http://localhost:8000)"
|
||||
).addText(
|
||||
(text) => text.setValue(this.plugin.settings.vaultIndexConfig.chromaURL || "http://localhost:8000").onChange(async (value) => {
|
||||
const trimmed = value.trim();
|
||||
this.plugin.settings.vaultIndexConfig.chromaURL = trimmed && trimmed.includes("://") ? trimmed : "http://localhost:8000";
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
new import_obsidian4.Setting(containerEl).setName("Vault Index Embedding Model").setDesc(
|
||||
"Ollama model used to generate embeddings for vault notes (default: nomic-embed-text)"
|
||||
).addText(
|
||||
(text) => text.setValue(this.plugin.settings.vaultIndexConfig.embeddingModel).onChange(async (value) => {
|
||||
this.plugin.settings.vaultIndexConfig.embeddingModel = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
new import_obsidian4.Setting(containerEl).setName("Vault Index Similarity Threshold").setDesc(
|
||||
"Minimum cosine similarity (0\u20131) for a vault search hit. Higher values require closer matches (default: 0.75)."
|
||||
).addText(
|
||||
(text) => text.setValue(String(this.plugin.settings.vaultIndexConfig.similarityThreshold)).onChange(async (value) => {
|
||||
const parsed = parseFloat(value);
|
||||
if (!isNaN(parsed) && parsed >= 0 && parsed <= 1) {
|
||||
this.plugin.settings.vaultIndexConfig.similarityThreshold = parsed;
|
||||
await this.plugin.saveSettings();
|
||||
} else {
|
||||
new import_obsidian4.Notice("Similarity threshold must be a number between 0 and 1.");
|
||||
}
|
||||
})
|
||||
);
|
||||
new import_obsidian4.Setting(containerEl).setName("Rebuild Vault Index").setDesc("Delete and rebuild the entire vault semantic index").addButton(
|
||||
(button) => button.setButtonText("Rebuild Index").onClick(async () => {
|
||||
try {
|
||||
new import_obsidian4.Notice("Rebuilding vault index...");
|
||||
await this.plugin.rebuildVaultIndex();
|
||||
new import_obsidian4.Notice("Vault index rebuilt.");
|
||||
} catch {
|
||||
new import_obsidian4.Notice("Failed to rebuild vault index. Is ChromaDB running?");
|
||||
}
|
||||
})
|
||||
);
|
||||
new import_obsidian4.Setting(containerEl).setName("Clear Vault Index").setDesc("Delete all indexed vault notes from ChromaDB").addButton(
|
||||
(button) => button.setButtonText("Clear Index").onClick(async () => {
|
||||
try {
|
||||
await this.plugin.clearVaultIndex();
|
||||
new import_obsidian4.Notice("Vault index cleared.");
|
||||
} catch {
|
||||
new import_obsidian4.Notice("Failed to clear vault index. Is ChromaDB running?");
|
||||
}
|
||||
})
|
||||
);
|
||||
containerEl.createEl("h3", { text: "Semantic Cache" });
|
||||
new import_obsidian4.Setting(containerEl).setName("Enable Semantic Cache").setDesc("Use semantic cache to store and retrieve previous responses").addToggle(
|
||||
(toggle) => toggle.setValue(this.plugin.settings.cacheConfig.enabled).onChange(async (value) => {
|
||||
this.plugin.settings.cacheConfig.enabled = value;
|
||||
|
||||
+9
-2
@@ -1,6 +1,7 @@
|
||||
import { ItemView, Notice, WorkspaceLeaf } from 'obsidian';
|
||||
import { OllamaClient } from './ollama-client';
|
||||
import { VaultIndexer } from './vault-indexer';
|
||||
import { VaultVectorStore } from './vault-vector-store';
|
||||
import { ToolExecutor } from './tool-executor';
|
||||
import { PluginSettings, OllamaMessage, OllamaTool, OllamaToolCall, ChatMessage } from './types';
|
||||
import { ConversationStateManager } from './conversation-state';
|
||||
@@ -22,7 +23,7 @@ export class ChatView extends ItemView {
|
||||
return this.newChatButtonClickHandler;
|
||||
}
|
||||
|
||||
constructor(leaf: WorkspaceLeaf, settings: PluginSettings) {
|
||||
constructor(leaf: WorkspaceLeaf, settings: PluginSettings, vectorStore?: VaultVectorStore) {
|
||||
super(leaf);
|
||||
this.messages = [];
|
||||
this.lastMessageEl = null;
|
||||
@@ -44,7 +45,7 @@ export class ChatView extends ItemView {
|
||||
undefined,
|
||||
settings.cacheConfig
|
||||
);
|
||||
this.vaultIndexer = new VaultIndexer(this.app.vault);
|
||||
this.vaultIndexer = new VaultIndexer(this.app.vault, undefined, vectorStore);
|
||||
this.toolExecutor = new ToolExecutor(this.app.vault, this.app);
|
||||
this.conversationStateManager = new ConversationStateManager();
|
||||
}
|
||||
@@ -64,6 +65,11 @@ export class ChatView extends ItemView {
|
||||
});
|
||||
}
|
||||
|
||||
setVectorStore(vectorStore: VaultVectorStore | undefined) {
|
||||
this.vectorStore = vectorStore;
|
||||
this.vaultIndexer.setVectorStore(vectorStore);
|
||||
}
|
||||
|
||||
async clearCache(): Promise<void> {
|
||||
await this.ollamaClient.clearCache();
|
||||
}
|
||||
@@ -567,6 +573,7 @@ export class ChatView extends ItemView {
|
||||
private vaultIndexer: VaultIndexer;
|
||||
private toolExecutor: ToolExecutor;
|
||||
private conversationStateManager: ConversationStateManager;
|
||||
private vectorStore?: VaultVectorStore;
|
||||
}
|
||||
|
||||
const MAX_TOOL_CALLS = 5;
|
||||
|
||||
@@ -11,4 +11,11 @@ export const DEFAULT_SETTINGS = {
|
||||
embeddingModel: 'nomic-embed-text',
|
||||
chromaURL: 'http://localhost:8000',
|
||||
},
|
||||
vaultIndexConfig: {
|
||||
enabled: false,
|
||||
similarityThreshold: 0.75,
|
||||
collectionName: 'ollama_vault_index',
|
||||
embeddingModel: 'nomic-embed-text',
|
||||
chromaURL: 'http://localhost:8000',
|
||||
},
|
||||
};
|
||||
|
||||
+240
-2
@@ -1,20 +1,28 @@
|
||||
import { Plugin, WorkspaceLeaf, App, Setting, Notice, PluginSettingTab } from 'obsidian';
|
||||
import { Plugin, WorkspaceLeaf, App, Setting, Notice, PluginSettingTab, TFile } from 'obsidian';
|
||||
import { ChatView } from './chat-view';
|
||||
import { DEFAULT_SETTINGS } from './constants';
|
||||
import { SemanticCacheService } from './semantic-cache';
|
||||
import { VaultVectorStore } from './vault-vector-store';
|
||||
import { PluginSettings } from './types';
|
||||
import { Logger } from './utils';
|
||||
|
||||
export default class OllamaPlugin extends Plugin {
|
||||
settings: PluginSettings = DEFAULT_SETTINGS;
|
||||
semanticCache?: SemanticCacheService;
|
||||
vaultVectorStore?: VaultVectorStore;
|
||||
|
||||
async onload() {
|
||||
await this.loadSettings();
|
||||
|
||||
// Initialize vault vector store if enabled
|
||||
if (this.settings.vaultIndexConfig.enabled) {
|
||||
await this.initializeVaultVectorStore();
|
||||
}
|
||||
|
||||
// Register the chat view
|
||||
this.registerView(
|
||||
'ollama-chat-view',
|
||||
(leaf: WorkspaceLeaf) => new ChatView(leaf, this.settings)
|
||||
(leaf: WorkspaceLeaf) => new ChatView(leaf, this.settings, this.vaultVectorStore)
|
||||
);
|
||||
|
||||
// Add a ribbon icon in the left sidebar
|
||||
@@ -41,6 +49,27 @@ export default class OllamaPlugin extends Plugin {
|
||||
},
|
||||
});
|
||||
|
||||
// Add a command to clear the vault index
|
||||
this.addCommand({
|
||||
id: 'clear-vault-index',
|
||||
name: 'Clear Vault Index',
|
||||
callback: async () => {
|
||||
await this.clearVaultIndex();
|
||||
new Notice('Vault index cleared.');
|
||||
},
|
||||
});
|
||||
|
||||
// Add a command to rebuild the vault index
|
||||
this.addCommand({
|
||||
id: 'rebuild-vault-index',
|
||||
name: 'Rebuild Vault Index',
|
||||
callback: async () => {
|
||||
new Notice('Rebuilding vault index...');
|
||||
await this.rebuildVaultIndex();
|
||||
new Notice('Vault index rebuilt.');
|
||||
},
|
||||
});
|
||||
|
||||
// Add a settings tab
|
||||
this.addSettingTab(new OllamaSettingTab(this.app, this));
|
||||
|
||||
@@ -56,6 +85,9 @@ export default class OllamaPlugin extends Plugin {
|
||||
new Notice('Semantic cache initialization failed. Check console for details.');
|
||||
}
|
||||
}
|
||||
|
||||
// Set up vault event listeners for incremental indexing
|
||||
this.registerVaultEventListeners();
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
@@ -77,6 +109,105 @@ export default class OllamaPlugin extends Plugin {
|
||||
await this.saveData(this.settings);
|
||||
}
|
||||
|
||||
async initializeVaultVectorStore(): Promise<void> {
|
||||
this.vaultVectorStore = new VaultVectorStore(
|
||||
this.settings.ollamaUrl,
|
||||
this.settings.vaultIndexConfig
|
||||
);
|
||||
try {
|
||||
await this.vaultVectorStore.initialize();
|
||||
// Run background indexing
|
||||
void this.backgroundIndexVault().catch((err) => {
|
||||
Logger.warn(`Background vault indexing failed: ${String(err)}`, 'main');
|
||||
});
|
||||
} catch {
|
||||
new Notice('Vault vector store initialization failed. Check console for details.');
|
||||
}
|
||||
}
|
||||
|
||||
async backgroundIndexVault(): Promise<void> {
|
||||
if (!this.vaultVectorStore) return;
|
||||
|
||||
const files = this.app.vault.getMarkdownFiles();
|
||||
Logger.info(`Starting background vault indexing for ${files.length} files...`, 'main');
|
||||
|
||||
let indexed = 0;
|
||||
for (const file of files) {
|
||||
try {
|
||||
const content = await this.app.vault.read(file);
|
||||
await this.vaultVectorStore.indexFile(file, content);
|
||||
indexed++;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, 'main');
|
||||
}
|
||||
}
|
||||
|
||||
this.settings.lastIndexTime = Date.now();
|
||||
await this.saveSettings();
|
||||
Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, 'main');
|
||||
new Notice(`Vault index updated: ${indexed} files indexed.`);
|
||||
}
|
||||
|
||||
async rebuildVaultIndex(): Promise<void> {
|
||||
if (this.vaultVectorStore) {
|
||||
await this.vaultVectorStore.clearIndex();
|
||||
await this.vaultVectorStore.initialize();
|
||||
}
|
||||
await this.backgroundIndexVault();
|
||||
}
|
||||
|
||||
async clearVaultIndex(): Promise<void> {
|
||||
if (this.vaultVectorStore) {
|
||||
await this.vaultVectorStore.clearIndex();
|
||||
}
|
||||
}
|
||||
|
||||
registerVaultEventListeners(): void {
|
||||
// Listen for file creation
|
||||
this.registerEvent(
|
||||
this.app.vault.on('create', (file) => {
|
||||
if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) {
|
||||
void this.app.vault.read(file).then((content) => {
|
||||
void this.vaultVectorStore?.indexFile(file, content);
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Listen for file modification
|
||||
this.registerEvent(
|
||||
this.app.vault.on('modify', (file) => {
|
||||
if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) {
|
||||
void this.app.vault.read(file).then((content) => {
|
||||
void this.vaultVectorStore?.indexFile(file, content);
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Listen for file deletion
|
||||
this.registerEvent(
|
||||
this.app.vault.on('delete', (file) => {
|
||||
if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) {
|
||||
void this.vaultVectorStore.deleteFile(file.path);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Listen for file renames
|
||||
this.registerEvent(
|
||||
this.app.vault.on('rename', (file, oldPath) => {
|
||||
if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) {
|
||||
void this.vaultVectorStore.deleteFile(oldPath);
|
||||
void this.app.vault.read(file).then((content) => {
|
||||
void this.vaultVectorStore?.indexFile(file, content);
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async activateChatView() {
|
||||
const existing = this.app.workspace.getLeavesOfType('ollama-chat-view');
|
||||
if (existing.length > 0) {
|
||||
@@ -103,6 +234,7 @@ export default class OllamaPlugin extends Plugin {
|
||||
leaves.forEach((leaf) => {
|
||||
if (leaf.view instanceof ChatView) {
|
||||
leaf.view.updateSettings(this.settings);
|
||||
leaf.view.setVectorStore(this.vaultVectorStore);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -171,6 +303,112 @@ class OllamaSettingTab extends PluginSettingTab {
|
||||
})
|
||||
);
|
||||
|
||||
// Vault Index Settings
|
||||
containerEl.createEl('h3', { text: 'Vault Semantic Index' });
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Enable Vault Semantic Index')
|
||||
.setDesc(
|
||||
'Automatically index vault notes into a vector database for semantic/RAG search. Requires ChromaDB and an embedding model.'
|
||||
)
|
||||
.addToggle((toggle) =>
|
||||
toggle.setValue(this.plugin.settings.vaultIndexConfig.enabled).onChange(async (value) => {
|
||||
this.plugin.settings.vaultIndexConfig.enabled = value;
|
||||
await this.plugin.saveSettings();
|
||||
this.plugin.notifyChatViews();
|
||||
if (value) {
|
||||
new Notice('Vault semantic index enabled. Rebuilding index...');
|
||||
await this.plugin.initializeVaultVectorStore();
|
||||
await this.plugin.rebuildVaultIndex();
|
||||
} else {
|
||||
await this.plugin.clearVaultIndex();
|
||||
this.plugin.vaultVectorStore = undefined;
|
||||
this.plugin.notifyChatViews();
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Vault Index ChromaDB URL')
|
||||
.setDesc(
|
||||
'URL for your ChromaDB instance used for the vault index (default: http://localhost:8000)'
|
||||
)
|
||||
.addText((text) =>
|
||||
text
|
||||
.setValue(this.plugin.settings.vaultIndexConfig.chromaURL || 'http://localhost:8000')
|
||||
.onChange(async (value) => {
|
||||
const trimmed = value.trim();
|
||||
this.plugin.settings.vaultIndexConfig.chromaURL =
|
||||
trimmed && trimmed.includes('://') ? trimmed : 'http://localhost:8000';
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Vault Index Embedding Model')
|
||||
.setDesc(
|
||||
'Ollama model used to generate embeddings for vault notes (default: nomic-embed-text)'
|
||||
)
|
||||
.addText((text) =>
|
||||
text
|
||||
.setValue(this.plugin.settings.vaultIndexConfig.embeddingModel)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.vaultIndexConfig.embeddingModel = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Vault Index Similarity Threshold')
|
||||
.setDesc(
|
||||
'Minimum cosine similarity (0–1) for a vault search hit. Higher values require closer matches (default: 0.75).'
|
||||
)
|
||||
.addText((text) =>
|
||||
text
|
||||
.setValue(String(this.plugin.settings.vaultIndexConfig.similarityThreshold))
|
||||
.onChange(async (value) => {
|
||||
const parsed = parseFloat(value);
|
||||
if (!isNaN(parsed) && parsed >= 0 && parsed <= 1) {
|
||||
this.plugin.settings.vaultIndexConfig.similarityThreshold = parsed;
|
||||
await this.plugin.saveSettings();
|
||||
} else {
|
||||
new Notice('Similarity threshold must be a number between 0 and 1.');
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Rebuild Vault Index')
|
||||
.setDesc('Delete and rebuild the entire vault semantic index')
|
||||
.addButton((button) =>
|
||||
button.setButtonText('Rebuild Index').onClick(async () => {
|
||||
try {
|
||||
new Notice('Rebuilding vault index...');
|
||||
await this.plugin.rebuildVaultIndex();
|
||||
new Notice('Vault index rebuilt.');
|
||||
} catch {
|
||||
new Notice('Failed to rebuild vault index. Is ChromaDB running?');
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Clear Vault Index')
|
||||
.setDesc('Delete all indexed vault notes from ChromaDB')
|
||||
.addButton((button) =>
|
||||
button.setButtonText('Clear Index').onClick(async () => {
|
||||
try {
|
||||
await this.plugin.clearVaultIndex();
|
||||
new Notice('Vault index cleared.');
|
||||
} catch {
|
||||
new Notice('Failed to clear vault index. Is ChromaDB running?');
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Semantic Cache Settings
|
||||
containerEl.createEl('h3', { text: 'Semantic Cache' });
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Enable Semantic Cache')
|
||||
.setDesc('Use semantic cache to store and retrieve previous responses')
|
||||
|
||||
@@ -133,6 +133,7 @@ export interface VaultIndexEntry {
|
||||
title: string;
|
||||
content: string;
|
||||
score: number;
|
||||
tags?: string;
|
||||
}
|
||||
|
||||
export interface ChatMessage {
|
||||
@@ -176,6 +177,14 @@ export interface CacheConfig {
|
||||
chromaURL?: string;
|
||||
}
|
||||
|
||||
export interface VaultIndexConfig {
|
||||
enabled: boolean;
|
||||
collectionName: string;
|
||||
embeddingModel: string;
|
||||
chromaURL?: string;
|
||||
similarityThreshold: number;
|
||||
}
|
||||
|
||||
export interface PluginSettings {
|
||||
ollamaUrl: string;
|
||||
model: string;
|
||||
@@ -183,6 +192,7 @@ export interface PluginSettings {
|
||||
maxMessageHistory: number;
|
||||
lastIndexTime: number;
|
||||
cacheConfig: CacheConfig;
|
||||
vaultIndexConfig: VaultIndexConfig;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
|
||||
+92
-14
@@ -3,6 +3,8 @@
|
||||
import { Vault, TFile } from 'obsidian';
|
||||
import { Logger } from './utils';
|
||||
import { Cache } from './cache';
|
||||
import { VaultVectorStore } from './vault-vector-store';
|
||||
import { VaultIndexEntry } from './types';
|
||||
|
||||
interface ParsedFrontmatter {
|
||||
title?: string;
|
||||
@@ -43,12 +45,63 @@ export class InMemoryCache implements Cache {
|
||||
}
|
||||
|
||||
const STOP_WORDS = new Set([
|
||||
'a', 'an', 'the', 'is', 'it', 'in', 'on', 'at', 'to', 'for', 'of',
|
||||
'and', 'or', 'but', 'with', 'by', 'from', 'up', 'about', 'into',
|
||||
'this', 'that', 'these', 'those', 'be', 'been', 'being', 'have',
|
||||
'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should',
|
||||
'may', 'might', 'can', 'are', 'was', 'were', 'as', 'so', 'if', 'not',
|
||||
'no', 'my', 'your', 'our', 'its', 'we', 'you', 'he', 'she', 'they',
|
||||
'a',
|
||||
'an',
|
||||
'the',
|
||||
'is',
|
||||
'it',
|
||||
'in',
|
||||
'on',
|
||||
'at',
|
||||
'to',
|
||||
'for',
|
||||
'of',
|
||||
'and',
|
||||
'or',
|
||||
'but',
|
||||
'with',
|
||||
'by',
|
||||
'from',
|
||||
'up',
|
||||
'about',
|
||||
'into',
|
||||
'this',
|
||||
'that',
|
||||
'these',
|
||||
'those',
|
||||
'be',
|
||||
'been',
|
||||
'being',
|
||||
'have',
|
||||
'has',
|
||||
'had',
|
||||
'do',
|
||||
'does',
|
||||
'did',
|
||||
'will',
|
||||
'would',
|
||||
'could',
|
||||
'should',
|
||||
'may',
|
||||
'might',
|
||||
'can',
|
||||
'are',
|
||||
'was',
|
||||
'were',
|
||||
'as',
|
||||
'so',
|
||||
'if',
|
||||
'not',
|
||||
'no',
|
||||
'my',
|
||||
'your',
|
||||
'our',
|
||||
'its',
|
||||
'we',
|
||||
'you',
|
||||
'he',
|
||||
'she',
|
||||
'they',
|
||||
]);
|
||||
|
||||
const CONTENT_PREVIEW_LENGTH = 500;
|
||||
@@ -56,6 +109,7 @@ const CONTENT_PREVIEW_LENGTH = 500;
|
||||
export class VaultIndexer {
|
||||
private vault: Vault;
|
||||
private cache?: Cache;
|
||||
private vectorStore?: VaultVectorStore;
|
||||
private readonly SCORING_WEIGHTS = {
|
||||
TITLE: 5,
|
||||
FRONTMATTER_TITLE: 4,
|
||||
@@ -64,9 +118,14 @@ export class VaultIndexer {
|
||||
CONTENT: 1,
|
||||
};
|
||||
|
||||
constructor(vault: Vault, cache?: Cache) {
|
||||
constructor(vault: Vault, cache?: Cache, vectorStore?: VaultVectorStore) {
|
||||
this.vault = vault;
|
||||
this.cache = cache;
|
||||
this.vectorStore = vectorStore;
|
||||
}
|
||||
|
||||
setVectorStore(vectorStore: VaultVectorStore | undefined): void {
|
||||
this.vectorStore = vectorStore;
|
||||
}
|
||||
|
||||
tokenize(text: string): string[] {
|
||||
@@ -95,10 +154,7 @@ export class VaultIndexer {
|
||||
};
|
||||
}
|
||||
|
||||
calculateWeightedScore(
|
||||
tokenized: TokenizedContent,
|
||||
queryTokens: string[]
|
||||
): { score: number } {
|
||||
calculateWeightedScore(tokenized: TokenizedContent, queryTokens: string[]): { score: number } {
|
||||
let score = 0;
|
||||
for (const token of queryTokens) {
|
||||
if (tokenized.frontmatter?.title && this.exactMatch(tokenized.frontmatter.title, token)) {
|
||||
@@ -150,11 +206,27 @@ export class VaultIndexer {
|
||||
return entries;
|
||||
}
|
||||
|
||||
async searchVault(query: string, limit = 3): Promise<VaultEntry[]> {
|
||||
async searchVault(query: string, limit = 3): Promise<VaultIndexEntry[]> {
|
||||
if (!query || !query.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Try semantic search first if vector store is available
|
||||
if (this.vectorStore) {
|
||||
try {
|
||||
const semanticResults = await this.vectorStore.search(query, limit);
|
||||
if (semanticResults.length > 0) {
|
||||
return semanticResults;
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(
|
||||
`Semantic search failed, falling back to keyword search: ${errorMessage}`,
|
||||
'vault-indexer'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const cacheKey = `query:${query.trim()}:limit:${limit}`;
|
||||
if (this.cache) {
|
||||
let cachedResults: string | null = null;
|
||||
@@ -166,7 +238,7 @@ export class VaultIndexer {
|
||||
if (cachedResults) {
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
||||
const parsedResults: VaultEntry[] = JSON.parse(cachedResults);
|
||||
const parsedResults: VaultIndexEntry[] = JSON.parse(cachedResults);
|
||||
return parsedResults.slice(0, limit);
|
||||
} catch {
|
||||
// ignore parse errors
|
||||
@@ -198,7 +270,13 @@ export class VaultIndexer {
|
||||
.filter((e) => e.score > 0);
|
||||
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
const results = scored.slice(0, limit);
|
||||
const results: VaultIndexEntry[] = scored.slice(0, limit).map((e) => ({
|
||||
path: e.file.path,
|
||||
title: e.title,
|
||||
content: e.content,
|
||||
score: e.score,
|
||||
tags: e.frontmatter?.tags,
|
||||
}));
|
||||
|
||||
if (this.cache) {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
// src/vault-vector-store.ts
|
||||
|
||||
import { ChromaClient, Collection } from 'chromadb';
|
||||
import { TFile } from 'obsidian';
|
||||
import { VaultIndexConfig, VaultIndexEntry } from './types';
|
||||
import { Logger } from './utils';
|
||||
import { ContentVectorizer } from './indexing-pipeline/vectorization';
|
||||
import { ContentExtractor } from './indexing-pipeline/extraction';
|
||||
import { ContentNormalizer } from './indexing-pipeline/normalization';
|
||||
|
||||
export class VaultVectorStore {
|
||||
private client: ChromaClient | null = null;
|
||||
private collection: Collection | null = null;
|
||||
private config: VaultIndexConfig;
|
||||
private ollamaURL: string;
|
||||
private vectorizer: ContentVectorizer;
|
||||
private extractor: ContentExtractor;
|
||||
private normalizer: ContentNormalizer;
|
||||
private isInitialized = false;
|
||||
|
||||
constructor(ollamaURL: string, config: VaultIndexConfig) {
|
||||
this.ollamaURL = ollamaURL.replace(/\/+$/, '');
|
||||
this.config = config;
|
||||
this.vectorizer = new ContentVectorizer({
|
||||
model: config.embeddingModel,
|
||||
ollamaUrl: this.ollamaURL,
|
||||
});
|
||||
this.extractor = new ContentExtractor();
|
||||
this.normalizer = new ContentNormalizer();
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!this.config.enabled || this.isInitialized) return;
|
||||
|
||||
try {
|
||||
const rawURL = this.config.chromaURL?.trim() || 'http://localhost:8000';
|
||||
const chromaURL = rawURL.includes('://') ? rawURL : 'http://localhost:8000';
|
||||
this.client = new ChromaClient({ path: chromaURL });
|
||||
this.collection = await this.client.getOrCreateCollection({
|
||||
name: this.config.collectionName,
|
||||
metadata: { 'hnsw:space': 'cosine' },
|
||||
});
|
||||
this.isInitialized = true;
|
||||
Logger.info(
|
||||
`Vault vector store initialized: ${this.config.collectionName}`,
|
||||
'vault-vector-store'
|
||||
);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.error(
|
||||
`Failed to initialize vault vector store: ${errorMessage}`,
|
||||
'vault-vector-store'
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a single vault file by generating an embedding and storing it in ChromaDB.
|
||||
*/
|
||||
async indexFile(file: TFile, content: string): Promise<void> {
|
||||
if (!this.collection || !this.config.enabled) return;
|
||||
if (!content.trim()) {
|
||||
// Remove empty files from index if they exist
|
||||
await this.deleteFile(file.path);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const extracted = this.extractor.extractFromFile(
|
||||
{ basename: file.basename, path: file.path },
|
||||
content
|
||||
);
|
||||
const normalized = this.normalizer.normalize(extracted);
|
||||
|
||||
const chunk = {
|
||||
id: file.path,
|
||||
path: file.path,
|
||||
title: normalized.title,
|
||||
content: normalized.content,
|
||||
tokens: normalized.tokens,
|
||||
headings: normalized.headings,
|
||||
frontmatter: normalized.frontmatter,
|
||||
firstParagraph: normalized.firstParagraph,
|
||||
wordCount: normalized.wordCount,
|
||||
chunkIndex: 0,
|
||||
chunkSize: 1,
|
||||
};
|
||||
|
||||
const embedding = await this.vectorizer.vectorize(chunk);
|
||||
if (embedding.length === 0) {
|
||||
Logger.warn(`Empty embedding for ${file.path}, skipping index`, 'vault-vector-store');
|
||||
return;
|
||||
}
|
||||
|
||||
// Upsert by path so re-indexing updates rather than duplicates
|
||||
await this.collection.upsert({
|
||||
ids: [file.path],
|
||||
documents: [this.createDocumentText(normalized)],
|
||||
embeddings: [embedding],
|
||||
metadatas: [
|
||||
{
|
||||
path: file.path,
|
||||
title: normalized.title,
|
||||
tags:
|
||||
typeof normalized.frontmatter.tags === 'string' ? normalized.frontmatter.tags : '',
|
||||
},
|
||||
],
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(`Failed to index file ${file.path}: ${errorMessage}`, 'vault-vector-store');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a file from the vector index.
|
||||
*/
|
||||
async deleteFile(path: string): Promise<void> {
|
||||
if (!this.collection || !this.config.enabled) return;
|
||||
|
||||
try {
|
||||
await this.collection.delete({ ids: [path] });
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(
|
||||
`Failed to delete file ${path} from index: ${errorMessage}`,
|
||||
'vault-vector-store'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform semantic search against the vault index.
|
||||
*/
|
||||
async search(query: string, limit = 3): Promise<VaultIndexEntry[]> {
|
||||
if (!this.collection || !this.config.enabled) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const queryEmbedding = await this.vectorizer.vectorize({
|
||||
id: 'query',
|
||||
path: 'query',
|
||||
title: query,
|
||||
content: query,
|
||||
tokens: query
|
||||
.toLowerCase()
|
||||
.split(/\s+/)
|
||||
.filter((t) => t.length > 1),
|
||||
headings: [query],
|
||||
frontmatter: {},
|
||||
firstParagraph: query,
|
||||
wordCount: query.split(/\s+/).length,
|
||||
chunkIndex: 0,
|
||||
chunkSize: 1,
|
||||
});
|
||||
|
||||
if (queryEmbedding.length === 0) {
|
||||
Logger.warn('Empty query embedding, skipping semantic search', 'vault-vector-store');
|
||||
return [];
|
||||
}
|
||||
|
||||
const results = await this.collection.query({
|
||||
queryEmbeddings: [queryEmbedding],
|
||||
nResults: limit,
|
||||
});
|
||||
|
||||
if (!results.ids[0] || results.ids[0].length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const entries: VaultIndexEntry[] = [];
|
||||
for (let i = 0; i < results.ids[0].length; i++) {
|
||||
const distance = results.distances?.[0]?.[i] ?? 0;
|
||||
// Cosine distance to score: closer to 1 is better
|
||||
const score = 1 - distance;
|
||||
if (score < this.config.similarityThreshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const metadata = results.metadatas?.[0]?.[i] as Record<string, string> | undefined;
|
||||
const document = results.documents?.[0]?.[i] as string | undefined;
|
||||
|
||||
entries.push({
|
||||
path: metadata?.path || String(results.ids[0][i]),
|
||||
title: metadata?.title || 'Untitled',
|
||||
content: document || '',
|
||||
score,
|
||||
tags: metadata?.tags,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by score descending
|
||||
entries.sort((a, b) => b.score - a.score);
|
||||
return entries.slice(0, limit);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(`Semantic search failed: ${errorMessage}`, 'vault-vector-store');
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the entire vault index.
|
||||
*/
|
||||
async clearIndex(): Promise<void> {
|
||||
if (!this.client || !this.config.enabled) return;
|
||||
|
||||
try {
|
||||
await this.client.deleteCollection({ name: this.config.collectionName });
|
||||
this.collection = null;
|
||||
this.isInitialized = false;
|
||||
Logger.info('Vault index cleared', 'vault-vector-store');
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.error(`Failed to clear vault index: ${errorMessage}`, 'vault-vector-store');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of indexed documents.
|
||||
*/
|
||||
async getIndexedCount(): Promise<number> {
|
||||
if (!this.collection || !this.config.enabled) return 0;
|
||||
try {
|
||||
const result = await this.collection.count();
|
||||
return result;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private createDocumentText(normalized: ReturnType<ContentNormalizer['normalize']>): string {
|
||||
const parts: string[] = [];
|
||||
if (normalized.title) parts.push(`Title: ${normalized.title}`);
|
||||
if (normalized.firstParagraph) parts.push(normalized.firstParagraph);
|
||||
if (normalized.headings.length > 0) parts.push(normalized.headings.join('\n'));
|
||||
parts.push(normalized.content.substring(0, 1000));
|
||||
return parts.join('\n\n');
|
||||
}
|
||||
}
|
||||
@@ -422,17 +422,17 @@ Rules:
|
||||
// Apply tag filter if specified
|
||||
const filtered = config.tagFilter
|
||||
? entries.filter((entry) => {
|
||||
const tags = entry.frontmatter?.tags ?? '';
|
||||
const tags = entry.tags ?? '';
|
||||
return tags.toLowerCase().includes(config.tagFilter!.toLowerCase());
|
||||
})
|
||||
: entries;
|
||||
|
||||
return filtered.map((entry) => ({
|
||||
path: entry.file.path,
|
||||
path: entry.path,
|
||||
title: entry.title,
|
||||
content: entry.content,
|
||||
score: entry.score,
|
||||
tags: entry.frontmatter?.tags,
|
||||
tags: entry.tags,
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
@@ -39,6 +39,13 @@ const mockSettings: PluginSettings = {
|
||||
embeddingModel: 'nomic-embed-text',
|
||||
chromaURL: 'http://localhost:8000',
|
||||
},
|
||||
vaultIndexConfig: {
|
||||
enabled: false,
|
||||
similarityThreshold: 0.75,
|
||||
collectionName: 'test-vault-index',
|
||||
embeddingModel: 'nomic-embed-text',
|
||||
chromaURL: 'http://localhost:8000',
|
||||
},
|
||||
};
|
||||
|
||||
describe('ChatView', () => {
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
import { ChromaClient } from 'chromadb';
|
||||
import { VaultIndexConfig } from '../src/types';
|
||||
|
||||
// Mock ChromaDB module
|
||||
jest.mock('chromadb', () => ({
|
||||
ChromaClient: jest.fn().mockImplementation(() => {
|
||||
return {
|
||||
getOrCreateCollection: jest.fn().mockResolvedValue({
|
||||
query: jest.fn(),
|
||||
upsert: jest.fn(),
|
||||
delete: jest.fn(),
|
||||
count: jest.fn().mockResolvedValue(5),
|
||||
}),
|
||||
deleteCollection: jest.fn(),
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
import { VaultVectorStore } from '../src/vault-vector-store';
|
||||
|
||||
describe('VaultVectorStore', () => {
|
||||
const mockOllamaUrl = 'http://localhost:11434';
|
||||
const mockConfig: VaultIndexConfig = {
|
||||
enabled: true,
|
||||
similarityThreshold: 0.75,
|
||||
collectionName: 'test_vault_index',
|
||||
embeddingModel: 'nomic-embed-text',
|
||||
chromaURL: 'http://localhost:8000',
|
||||
};
|
||||
|
||||
let store: VaultVectorStore;
|
||||
let mockChromaClient: any;
|
||||
let mockCollection: any;
|
||||
|
||||
beforeEach(async () => {
|
||||
jest.clearAllMocks();
|
||||
global.fetch = jest.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ embedding: [0.1, 0.2, 0.3] }),
|
||||
});
|
||||
|
||||
store = new VaultVectorStore(mockOllamaUrl, mockConfig);
|
||||
await store.initialize();
|
||||
|
||||
mockChromaClient = (ChromaClient as jest.Mock).mock.results[0].value;
|
||||
mockCollection = await mockChromaClient.getOrCreateCollection.mock.results[0].value;
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should initialize with provided config', () => {
|
||||
expect(store).toBeInstanceOf(VaultVectorStore);
|
||||
});
|
||||
});
|
||||
|
||||
describe('initialize', () => {
|
||||
it('should initialize the collection', async () => {
|
||||
expect(mockChromaClient.getOrCreateCollection).toHaveBeenCalledWith({
|
||||
name: mockConfig.collectionName,
|
||||
metadata: { 'hnsw:space': 'cosine' },
|
||||
});
|
||||
});
|
||||
|
||||
it('should not initialize when disabled', async () => {
|
||||
jest.clearAllMocks();
|
||||
const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false };
|
||||
const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig);
|
||||
await disabledStore.initialize();
|
||||
expect(mockChromaClient.getOrCreateCollection).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('indexFile', () => {
|
||||
it('should upsert a file into the collection', async () => {
|
||||
const mockFile = {
|
||||
basename: 'test.md',
|
||||
path: 'test.md',
|
||||
extension: 'md',
|
||||
} as any;
|
||||
|
||||
await store.indexFile(mockFile, '# Test\n\nThis is test content.');
|
||||
|
||||
expect(mockCollection.upsert).toHaveBeenCalled();
|
||||
const upsertCall = mockCollection.upsert.mock.calls[0][0];
|
||||
expect(upsertCall.ids).toContain('test.md');
|
||||
expect(upsertCall.metadatas[0].title).toBe('test');
|
||||
});
|
||||
|
||||
it('should delete file from index when content is empty', async () => {
|
||||
const mockFile = {
|
||||
basename: 'empty.md',
|
||||
path: 'empty.md',
|
||||
extension: 'md',
|
||||
} as any;
|
||||
|
||||
await store.indexFile(mockFile, ' ');
|
||||
expect(mockCollection.delete).toHaveBeenCalledWith({ ids: ['empty.md'] });
|
||||
});
|
||||
|
||||
it('should not index when collection is null', async () => {
|
||||
jest.clearAllMocks();
|
||||
const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false };
|
||||
const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig);
|
||||
await disabledStore.initialize();
|
||||
|
||||
const mockFile = { basename: 'test.md', path: 'test.md' } as any;
|
||||
await disabledStore.indexFile(mockFile, 'content');
|
||||
expect(mockCollection.upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteFile', () => {
|
||||
it('should delete a file from the collection', async () => {
|
||||
await store.deleteFile('test.md');
|
||||
expect(mockCollection.delete).toHaveBeenCalledWith({ ids: ['test.md'] });
|
||||
});
|
||||
});
|
||||
|
||||
describe('search', () => {
|
||||
it('should return empty array when disabled', async () => {
|
||||
jest.clearAllMocks();
|
||||
const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false };
|
||||
const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig);
|
||||
await disabledStore.initialize();
|
||||
|
||||
const results = await disabledStore.search('test query', 3);
|
||||
expect(results).toEqual([]);
|
||||
expect(mockCollection.query).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return semantic search results', async () => {
|
||||
mockCollection.query.mockResolvedValue({
|
||||
ids: [['file1.md', 'file2.md']],
|
||||
documents: [['Doc 1 content', 'Doc 2 content']],
|
||||
distances: [[0.1, 0.15]],
|
||||
metadatas: [
|
||||
[
|
||||
{ path: 'file1.md', title: 'File 1' },
|
||||
{ path: 'file2.md', title: 'File 2' },
|
||||
],
|
||||
],
|
||||
});
|
||||
|
||||
const results = await store.search('test query', 2);
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].path).toBe('file1.md');
|
||||
expect(results[0].title).toBe('File 1');
|
||||
expect(results[0].score).toBe(0.9); // 1 - 0.1
|
||||
expect(results[1].score).toBe(0.85); // 1 - 0.15
|
||||
});
|
||||
|
||||
it('should filter results below similarity threshold', async () => {
|
||||
mockCollection.query.mockResolvedValue({
|
||||
ids: [['file1.md', 'file2.md']],
|
||||
documents: [['Doc 1', 'Doc 2']],
|
||||
distances: [[0.1, 0.5]], // scores: 0.9 and 0.5; threshold is 0.75
|
||||
metadatas: [
|
||||
[
|
||||
{ path: 'file1.md', title: 'File 1' },
|
||||
{ path: 'file2.md', title: 'File 2' },
|
||||
],
|
||||
],
|
||||
});
|
||||
|
||||
const results = await store.search('test', 2);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].path).toBe('file1.md');
|
||||
});
|
||||
|
||||
it('should return empty array when no results', async () => {
|
||||
mockCollection.query.mockResolvedValue({
|
||||
ids: [[]],
|
||||
documents: [[]],
|
||||
distances: [[]],
|
||||
metadatas: [[]],
|
||||
});
|
||||
|
||||
const results = await store.search('test', 3);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return empty array on query failure', async () => {
|
||||
mockCollection.query.mockRejectedValue(new Error('Query failed'));
|
||||
|
||||
const results = await store.search('test', 3);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('clearIndex', () => {
|
||||
it('should delete the collection', async () => {
|
||||
await store.clearIndex();
|
||||
expect(mockChromaClient.deleteCollection).toHaveBeenCalledWith({
|
||||
name: mockConfig.collectionName,
|
||||
});
|
||||
});
|
||||
|
||||
it('should not clear when disabled', async () => {
|
||||
jest.clearAllMocks();
|
||||
const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false };
|
||||
const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig);
|
||||
await disabledStore.clearIndex();
|
||||
expect(mockChromaClient.deleteCollection).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIndexedCount', () => {
|
||||
it('should return the collection count', async () => {
|
||||
const count = await store.getIndexedCount();
|
||||
expect(count).toBe(5);
|
||||
expect(mockCollection.count).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return 0 when disabled', async () => {
|
||||
jest.clearAllMocks();
|
||||
const disabledConfig: VaultIndexConfig = { ...mockConfig, enabled: false };
|
||||
const disabledStore = new VaultVectorStore(mockOllamaUrl, disabledConfig);
|
||||
const count = await disabledStore.getIndexedCount();
|
||||
expect(count).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user