fegger/obsidian_ollama
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: use collection deletion instead of reset, add embedding retry, slow down indexing

Browse Source 
- semantic-cache.clearCache: delete collection instead of client.reset()
  to avoid 403 Forbidden on newer ChromaDB versions
- vault-vector-store.clearIndex: same collection deletion approach
- ContentVectorizer: add 3-attempt retry with exponential backoff (1s, 2s, 4s)
- main.ts: reduce indexing batch size from 5 to 2, increase delay from 100ms to 500ms
- Update semantic-cache tests for deleteCollection
This commit is contained in:
Florian Egger
2026-05-19 23:41:05 +02:00
parent ce109cf6fb
commit cc97d77810
5 changed files with 104 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.js
+47 -24
View File
@@ -8004,7 +8004,7 @@ var SemanticCacheService = class _SemanticCacheService {
async clearCache() {
if (!this.config.enabled || !this.client) return;
try {
await this.client.reset();
await this.client.deleteCollection({ name: this.config.collectionName });
Logger.info("Semantic cache cleared", "semantic-cache");
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
@@ -9448,31 +9448,54 @@ var ContentVectorizer = class {
this.fetchFn = fetchFn ?? ((url, init) => fetch(url, init));
}
/**
* Generates embeddings for a content chunk
* Generates embeddings for a content chunk with retry logic
*/
async vectorize(chunk) {
try {
const prompt = this.createPrompt(chunk);
const response = await this.fetchFn(`${this.ollamaUrl}/api/embeddings`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model: this.model,
prompt
})
});
if (!response.ok) {
throw new Error(`Embedding failed with status ${response.status}`);
const prompt = this.createPrompt(chunk);
const maxRetries = 3;
const baseDelay = 1e3;
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
if (attempt > 0) {
const delay = baseDelay * Math.pow(2, attempt - 1);
Logger.info(
`Retrying embedding (attempt ${attempt + 1}/${maxRetries}) after ${delay}ms`,
"indexing-pipeline"
);
await new Promise((resolve) => setTimeout(resolve, delay));
}
const response = await this.fetchFn(`${this.ollamaUrl}/api/embeddings`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model: this.model,
prompt
})
});
if (!response.ok) {
throw new Error(`Embedding failed with status ${response.status}`);
}
const data = await response.json();
if (!this.isEmbeddingResponse(data)) {
throw new Error("Invalid embedding response");
}
return data.embedding;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(
`Embedding attempt ${attempt + 1} failed: ${errorMessage}`,
"indexing-pipeline"
);
if (attempt === maxRetries - 1) {
Logger.warn(
`Failed to generate embedding after ${maxRetries} attempts`,
"indexing-pipeline"
);
return [];
}
}
const data = await response.json();
if (!this.isEmbeddingResponse(data)) {
throw new Error("Invalid embedding response");
}
return data.embedding;
} catch (error) {
Logger.warn(`Failed to generate embedding: ${String(error)}`, "indexing-pipeline");
return [];
}
return [];
}
isEmbeddingResponse(data) {
return typeof data === "object" && data !== null && Array.isArray(data.embedding) && data.embedding.every((value) => typeof value === "number");
@@ -10010,8 +10033,8 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
const files = this.app.vault.getMarkdownFiles();
Logger.info(`Starting background vault indexing for ${files.length} files...`, "main");
let indexed = 0;
const BATCH_SIZE = 5;
const DELAY_MS = 100;
const BATCH_SIZE = 2;
const DELAY_MS = 500;
for (let i = 0; i < files.length; i += BATCH_SIZE) {
if (signal.aborted) {
Logger.info("Vault indexing cancelled.", "main");
src/indexing-pipeline/vectorization.ts
+48 -24
View File
@@ -23,36 +23,60 @@ export class ContentVectorizer {
}
/**
* Generates embeddings for a content chunk
* Generates embeddings for a content chunk with retry logic
*/
async vectorize(chunk: ContentChunk): Promise<number[]> {
try {
const prompt = this.createPrompt(chunk);
const prompt = this.createPrompt(chunk);
const maxRetries = 3;
const baseDelay = 1000;
const response = await this.fetchFn(`${this.ollamaUrl}/api/embeddings`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: this.model,
prompt: prompt,
}),
});
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
if (attempt > 0) {
const delay = baseDelay * Math.pow(2, attempt - 1);
Logger.info(
`Retrying embedding (attempt ${attempt + 1}/${maxRetries}) after ${delay}ms`,
'indexing-pipeline'
);
await new Promise((resolve) => setTimeout(resolve, delay));
}
if (!response.ok) {
throw new Error(`Embedding failed with status ${response.status}`);
const response = await this.fetchFn(`${this.ollamaUrl}/api/embeddings`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: this.model,
prompt: prompt,
}),
});
if (!response.ok) {
throw new Error(`Embedding failed with status ${response.status}`);
}
const data: unknown = await response.json();
if (!this.isEmbeddingResponse(data)) {
throw new Error('Invalid embedding response');
}
return data.embedding;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(
`Embedding attempt ${attempt + 1} failed: ${errorMessage}`,
'indexing-pipeline'
);
if (attempt === maxRetries - 1) {
Logger.warn(
`Failed to generate embedding after ${maxRetries} attempts`,
'indexing-pipeline'
);
return [];
}
}
const data: unknown = await response.json();
if (!this.isEmbeddingResponse(data)) {
throw new Error('Invalid embedding response');
}
return data.embedding;
} catch (error) {
// Return empty array on failure to maintain compatibility
Logger.warn(`Failed to generate embedding: ${String(error)}`, 'indexing-pipeline');
return [];
}
return [];
}
private isEmbeddingResponse(data: unknown): data is { embedding: number[] } {
src/main.ts
+3 -3
View File
@@ -166,8 +166,8 @@ export default class OllamaPlugin extends Plugin {
Logger.info(`Starting background vault indexing for ${files.length} files...`, 'main');
let indexed = 0;
const BATCH_SIZE = 5;
const DELAY_MS = 100;
const BATCH_SIZE = 2;
const DELAY_MS = 500;
for (let i = 0; i < files.length; i += BATCH_SIZE) {
if (signal.aborted) {
@@ -191,7 +191,7 @@ export default class OllamaPlugin extends Plugin {
})
);
// Small delay between batches to avoid overloading Ollama
// Delay between batches to avoid overloading Ollama
if (i + BATCH_SIZE < files.length) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
}
src/semantic-cache.ts
+1 -1
View File
@@ -92,7 +92,7 @@ export class SemanticCacheService {
if (!this.config.enabled || !this.client) return;
try {
await this.client.reset();
await this.client.deleteCollection({ name: this.config.collectionName });
Logger.info('Semantic cache cleared', 'semantic-cache');
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
tests/semantic-cache.test.ts
+5 -3
View File
@@ -155,13 +155,15 @@ describe('SemanticCacheService', () => {
await disabledCacheService.clearCache();
expect(mockChromaClient.reset).not.toHaveBeenCalled();
expect(mockChromaClient.deleteCollection).not.toHaveBeenCalled();
});
it('should clear the cache via the ChromaClient', async () => {
it('should clear the cache via deleteCollection', async () => {
await cacheService.clearCache();
expect(mockChromaClient.reset).toHaveBeenCalled();
expect(mockChromaClient.deleteCollection).toHaveBeenCalledWith({
name: mockCacheConfig.collectionName,
});
});
});
});