From ce109cf6fbf372d414751716c3fd9caa47c98d28 Mon Sep 17 00:00:00 2001 From: Florian Egger Date: Tue, 19 May 2026 23:29:34 +0200 Subject: [PATCH] fix: prevent race conditions during vault index rebuild and rate-limit embeddings - Add AbortController to cancel ongoing indexing before clear/rebuild - Track currentIndexingPromise to await cancellation before clearing collection - Batch background indexing (5 files at a time) with 100ms delays between batches - Skip incremental event-based indexing while a full rebuild is in progress - Cancel indexing on plugin unload - Fix nginx CORS headers on OPTIONS preflight responses --- main.js | 88 ++++++++++++++++++++++++++++++++-------- nginx.conf | 5 ++- src/main.ts | 114 +++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 170 insertions(+), 37 deletions(-) diff --git a/main.js b/main.js index d33e6bc..95ac664 100644 --- a/main.js +++ b/main.js @@ -9958,6 +9958,7 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { } // eslint-disable-next-line @typescript-eslint/no-misused-promises onunload() { + this.cancelBackgroundIndexing(); if (this.semanticCache) { void this.semanticCache.clearCache(); } @@ -9970,45 +9971,92 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { await this.saveData(this.settings); } async initializeVaultVectorStore() { + this.cancelBackgroundIndexing(); + await this.awaitBackgroundIndexing(); this.vaultVectorStore = new VaultVectorStore( this.settings.ollamaUrl, this.settings.vaultIndexConfig ); try { await this.vaultVectorStore.initialize(); - void this.backgroundIndexVault().catch((err) => { + this.currentIndexingPromise = this.backgroundIndexVault(); + void this.currentIndexingPromise.catch((err) => { Logger.warn(`Background vault indexing failed: ${String(err)}`, "main"); + }).finally(() => { + this.currentIndexingPromise = void 0; + this.indexingAbortController = void 0; }); } catch { new import_obsidian4.Notice("Vault vector store initialization failed. Check console for details."); } } + cancelBackgroundIndexing() { + if (this.indexingAbortController) { + this.indexingAbortController.abort(); + } + } + async awaitBackgroundIndexing() { + if (this.currentIndexingPromise) { + try { + await this.currentIndexingPromise; + } catch { + } + } + } async backgroundIndexVault() { if (!this.vaultVectorStore) return; + this.indexingAbortController = new AbortController(); + const signal = this.indexingAbortController.signal; const files = this.app.vault.getMarkdownFiles(); Logger.info(`Starting background vault indexing for ${files.length} files...`, "main"); let indexed = 0; - for (const file of files) { - try { - const content = await this.app.vault.read(file); - await this.vaultVectorStore.indexFile(file, content); - indexed++; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main"); + const BATCH_SIZE = 5; + const DELAY_MS = 100; + for (let i = 0; i < files.length; i += BATCH_SIZE) { + if (signal.aborted) { + Logger.info("Vault indexing cancelled.", "main"); + return; + } + const batch = files.slice(i, i + BATCH_SIZE); + await Promise.all( + batch.map(async (file) => { + if (signal.aborted) return; + try { + const content = await this.app.vault.read(file); + if (signal.aborted) return; + await this.vaultVectorStore.indexFile(file, content); + indexed++; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main"); + } + }) + ); + if (i + BATCH_SIZE < files.length) { + await new Promise((resolve) => setTimeout(resolve, DELAY_MS)); } } - this.settings.lastIndexTime = Date.now(); - await this.saveSettings(); - Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, "main"); - new import_obsidian4.Notice(`Vault index updated: ${indexed} files indexed.`); + if (!signal.aborted) { + this.settings.lastIndexTime = Date.now(); + await this.saveSettings(); + Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, "main"); + new import_obsidian4.Notice(`Vault index updated: ${indexed} files indexed.`); + } } async rebuildVaultIndex() { + this.cancelBackgroundIndexing(); + await this.awaitBackgroundIndexing(); if (this.vaultVectorStore) { await this.vaultVectorStore.clearIndex(); await this.vaultVectorStore.initialize(); } - await this.backgroundIndexVault(); + this.currentIndexingPromise = this.backgroundIndexVault(); + void this.currentIndexingPromise.catch((err) => { + Logger.warn(`Rebuild vault indexing failed: ${String(err)}`, "main"); + }).finally(() => { + this.currentIndexingPromise = void 0; + this.indexingAbortController = void 0; + }); } async clearVaultIndex() { if (this.vaultVectorStore) { @@ -10020,7 +10068,9 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { this.app.vault.on("create", (file) => { if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) { void this.app.vault.read(file).then((content) => { - void this.vaultVectorStore?.indexFile(file, content); + if (!this.currentIndexingPromise) { + void this.vaultVectorStore?.indexFile(file, content); + } }); } }) @@ -10029,7 +10079,9 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { this.app.vault.on("modify", (file) => { if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) { void this.app.vault.read(file).then((content) => { - void this.vaultVectorStore?.indexFile(file, content); + if (!this.currentIndexingPromise) { + void this.vaultVectorStore?.indexFile(file, content); + } }); } }) @@ -10046,7 +10098,9 @@ var OllamaPlugin = class extends import_obsidian4.Plugin { if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) { void this.vaultVectorStore.deleteFile(oldPath); void this.app.vault.read(file).then((content) => { - void this.vaultVectorStore?.indexFile(file, content); + if (!this.currentIndexingPromise) { + void this.vaultVectorStore?.indexFile(file, content); + } }); } }) diff --git a/nginx.conf b/nginx.conf index 08c804c..e27c95f 100644 --- a/nginx.conf +++ b/nginx.conf @@ -8,7 +8,7 @@ http { server_name localhost; location / { - # CORS headers for every response (including 4xx/5xx from upstream) + # CORS headers must be set for every response, including OPTIONS preflight add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; @@ -16,6 +16,9 @@ http { # Preflight OPTIONS if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; add_header 'Access-Control-Max-Age' 1728000; add_header 'Content-Type' 'text/plain; charset=utf-8'; add_header 'Content-Length' 0; diff --git a/src/main.ts b/src/main.ts index c038a15..bd17e90 100755 --- a/src/main.ts +++ b/src/main.ts @@ -10,6 +10,8 @@ export default class OllamaPlugin extends Plugin { settings: PluginSettings = DEFAULT_SETTINGS; semanticCache?: SemanticCacheService; vaultVectorStore?: VaultVectorStore; + private indexingAbortController?: AbortController; + private currentIndexingPromise?: Promise; async onload() { await this.loadSettings(); @@ -92,6 +94,9 @@ export default class OllamaPlugin extends Plugin { // eslint-disable-next-line @typescript-eslint/no-misused-promises onunload() { + // Cancel any ongoing indexing + this.cancelBackgroundIndexing(); + // Clean up any active semantic cache resources on plugin unload // Using fire-and-forget pattern since onunload cannot be async per Obsidian API if (this.semanticCache) { @@ -110,6 +115,10 @@ export default class OllamaPlugin extends Plugin { } async initializeVaultVectorStore(): Promise { + // Abort any ongoing indexing before re-initializing + this.cancelBackgroundIndexing(); + await this.awaitBackgroundIndexing(); + this.vaultVectorStore = new VaultVectorStore( this.settings.ollamaUrl, this.settings.vaultIndexConfig @@ -117,44 +126,104 @@ export default class OllamaPlugin extends Plugin { try { await this.vaultVectorStore.initialize(); // Run background indexing - void this.backgroundIndexVault().catch((err) => { - Logger.warn(`Background vault indexing failed: ${String(err)}`, 'main'); - }); + this.currentIndexingPromise = this.backgroundIndexVault(); + void this.currentIndexingPromise + .catch((err) => { + Logger.warn(`Background vault indexing failed: ${String(err)}`, 'main'); + }) + .finally(() => { + this.currentIndexingPromise = undefined; + this.indexingAbortController = undefined; + }); } catch { new Notice('Vault vector store initialization failed. Check console for details.'); } } + private cancelBackgroundIndexing(): void { + if (this.indexingAbortController) { + this.indexingAbortController.abort(); + } + } + + private async awaitBackgroundIndexing(): Promise { + if (this.currentIndexingPromise) { + try { + await this.currentIndexingPromise; + } catch { + // ignore errors from cancelled indexing + } + } + } + async backgroundIndexVault(): Promise { if (!this.vaultVectorStore) return; + this.indexingAbortController = new AbortController(); + const signal = this.indexingAbortController.signal; + const files = this.app.vault.getMarkdownFiles(); Logger.info(`Starting background vault indexing for ${files.length} files...`, 'main'); let indexed = 0; - for (const file of files) { - try { - const content = await this.app.vault.read(file); - await this.vaultVectorStore.indexFile(file, content); - indexed++; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, 'main'); + const BATCH_SIZE = 5; + const DELAY_MS = 100; + + for (let i = 0; i < files.length; i += BATCH_SIZE) { + if (signal.aborted) { + Logger.info('Vault indexing cancelled.', 'main'); + return; + } + + const batch = files.slice(i, i + BATCH_SIZE); + await Promise.all( + batch.map(async (file) => { + if (signal.aborted) return; + try { + const content = await this.app.vault.read(file); + if (signal.aborted) return; + await this.vaultVectorStore!.indexFile(file, content); + indexed++; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, 'main'); + } + }) + ); + + // Small delay between batches to avoid overloading Ollama + if (i + BATCH_SIZE < files.length) { + await new Promise((resolve) => setTimeout(resolve, DELAY_MS)); } } - this.settings.lastIndexTime = Date.now(); - await this.saveSettings(); - Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, 'main'); - new Notice(`Vault index updated: ${indexed} files indexed.`); + if (!signal.aborted) { + this.settings.lastIndexTime = Date.now(); + await this.saveSettings(); + Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, 'main'); + new Notice(`Vault index updated: ${indexed} files indexed.`); + } } async rebuildVaultIndex(): Promise { + // Cancel and await any ongoing indexing before clearing + this.cancelBackgroundIndexing(); + await this.awaitBackgroundIndexing(); + if (this.vaultVectorStore) { await this.vaultVectorStore.clearIndex(); await this.vaultVectorStore.initialize(); } - await this.backgroundIndexVault(); + + this.currentIndexingPromise = this.backgroundIndexVault(); + void this.currentIndexingPromise + .catch((err) => { + Logger.warn(`Rebuild vault indexing failed: ${String(err)}`, 'main'); + }) + .finally(() => { + this.currentIndexingPromise = undefined; + this.indexingAbortController = undefined; + }); } async clearVaultIndex(): Promise { @@ -169,7 +238,10 @@ export default class OllamaPlugin extends Plugin { this.app.vault.on('create', (file) => { if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) { void this.app.vault.read(file).then((content) => { - void this.vaultVectorStore?.indexFile(file, content); + // Skip if a full rebuild is in progress to avoid race conditions + if (!this.currentIndexingPromise) { + void this.vaultVectorStore?.indexFile(file, content); + } }); } }) @@ -180,7 +252,9 @@ export default class OllamaPlugin extends Plugin { this.app.vault.on('modify', (file) => { if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) { void this.app.vault.read(file).then((content) => { - void this.vaultVectorStore?.indexFile(file, content); + if (!this.currentIndexingPromise) { + void this.vaultVectorStore?.indexFile(file, content); + } }); } }) @@ -201,7 +275,9 @@ export default class OllamaPlugin extends Plugin { if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) { void this.vaultVectorStore.deleteFile(oldPath); void this.app.vault.read(file).then((content) => { - void this.vaultVectorStore?.indexFile(file, content); + if (!this.currentIndexingPromise) { + void this.vaultVectorStore?.indexFile(file, content); + } }); } })