fegger/obsidian_ollama
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: prevent race conditions during vault index rebuild and rate-limit embeddings

Browse Source 
- Add AbortController to cancel ongoing indexing before clear/rebuild
- Track currentIndexingPromise to await cancellation before clearing collection
- Batch background indexing (5 files at a time) with 100ms delays between batches
- Skip incremental event-based indexing while a full rebuild is in progress
- Cancel indexing on plugin unload
- Fix nginx CORS headers on OPTIONS preflight responses
This commit is contained in:
Florian Egger
2026-05-19 23:29:34 +02:00
parent fae74ade95
commit ce109cf6fb
3 changed files with 170 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.js
+71 -17
View File
@@ -9958,6 +9958,7 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
}
// eslint-disable-next-line @typescript-eslint/no-misused-promises
onunload() {
this.cancelBackgroundIndexing();
if (this.semanticCache) {
void this.semanticCache.clearCache();
}
@@ -9970,45 +9971,92 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
await this.saveData(this.settings);
}
async initializeVaultVectorStore() {
this.cancelBackgroundIndexing();
await this.awaitBackgroundIndexing();
this.vaultVectorStore = new VaultVectorStore(
this.settings.ollamaUrl,
this.settings.vaultIndexConfig
);
try {
await this.vaultVectorStore.initialize();
void this.backgroundIndexVault().catch((err) => {
this.currentIndexingPromise = this.backgroundIndexVault();
void this.currentIndexingPromise.catch((err) => {
Logger.warn(`Background vault indexing failed: ${String(err)}`, "main");
}).finally(() => {
this.currentIndexingPromise = void 0;
this.indexingAbortController = void 0;
});
} catch {
new import_obsidian4.Notice("Vault vector store initialization failed. Check console for details.");
}
}
cancelBackgroundIndexing() {
if (this.indexingAbortController) {
this.indexingAbortController.abort();
}
}
async awaitBackgroundIndexing() {
if (this.currentIndexingPromise) {
try {
await this.currentIndexingPromise;
} catch {
}
}
}
async backgroundIndexVault() {
if (!this.vaultVectorStore) return;
this.indexingAbortController = new AbortController();
const signal = this.indexingAbortController.signal;
const files = this.app.vault.getMarkdownFiles();
Logger.info(`Starting background vault indexing for ${files.length} files...`, "main");
let indexed = 0;
for (const file of files) {
try {
const content = await this.app.vault.read(file);
await this.vaultVectorStore.indexFile(file, content);
indexed++;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main");
const BATCH_SIZE = 5;
const DELAY_MS = 100;
for (let i = 0; i < files.length; i += BATCH_SIZE) {
if (signal.aborted) {
Logger.info("Vault indexing cancelled.", "main");
return;
}
const batch = files.slice(i, i + BATCH_SIZE);
await Promise.all(
batch.map(async (file) => {
if (signal.aborted) return;
try {
const content = await this.app.vault.read(file);
if (signal.aborted) return;
await this.vaultVectorStore.indexFile(file, content);
indexed++;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main");
}
})
);
if (i + BATCH_SIZE < files.length) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
}
}
this.settings.lastIndexTime = Date.now();
await this.saveSettings();
Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, "main");
new import_obsidian4.Notice(`Vault index updated: ${indexed} files indexed.`);
if (!signal.aborted) {
this.settings.lastIndexTime = Date.now();
await this.saveSettings();
Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, "main");
new import_obsidian4.Notice(`Vault index updated: ${indexed} files indexed.`);
}
}
async rebuildVaultIndex() {
this.cancelBackgroundIndexing();
await this.awaitBackgroundIndexing();
if (this.vaultVectorStore) {
await this.vaultVectorStore.clearIndex();
await this.vaultVectorStore.initialize();
}
await this.backgroundIndexVault();
this.currentIndexingPromise = this.backgroundIndexVault();
void this.currentIndexingPromise.catch((err) => {
Logger.warn(`Rebuild vault indexing failed: ${String(err)}`, "main");
}).finally(() => {
this.currentIndexingPromise = void 0;
this.indexingAbortController = void 0;
});
}
async clearVaultIndex() {
if (this.vaultVectorStore) {
@@ -10020,7 +10068,9 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
this.app.vault.on("create", (file) => {
if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) {
void this.app.vault.read(file).then((content) => {
void this.vaultVectorStore?.indexFile(file, content);
if (!this.currentIndexingPromise) {
void this.vaultVectorStore?.indexFile(file, content);
}
});
}
})
@@ -10029,7 +10079,9 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
this.app.vault.on("modify", (file) => {
if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) {
void this.app.vault.read(file).then((content) => {
void this.vaultVectorStore?.indexFile(file, content);
if (!this.currentIndexingPromise) {
void this.vaultVectorStore?.indexFile(file, content);
}
});
}
})
@@ -10046,7 +10098,9 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
if (file instanceof import_obsidian4.TFile && file.extension === "md" && this.vaultVectorStore) {
void this.vaultVectorStore.deleteFile(oldPath);
void this.app.vault.read(file).then((content) => {
void this.vaultVectorStore?.indexFile(file, content);
if (!this.currentIndexingPromise) {
void this.vaultVectorStore?.indexFile(file, content);
}
});
}
})
nginx.conf
+4 -1
View File
@@ -8,7 +8,7 @@ http {
server_name localhost;
location / {
# CORS headers for every response (including 4xx/5xx from upstream)
# CORS headers must be set for every response, including OPTIONS preflight
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
@@ -16,6 +16,9 @@ http {
# Preflight OPTIONS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain; charset=utf-8';
add_header 'Content-Length' 0;
src/main.ts
+95 -19
View File
@@ -10,6 +10,8 @@ export default class OllamaPlugin extends Plugin {
settings: PluginSettings = DEFAULT_SETTINGS;
semanticCache?: SemanticCacheService;
vaultVectorStore?: VaultVectorStore;
private indexingAbortController?: AbortController;
private currentIndexingPromise?: Promise<void>;
async onload() {
await this.loadSettings();
@@ -92,6 +94,9 @@ export default class OllamaPlugin extends Plugin {
// eslint-disable-next-line @typescript-eslint/no-misused-promises
onunload() {
// Cancel any ongoing indexing
this.cancelBackgroundIndexing();
// Clean up any active semantic cache resources on plugin unload
// Using fire-and-forget pattern since onunload cannot be async per Obsidian API
if (this.semanticCache) {
@@ -110,6 +115,10 @@ export default class OllamaPlugin extends Plugin {
}
async initializeVaultVectorStore(): Promise<void> {
// Abort any ongoing indexing before re-initializing
this.cancelBackgroundIndexing();
await this.awaitBackgroundIndexing();
this.vaultVectorStore = new VaultVectorStore(
this.settings.ollamaUrl,
this.settings.vaultIndexConfig
@@ -117,44 +126,104 @@ export default class OllamaPlugin extends Plugin {
try {
await this.vaultVectorStore.initialize();
// Run background indexing
void this.backgroundIndexVault().catch((err) => {
Logger.warn(`Background vault indexing failed: ${String(err)}`, 'main');
});
this.currentIndexingPromise = this.backgroundIndexVault();
void this.currentIndexingPromise
.catch((err) => {
Logger.warn(`Background vault indexing failed: ${String(err)}`, 'main');
})
.finally(() => {
this.currentIndexingPromise = undefined;
this.indexingAbortController = undefined;
});
} catch {
new Notice('Vault vector store initialization failed. Check console for details.');
}
}
private cancelBackgroundIndexing(): void {
if (this.indexingAbortController) {
this.indexingAbortController.abort();
}
}
private async awaitBackgroundIndexing(): Promise<void> {
if (this.currentIndexingPromise) {
try {
await this.currentIndexingPromise;
} catch {
// ignore errors from cancelled indexing
}
}
}
async backgroundIndexVault(): Promise<void> {
if (!this.vaultVectorStore) return;
this.indexingAbortController = new AbortController();
const signal = this.indexingAbortController.signal;
const files = this.app.vault.getMarkdownFiles();
Logger.info(`Starting background vault indexing for ${files.length} files...`, 'main');
let indexed = 0;
for (const file of files) {
try {
const content = await this.app.vault.read(file);
await this.vaultVectorStore.indexFile(file, content);
indexed++;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, 'main');
const BATCH_SIZE = 5;
const DELAY_MS = 100;
for (let i = 0; i < files.length; i += BATCH_SIZE) {
if (signal.aborted) {
Logger.info('Vault indexing cancelled.', 'main');
return;
}
const batch = files.slice(i, i + BATCH_SIZE);
await Promise.all(
batch.map(async (file) => {
if (signal.aborted) return;
try {
const content = await this.app.vault.read(file);
if (signal.aborted) return;
await this.vaultVectorStore!.indexFile(file, content);
indexed++;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, 'main');
}
})
);
// Small delay between batches to avoid overloading Ollama
if (i + BATCH_SIZE < files.length) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
}
}
this.settings.lastIndexTime = Date.now();
await this.saveSettings();
Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, 'main');
new Notice(`Vault index updated: ${indexed} files indexed.`);
if (!signal.aborted) {
this.settings.lastIndexTime = Date.now();
await this.saveSettings();
Logger.info(`Vault indexing complete: ${indexed}/${files.length} files indexed.`, 'main');
new Notice(`Vault index updated: ${indexed} files indexed.`);
}
}
async rebuildVaultIndex(): Promise<void> {
// Cancel and await any ongoing indexing before clearing
this.cancelBackgroundIndexing();
await this.awaitBackgroundIndexing();
if (this.vaultVectorStore) {
await this.vaultVectorStore.clearIndex();
await this.vaultVectorStore.initialize();
}
await this.backgroundIndexVault();
this.currentIndexingPromise = this.backgroundIndexVault();
void this.currentIndexingPromise
.catch((err) => {
Logger.warn(`Rebuild vault indexing failed: ${String(err)}`, 'main');
})
.finally(() => {
this.currentIndexingPromise = undefined;
this.indexingAbortController = undefined;
});
}
async clearVaultIndex(): Promise<void> {
@@ -169,7 +238,10 @@ export default class OllamaPlugin extends Plugin {
this.app.vault.on('create', (file) => {
if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) {
void this.app.vault.read(file).then((content) => {
void this.vaultVectorStore?.indexFile(file, content);
// Skip if a full rebuild is in progress to avoid race conditions
if (!this.currentIndexingPromise) {
void this.vaultVectorStore?.indexFile(file, content);
}
});
}
})
@@ -180,7 +252,9 @@ export default class OllamaPlugin extends Plugin {
this.app.vault.on('modify', (file) => {
if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) {
void this.app.vault.read(file).then((content) => {
void this.vaultVectorStore?.indexFile(file, content);
if (!this.currentIndexingPromise) {
void this.vaultVectorStore?.indexFile(file, content);
}
});
}
})
@@ -201,7 +275,9 @@ export default class OllamaPlugin extends Plugin {
if (file instanceof TFile && file.extension === 'md' && this.vaultVectorStore) {
void this.vaultVectorStore.deleteFile(oldPath);
void this.app.vault.read(file).then((content) => {
void this.vaultVectorStore?.indexFile(file, content);
if (!this.currentIndexingPromise) {
void this.vaultVectorStore?.indexFile(file, content);
}
});
}
})