fix: process embeddings sequentially, shorten prompts, fix tests

- main.ts: process files sequentially (not Promise.all) to avoid concurrent
  embedding requests hammering Ollama; batch size reduced to 1
- vectorization.ts: shorten embedding prompts from 1000 to 500 chars,
  limit headings to 5, remove frontmatter from prompt to stay well within
  embedding model context window
- Update vectorization and indexing-pipeline tests for new prompt format
This commit is contained in:
2026-05-19 23:51:29 +02:00
parent cc97d77810
commit cb621c83b5
5 changed files with 42 additions and 47 deletions
+16 -18
View File
@@ -9507,10 +9507,10 @@ var ContentVectorizer = class {
const parts = [
chunk.title,
chunk.firstParagraph,
chunk.content.substring(0, 1e3),
chunk.content.substring(0, 500),
// Limit content to avoid long prompts
chunk.headings.join(" "),
JSON.stringify(chunk.frontmatter)
chunk.headings.slice(0, 5).join(" ")
// Limit headings
].filter(Boolean);
return parts.join("\n\n");
}
@@ -10033,7 +10033,7 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
const files = this.app.vault.getMarkdownFiles();
Logger.info(`Starting background vault indexing for ${files.length} files...`, "main");
let indexed = 0;
const BATCH_SIZE = 2;
const BATCH_SIZE = 1;
const DELAY_MS = 500;
for (let i = 0; i < files.length; i += BATCH_SIZE) {
if (signal.aborted) {
@@ -10041,20 +10041,18 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
return;
}
const batch = files.slice(i, i + BATCH_SIZE);
await Promise.all(
batch.map(async (file) => {
if (signal.aborted) return;
try {
const content = await this.app.vault.read(file);
if (signal.aborted) return;
await this.vaultVectorStore.indexFile(file, content);
indexed++;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main");
}
})
);
for (const file of batch) {
if (signal.aborted) break;
try {
const content = await this.app.vault.read(file);
if (signal.aborted) break;
await this.vaultVectorStore.indexFile(file, content);
indexed++;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main");
}
}
if (i + BATCH_SIZE < files.length) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
}