fix: process embeddings sequentially, shorten prompts, fix tests
- main.ts: process files sequentially (not Promise.all) to avoid concurrent embedding requests hammering Ollama; batch size reduced to 1 - vectorization.ts: shorten embedding prompts from 1000 to 500 chars, limit headings to 5, remove frontmatter from prompt to stay well within embedding model context window - Update vectorization and indexing-pipeline tests for new prompt format
This commit is contained in:
@@ -9507,10 +9507,10 @@ var ContentVectorizer = class {
|
||||
const parts = [
|
||||
chunk.title,
|
||||
chunk.firstParagraph,
|
||||
chunk.content.substring(0, 1e3),
|
||||
chunk.content.substring(0, 500),
|
||||
// Limit content to avoid long prompts
|
||||
chunk.headings.join(" "),
|
||||
JSON.stringify(chunk.frontmatter)
|
||||
chunk.headings.slice(0, 5).join(" ")
|
||||
// Limit headings
|
||||
].filter(Boolean);
|
||||
return parts.join("\n\n");
|
||||
}
|
||||
@@ -10033,7 +10033,7 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
|
||||
const files = this.app.vault.getMarkdownFiles();
|
||||
Logger.info(`Starting background vault indexing for ${files.length} files...`, "main");
|
||||
let indexed = 0;
|
||||
const BATCH_SIZE = 2;
|
||||
const BATCH_SIZE = 1;
|
||||
const DELAY_MS = 500;
|
||||
for (let i = 0; i < files.length; i += BATCH_SIZE) {
|
||||
if (signal.aborted) {
|
||||
@@ -10041,20 +10041,18 @@ var OllamaPlugin = class extends import_obsidian4.Plugin {
|
||||
return;
|
||||
}
|
||||
const batch = files.slice(i, i + BATCH_SIZE);
|
||||
await Promise.all(
|
||||
batch.map(async (file) => {
|
||||
if (signal.aborted) return;
|
||||
try {
|
||||
const content = await this.app.vault.read(file);
|
||||
if (signal.aborted) return;
|
||||
await this.vaultVectorStore.indexFile(file, content);
|
||||
indexed++;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main");
|
||||
}
|
||||
})
|
||||
);
|
||||
for (const file of batch) {
|
||||
if (signal.aborted) break;
|
||||
try {
|
||||
const content = await this.app.vault.read(file);
|
||||
if (signal.aborted) break;
|
||||
await this.vaultVectorStore.indexFile(file, content);
|
||||
indexed++;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
Logger.warn(`Failed to index ${file.path}: ${errorMessage}`, "main");
|
||||
}
|
||||
}
|
||||
if (i + BATCH_SIZE < files.length) {
|
||||
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user