fix: process embeddings sequentially, shorten prompts, fix tests

- main.ts: process files sequentially (not Promise.all) to avoid concurrent
  embedding requests hammering Ollama; batch size reduced to 1
- vectorization.ts: shorten embedding prompts from 1000 to 500 chars,
  limit headings to 5, remove frontmatter from prompt to stay well within
  embedding model context window
- Update vectorization and indexing-pipeline tests for new prompt format
This commit is contained in:
2026-05-19 23:51:29 +02:00
parent cc97d77810
commit cb621c83b5
5 changed files with 42 additions and 47 deletions
+5 -5
View File
@@ -207,7 +207,7 @@ Content`;
firstParagraph: 'First paragraph',
wordCount: 2,
chunkIndex: 0,
chunkSize: 100
chunkSize: 100,
};
const prompt = (vectorizer as any).createPrompt(mockChunk);
@@ -215,7 +215,7 @@ Content`;
expect(prompt).toContain('Test');
expect(prompt).toContain('First paragraph');
expect(prompt).toContain('Heading');
expect(prompt).toContain('tags');
// Frontmatter is no longer included in embedding prompts
});
// Note: Actual embedding tests would require mocking fetch or integration testing
@@ -233,7 +233,7 @@ Content`;
firstParagraph: 'First paragraph',
wordCount: 2,
chunkIndex: 0,
chunkSize: 100
chunkSize: 100,
};
// Mock fetch to simulate an error
@@ -290,12 +290,12 @@ This is a test document for pipeline processing.`;
it('should process files in batches', async () => {
const files: MockVaultFile[] = [
{ basename: 'file1', path: 'file1.md' },
{ basename: 'file2', path: 'file2.md' }
{ basename: 'file2', path: 'file2.md' },
];
const fileContents = {
'file1.md': '# File 1\n\nContent 1',
'file2.md': '# File 2\n\nContent 2'
'file2.md': '# File 2\n\nContent 2',
};
const results = await pipeline.processFilesInBatches(files, fileContents, 1);
+3 -5
View File
@@ -200,8 +200,7 @@ describe('ContentVectorizer', () => {
expect(prompt).toContain('This is the first paragraph');
expect(prompt).toContain('Main Heading');
expect(prompt).toContain('Sub Heading');
expect(prompt).toContain('test');
expect(prompt).toContain('2024-01-01');
// Frontmatter is no longer included in embedding prompts
});
it('should handle empty content fields gracefully', () => {
@@ -221,8 +220,7 @@ describe('ContentVectorizer', () => {
const prompt = (vectorizer as any).createPrompt(chunk);
expect(prompt).toContain('Only content');
// JSON.stringify({}) produces "{}", which is truthy so it's included
expect(prompt).toContain('{}');
// Frontmatter is no longer included in embedding prompts
});
it('should limit content length', () => {
@@ -243,7 +241,7 @@ describe('ContentVectorizer', () => {
const prompt = (vectorizer as any).createPrompt(chunk);
expect(prompt).not.toContain('a'.repeat(1500));
expect(prompt).toContain('a'.repeat(1000));
expect(prompt).toContain('a'.repeat(500));
});
it('should handle missing frontmatter gracefully', () => {