fix: process embeddings sequentially, shorten prompts, fix tests

- main.ts: process files sequentially (not Promise.all) to avoid concurrent
  embedding requests hammering Ollama; batch size reduced to 1
- vectorization.ts: shorten embedding prompts from 1000 to 500 chars,
  limit headings to 5, remove frontmatter from prompt to stay well within
  embedding model context window
- Update vectorization and indexing-pipeline tests for new prompt format
This commit is contained in:
2026-05-19 23:51:29 +02:00
parent cc97d77810
commit cb621c83b5
5 changed files with 42 additions and 47 deletions
+3 -5
View File
@@ -200,8 +200,7 @@ describe('ContentVectorizer', () => {
expect(prompt).toContain('This is the first paragraph');
expect(prompt).toContain('Main Heading');
expect(prompt).toContain('Sub Heading');
expect(prompt).toContain('test');
expect(prompt).toContain('2024-01-01');
// Frontmatter is no longer included in embedding prompts
});
it('should handle empty content fields gracefully', () => {
@@ -221,8 +220,7 @@ describe('ContentVectorizer', () => {
const prompt = (vectorizer as any).createPrompt(chunk);
expect(prompt).toContain('Only content');
// JSON.stringify({}) produces "{}", which is truthy so it's included
expect(prompt).toContain('{}');
// Frontmatter is no longer included in embedding prompts
});
it('should limit content length', () => {
@@ -243,7 +241,7 @@ describe('ContentVectorizer', () => {
const prompt = (vectorizer as any).createPrompt(chunk);
expect(prompt).not.toContain('a'.repeat(1500));
expect(prompt).toContain('a'.repeat(1000));
expect(prompt).toContain('a'.repeat(500));
});
it('should handle missing frontmatter gracefully', () => {