fix: process embeddings sequentially, shorten prompts, fix tests
- main.ts: process files sequentially (not Promise.all) to avoid concurrent embedding requests hammering Ollama; batch size reduced to 1 - vectorization.ts: shorten embedding prompts from 1000 to 500 chars, limit headings to 5, remove frontmatter from prompt to stay well within embedding model context window - Update vectorization and indexing-pipeline tests for new prompt format
This commit is contained in:
@@ -200,8 +200,7 @@ describe('ContentVectorizer', () => {
|
||||
expect(prompt).toContain('This is the first paragraph');
|
||||
expect(prompt).toContain('Main Heading');
|
||||
expect(prompt).toContain('Sub Heading');
|
||||
expect(prompt).toContain('test');
|
||||
expect(prompt).toContain('2024-01-01');
|
||||
// Frontmatter is no longer included in embedding prompts
|
||||
});
|
||||
|
||||
it('should handle empty content fields gracefully', () => {
|
||||
@@ -221,8 +220,7 @@ describe('ContentVectorizer', () => {
|
||||
|
||||
const prompt = (vectorizer as any).createPrompt(chunk);
|
||||
expect(prompt).toContain('Only content');
|
||||
// JSON.stringify({}) produces "{}", which is truthy so it's included
|
||||
expect(prompt).toContain('{}');
|
||||
// Frontmatter is no longer included in embedding prompts
|
||||
});
|
||||
|
||||
it('should limit content length', () => {
|
||||
@@ -243,7 +241,7 @@ describe('ContentVectorizer', () => {
|
||||
|
||||
const prompt = (vectorizer as any).createPrompt(chunk);
|
||||
expect(prompt).not.toContain('a'.repeat(1500));
|
||||
expect(prompt).toContain('a'.repeat(1000));
|
||||
expect(prompt).toContain('a'.repeat(500));
|
||||
});
|
||||
|
||||
it('should handle missing frontmatter gracefully', () => {
|
||||
|
||||
Reference in New Issue
Block a user