fix: process embeddings sequentially, shorten prompts, fix tests
- main.ts: process files sequentially (not Promise.all) to avoid concurrent embedding requests hammering Ollama; batch size reduced to 1 - vectorization.ts: shorten embedding prompts from 1000 to 500 chars, limit headings to 5, remove frontmatter from prompt to stay well within embedding model context window - Update vectorization and indexing-pipeline tests for new prompt format
This commit is contained in:
@@ -207,7 +207,7 @@ Content`;
|
||||
firstParagraph: 'First paragraph',
|
||||
wordCount: 2,
|
||||
chunkIndex: 0,
|
||||
chunkSize: 100
|
||||
chunkSize: 100,
|
||||
};
|
||||
|
||||
const prompt = (vectorizer as any).createPrompt(mockChunk);
|
||||
@@ -215,7 +215,7 @@ Content`;
|
||||
expect(prompt).toContain('Test');
|
||||
expect(prompt).toContain('First paragraph');
|
||||
expect(prompt).toContain('Heading');
|
||||
expect(prompt).toContain('tags');
|
||||
// Frontmatter is no longer included in embedding prompts
|
||||
});
|
||||
|
||||
// Note: Actual embedding tests would require mocking fetch or integration testing
|
||||
@@ -233,7 +233,7 @@ Content`;
|
||||
firstParagraph: 'First paragraph',
|
||||
wordCount: 2,
|
||||
chunkIndex: 0,
|
||||
chunkSize: 100
|
||||
chunkSize: 100,
|
||||
};
|
||||
|
||||
// Mock fetch to simulate an error
|
||||
@@ -290,12 +290,12 @@ This is a test document for pipeline processing.`;
|
||||
it('should process files in batches', async () => {
|
||||
const files: MockVaultFile[] = [
|
||||
{ basename: 'file1', path: 'file1.md' },
|
||||
{ basename: 'file2', path: 'file2.md' }
|
||||
{ basename: 'file2', path: 'file2.md' },
|
||||
];
|
||||
|
||||
const fileContents = {
|
||||
'file1.md': '# File 1\n\nContent 1',
|
||||
'file2.md': '# File 2\n\nContent 2'
|
||||
'file2.md': '# File 2\n\nContent 2',
|
||||
};
|
||||
|
||||
const results = await pipeline.processFilesInBatches(files, fileContents, 1);
|
||||
|
||||
@@ -200,8 +200,7 @@ describe('ContentVectorizer', () => {
|
||||
expect(prompt).toContain('This is the first paragraph');
|
||||
expect(prompt).toContain('Main Heading');
|
||||
expect(prompt).toContain('Sub Heading');
|
||||
expect(prompt).toContain('test');
|
||||
expect(prompt).toContain('2024-01-01');
|
||||
// Frontmatter is no longer included in embedding prompts
|
||||
});
|
||||
|
||||
it('should handle empty content fields gracefully', () => {
|
||||
@@ -221,8 +220,7 @@ describe('ContentVectorizer', () => {
|
||||
|
||||
const prompt = (vectorizer as any).createPrompt(chunk);
|
||||
expect(prompt).toContain('Only content');
|
||||
// JSON.stringify({}) produces "{}", which is truthy so it's included
|
||||
expect(prompt).toContain('{}');
|
||||
// Frontmatter is no longer included in embedding prompts
|
||||
});
|
||||
|
||||
it('should limit content length', () => {
|
||||
@@ -243,7 +241,7 @@ describe('ContentVectorizer', () => {
|
||||
|
||||
const prompt = (vectorizer as any).createPrompt(chunk);
|
||||
expect(prompt).not.toContain('a'.repeat(1500));
|
||||
expect(prompt).toContain('a'.repeat(1000));
|
||||
expect(prompt).toContain('a'.repeat(500));
|
||||
});
|
||||
|
||||
it('should handle missing frontmatter gracefully', () => {
|
||||
|
||||
Reference in New Issue
Block a user