import { ContentExtractor } from '../src/indexing-pipeline/extraction'; import { ContentNormalizer } from '../src/indexing-pipeline/normalization'; import { ContentVectorizer } from '../src/indexing-pipeline/vectorization'; import { IndexingPipeline } from '../src/indexing-pipeline/pipeline'; // Mock VaultFile interface for testing interface MockVaultFile { basename: string; path: string; } describe('Indexing Pipeline Components', () => { describe('ContentExtractor', () => { let extractor: ContentExtractor; beforeEach(() => { extractor = new ContentExtractor(); }); it('should extract frontmatter correctly', () => { const content = `--- title: Test Title tags: algorithm, programming date: 2023-01-01 --- # Heading Content here`; const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); expect(extracted.frontmatter.title).toBe('Test Title'); expect(extracted.frontmatter.tags).toBe('algorithm, programming'); expect(extracted.frontmatter.date).toBe('2023-01-01'); expect(extracted.headings).toContain('Heading'); }); it('should extract headings correctly', () => { const content = `# Heading 1 ## Heading 2 ### Heading 3 Content`; const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); expect(extracted.headings).toEqual(['Heading 1', 'Heading 2', 'Heading 3']); }); it('should extract embedded code blocks', () => { const content = `# Code Example \`\`\`javascript console.log('hello world'); \`\`\` Some content`; const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); expect(extracted.embeddedCodeBlocks).toHaveLength(1); expect(extracted.embeddedCodeBlocks[0]).toContain('console.log'); }); it('should extract first paragraph', () => { const content = `First paragraph here. Second paragraph here. # Heading`; const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); expect(extracted.firstParagraph).toBe('First paragraph here.'); }); it('should extract raw text correctly', () => { const content = `--- title: Test --- # Heading Content with **bold** and [link](url). \`\`\`javascript code \`\`\``; const rawText = extractor.extractRawText(content); expect(rawText).not.toContain('---'); expect(rawText).not.toContain('# Heading'); expect(rawText).not.toContain('```javascript'); expect(rawText).toContain('Content with bold and link'); }); }); describe('ContentNormalizer', () => { let normalizer: ContentNormalizer; let extractor: ContentExtractor; beforeEach(() => { normalizer = new ContentNormalizer(); extractor = new ContentExtractor(); }); it('should normalize frontmatter dates to ISO format', () => { const content = `--- title: Test date: 2023-01-01 created: 2023-06-15 updated: invalid-date tags: algorithm --- Content`; const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); const normalized = normalizer.normalize(extracted); expect(normalized.frontmatter.date).toBe('2023-01-01T00:00:00.000Z'); expect(normalized.frontmatter.created).toBe('2023-06-15T00:00:00.000Z'); expect(normalized.frontmatter.updated).toBe('invalid-date'); // Should preserve invalid dates }); it('should convert tags to array format', () => { const content = `--- title: Test tags: algorithm, programming, javascript --- Content`; const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); const normalized = normalizer.normalize(extracted); expect(normalized.frontmatter.tags).toEqual(['algorithm', 'programming', 'javascript']); }); it('should calculate word count correctly', () => { const content = `# Title This is a test document with several words to count. It has multiple sentences and words to make it longer.`; const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); const normalized = normalizer.normalize(extracted); expect(normalized.wordCount).toBeGreaterThan(0); }); it('should extract tokens correctly', () => { const content = `# Test Document This is a test document with important keywords.`; const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); const normalized = normalizer.normalize(extracted); expect(normalized.tokens).toContain('test'); expect(normalized.tokens).toContain('document'); expect(normalized.tokens).toContain('important'); expect(normalized.tokens).toContain('keywords'); }); it('should extract title correctly', () => { const content = `# Test Document Content`; const file: MockVaultFile = { basename: 'test.md', path: 'test.md' }; const extracted = extractor.extractFromFile(file, content); const normalized = normalizer.normalize(extracted); expect(normalized.title).toBe('test'); }); }); describe('ContentVectorizer', () => { let vectorizer: ContentVectorizer; beforeEach(() => { vectorizer = new ContentVectorizer({ model: 'nomic-embed-text', ollamaUrl: 'http://localhost:11434', }); }); it('should create a proper prompt from content chunk', () => { const mockChunk = { id: 'test', path: 'test.md', title: 'Test', content: 'Test content', tokens: ['test', 'content'], headings: ['Heading'], frontmatter: { tags: ['test'] }, firstParagraph: 'First paragraph', wordCount: 2, chunkIndex: 0, chunkSize: 100, }; const prompt = (vectorizer as any).createPrompt(mockChunk); expect(prompt).toContain('Test'); expect(prompt).toContain('First paragraph'); expect(prompt).toContain('Heading'); // Frontmatter is no longer included in embedding prompts }); // Note: Actual embedding tests would require mocking fetch or integration testing it('should handle vectorization errors gracefully', async () => { // This test would require mocking fetch to simulate error responses // For now, we're just ensuring the method exists and doesn't crash const mockChunk = { id: 'test', path: 'test.md', title: 'Test', content: 'Test content', tokens: ['test', 'content'], headings: ['Heading'], frontmatter: { tags: ['test'] }, firstParagraph: 'First paragraph', wordCount: 2, chunkIndex: 0, chunkSize: 100, }; // Mock fetch to simulate an error const originalFetch = global.fetch; (global.fetch as any) = jest.fn().mockRejectedValue(new Error('Network error')); try { const result = await vectorizer.vectorize(mockChunk); expect(result).toEqual([]); } finally { global.fetch = originalFetch; } }); }); describe('IndexingPipeline', () => { let pipeline: IndexingPipeline; beforeEach(() => { pipeline = new IndexingPipeline({ ollamaUrl: 'http://localhost:11434', embeddingModel: 'nomic-embed-text', }); }); it('should process files through the pipeline', async () => { const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const content = `--- title: Test Document tags: test, example --- # Introduction This is a test document for pipeline processing.`; const result = await pipeline.processFile(file, content); expect(result).not.toBeNull(); expect(result?.title).toBe('test'); expect(result?.path).toBe('test.md'); expect(result?.content).toContain('This is a test document for pipeline processing'); }); it('should handle processing errors gracefully', async () => { const file: MockVaultFile = { basename: 'test', path: 'test.md' }; const result = await pipeline.processFile(file, ''); // Should not crash, but might return null or incomplete result expect(result).toBeNull(); // Empty content should return null }); it('should process files in batches', async () => { const files: MockVaultFile[] = [ { basename: 'file1', path: 'file1.md' }, { basename: 'file2', path: 'file2.md' }, ]; const fileContents = { 'file1.md': '# File 1\n\nContent 1', 'file2.md': '# File 2\n\nContent 2', }; const results = await pipeline.processFilesInBatches(files, fileContents, 1); expect(results).toHaveLength(2); expect(results[0].title).toBe('file1'); expect(results[1].title).toBe('file2'); }); }); });