Add semantic caching and indexing pipeline

This commit adds semantic caching functionality to speed up repeated queries and implements a complete indexing pipeline for processing vault files. The changes include: - Added semantic cache service using ChromaDB for storing and retrieving cached responses - Implemented indexing pipeline with extraction, normalization, and vectorization steps - Added cache configuration settings to the plugin - Updated Ollama client to support cache integration - Added tests for all new indexing components - Extended vault indexer with indexing pipeline support
2026-05-07 22:23:35 +02:00
parent 47ecd3f803
commit d37b9f23bd
14 changed files with 899 additions and 42 deletions
@@ -0,0 +1,308 @@
+import { ContentExtractor } from '../src/indexing-pipeline/extraction';
+import { ContentNormalizer } from '../src/indexing-pipeline/normalization';
+import { ContentVectorizer } from '../src/indexing-pipeline/vectorization';
+import { IndexingPipeline } from '../src/indexing-pipeline/pipeline';
+
+// Mock VaultFile interface for testing
+interface MockVaultFile {
+  basename: string;
+  path: string;
+}
+
+describe('Indexing Pipeline Components', () => {
+  describe('ContentExtractor', () => {
+    let extractor: ContentExtractor;
+
+    beforeEach(() => {
+      extractor = new ContentExtractor();
+    });
+
+    it('should extract frontmatter correctly', () => {
+      const content = `---
+title: Test Title
+tags: algorithm, programming
+date: 2023-01-01
+---
+
+# Heading
+
+Content here`;
+
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+
+      expect(extracted.frontmatter.title).toBe('Test Title');
+      expect(extracted.frontmatter.tags).toBe('algorithm, programming');
+      expect(extracted.frontmatter.date).toBe('2023-01-01');
+      expect(extracted.headings).toContain('Heading');
+    });
+
+    it('should extract headings correctly', () => {
+      const content = `# Heading 1
+## Heading 2
+### Heading 3
+
+Content`;
+
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+
+      expect(extracted.headings).toEqual(['Heading 1', 'Heading 2', 'Heading 3']);
+    });
+
+    it('should extract embedded code blocks', () => {
+      const content = `# Code Example
+
+\`\`\`javascript
+console.log('hello world');
+\`\`\`
+
+Some content`;
+
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+
+      expect(extracted.embeddedCodeBlocks).toHaveLength(1);
+      expect(extracted.embeddedCodeBlocks[0]).toContain('console.log');
+    });
+
+    it('should extract first paragraph', () => {
+      const content = `First paragraph here.
+
+Second paragraph here.
+
+# Heading`;
+
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+
+      expect(extracted.firstParagraph).toBe('First paragraph here.');
+    });
+
+    it('should extract raw text correctly', () => {
+      const content = `---
+title: Test
+---
+
+# Heading
+
+Content with **bold** and [link](url).
+
+\`\`\`javascript
+code
+\`\`\``;
+
+      const rawText = extractor.extractRawText(content);
+      expect(rawText).not.toContain('---');
+      expect(rawText).not.toContain('# Heading');
+      expect(rawText).not.toContain('```javascript');
+      expect(rawText).toContain('Content with bold and link');
+    });
+  });
+
+  describe('ContentNormalizer', () => {
+    let normalizer: ContentNormalizer;
+    let extractor: ContentExtractor;
+
+    beforeEach(() => {
+      normalizer = new ContentNormalizer();
+      extractor = new ContentExtractor();
+    });
+
+    it('should normalize frontmatter dates to ISO format', () => {
+      const content = `---
+title: Test
+date: 2023-01-01
+created: 2023-06-15
+updated: invalid-date
+tags: algorithm
+---
+
+Content`;
+
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+      const normalized = normalizer.normalize(extracted);
+
+      expect(normalized.frontmatter.date).toBe('2023-01-01T00:00:00.000Z');
+      expect(normalized.frontmatter.created).toBe('2023-06-15T00:00:00.000Z');
+      expect(normalized.frontmatter.updated).toBe('invalid-date'); // Should preserve invalid dates
+    });
+
+    it('should convert tags to array format', () => {
+      const content = `---
+title: Test
+tags: algorithm, programming, javascript
+---
+
+Content`;
+
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+      const normalized = normalizer.normalize(extracted);
+
+      expect(normalized.frontmatter.tags).toEqual(['algorithm', 'programming', 'javascript']);
+    });
+
+    it('should calculate word count correctly', () => {
+      const content = `# Title
+
+This is a test document with several words to count.
+It has multiple sentences and words to make it longer.`;
+
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+      const normalized = normalizer.normalize(extracted);
+
+      expect(normalized.wordCount).toBeGreaterThan(0);
+    });
+
+    it('should extract tokens correctly', () => {
+      const content = `# Test Document
+
+This is a test document with important keywords.`;
+
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+      const normalized = normalizer.normalize(extracted);
+
+      expect(normalized.tokens).toContain('test');
+      expect(normalized.tokens).toContain('document');
+      expect(normalized.tokens).toContain('important');
+      expect(normalized.tokens).toContain('keywords');
+    });
+
+    it('should extract title correctly', () => {
+      const content = `# Test Document
+
+Content`;
+
+      const file: MockVaultFile = { basename: 'test.md', path: 'test.md' };
+      const extracted = extractor.extractFromFile(file, content);
+      const normalized = normalizer.normalize(extracted);
+
+      expect(normalized.title).toBe('test');
+    });
+  });
+
+  describe('ContentVectorizer', () => {
+    let vectorizer: ContentVectorizer;
+
+    beforeEach(() => {
+      vectorizer = new ContentVectorizer({
+        model: 'nomic-embed-text',
+        ollamaUrl: 'http://localhost:11434',
+      });
+    });
+
+    it('should create a proper prompt from content chunk', () => {
+      const mockChunk = {
+        id: 'test',
+        path: 'test.md',
+        title: 'Test',
+        content: 'Test content',
+        tokens: ['test', 'content'],
+        headings: ['Heading'],
+        frontmatter: { tags: ['test'] },
+        firstParagraph: 'First paragraph',
+        wordCount: 2,
+        chunkIndex: 0,
+        chunkSize: 100
+      };
+
+      const prompt = (vectorizer as any).createPrompt(mockChunk);
+
+      expect(prompt).toContain('Test');
+      expect(prompt).toContain('First paragraph');
+      expect(prompt).toContain('Heading');
+      expect(prompt).toContain('tags');
+    });
+
+    // Note: Actual embedding tests would require mocking fetch or integration testing
+    it('should handle vectorization errors gracefully', async () => {
+      // This test would require mocking fetch to simulate error responses
+      // For now, we're just ensuring the method exists and doesn't crash
+      const mockChunk = {
+        id: 'test',
+        path: 'test.md',
+        title: 'Test',
+        content: 'Test content',
+        tokens: ['test', 'content'],
+        headings: ['Heading'],
+        frontmatter: { tags: ['test'] },
+        firstParagraph: 'First paragraph',
+        wordCount: 2,
+        chunkIndex: 0,
+        chunkSize: 100
+      };
+
+      // Mock fetch to simulate an error
+      const originalFetch = global.fetch;
+      (global.fetch as any) = jest.fn().mockRejectedValue(new Error('Network error'));
+
+      try {
+        const result = await vectorizer.vectorize(mockChunk);
+        expect(result).toEqual([]);
+      } finally {
+        global.fetch = originalFetch;
+      }
+    });
+  });
+
+  describe('IndexingPipeline', () => {
+    let pipeline: IndexingPipeline;
+
+    beforeEach(() => {
+      pipeline = new IndexingPipeline({
+        ollamaUrl: 'http://localhost:11434',
+        embeddingModel: 'nomic-embed-text',
+      });
+    });
+
+    it('should process files through the pipeline', async () => {
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+      const content = `---
+title: Test Document
+tags: test, example
+---
+
+# Introduction
+
+This is a test document for pipeline processing.`;
+
+      const result = await pipeline.processFile(file, content);
+
+      expect(result).not.toBeNull();
+      expect(result?.title).toBe('test');
+      expect(result?.path).toBe('test.md');
+      expect(result?.content).toContain('This is a test document for pipeline processing');
+    });
+
+    it('should handle processing errors gracefully', async () => {
+      const file: MockVaultFile = { basename: 'test', path: 'test.md' };
+
+      const result = await pipeline.processFile(file, '');
+
+      // Should not crash, but might return null or incomplete result
+      expect(result).toBeNull(); // Empty content should return null
+    });
+
+    it('should process files in batches', async () => {
+      const files: MockVaultFile[] = [
+        { basename: 'file1', path: 'file1.md' },
+        { basename: 'file2', path: 'file2.md' }
+      ];
+
+      const fileContents = {
+        'file1.md': '# File 1\n\nContent 1',
+        'file2.md': '# File 2\n\nContent 2'
+      };
+
+      const results = await pipeline.processFilesInBatches(files, fileContents, 1);
+
+      expect(results).toHaveLength(2);
+      expect(results[0].title).toBe('file1');
+      expect(results[1].title).toBe('file2');
+    });
+  });
+});