From 3c7c4d58bb8f23d219adeb2d7a1e530e0429fef2 Mon Sep 17 00:00:00 2001 From: Florian Egger Date: Wed, 20 May 2026 19:52:49 +0200 Subject: [PATCH] Add hybrid search with filtering and recency boost Introduces SearchOptions interface with folder/tag filters, exact phrase matching via quoted queries, and optional recency boosting with configurable half-life. Replaces pure semantic or keyword search with a combined scoring model: keyword scores are blended with semantic results, then adjusted by filters and recency. Adds mtime to vault index entries and expands test coverage for the new options. --- src/types.ts | 9 +++ src/vault-indexer.ts | 142 ++++++++++++++++++++++++++++++------ tests/vault-indexer.test.ts | 115 ++++++++++++++++++++++++++--- 3 files changed, 234 insertions(+), 32 deletions(-) diff --git a/src/types.ts b/src/types.ts index 6800ac4..b532637 100644 --- a/src/types.ts +++ b/src/types.ts @@ -157,6 +157,15 @@ export interface VaultIndexEntry { content: string; score: number; tags?: string; + mtime?: number; +} + +export interface SearchOptions { + folder?: string; + tag?: string; + includeExactPhrase?: boolean; + recencyBoost?: boolean; + recencyHalfLifeDays?: number; } export interface ChatMessage { diff --git a/src/vault-indexer.ts b/src/vault-indexer.ts index fa77218..46be145 100644 --- a/src/vault-indexer.ts +++ b/src/vault-indexer.ts @@ -4,7 +4,7 @@ import { Vault, TFile } from 'obsidian'; import { Logger } from './utils'; import { Cache } from './cache'; import { VaultVectorStore } from './vault-vector-store'; -import { VaultIndexEntry } from './types'; +import { VaultIndexEntry, SearchOptions } from './types'; interface ParsedFrontmatter { title?: string; @@ -105,6 +105,8 @@ const STOP_WORDS = new Set([ ]); const CONTENT_PREVIEW_LENGTH = 500; +const DAYS_TO_MS = 86400000; +const DEFAULT_RECENCY_HALF_LIFE = 30; // 30 days export class VaultIndexer { private vault: Vault; @@ -116,6 +118,10 @@ export class VaultIndexer { FRONTMATTER_TAGS: 3, HEADINGS: 2, CONTENT: 1, + FILENAME: 3, + EXACT_PHRASE: 8, + LINKED: 2, + RECENT: 0.5, // multiplier, not additive }; constructor(vault: Vault, cache?: Cache, vectorStore?: VaultVectorStore) { @@ -154,14 +160,30 @@ export class VaultIndexer { }; } - calculateWeightedScore(tokenized: TokenizedContent, queryTokens: string[]): { score: number } { + calculateWeightedScore( + tokenized: TokenizedContent, + queryTokens: string[], + exactPhrases: string[] + ): number { let score = 0; + const fullText = [ + tokenized.title, + tokenized.headings.join(' '), + tokenized.frontmatter.title ?? '', + tokenized.frontmatter.tags ?? '', + tokenized.content, + tokenized.firstParagraph, + tokenized.basename, + ] + .join(' ') + .toLowerCase(); + for (const token of queryTokens) { if (tokenized.frontmatter?.title && this.exactMatch(tokenized.frontmatter.title, token)) { score += this.SCORING_WEIGHTS.FRONTMATTER_TITLE; } if (tokenized.basename && this.exactMatch(tokenized.basename, token)) { - score += this.SCORING_WEIGHTS.FRONTMATTER_TITLE; + score += this.SCORING_WEIGHTS.FILENAME; } if (tokenized.frontmatter?.tags && this.exactMatch(tokenized.frontmatter.tags, token)) { score += this.SCORING_WEIGHTS.FRONTMATTER_TAGS; @@ -175,8 +197,19 @@ export class VaultIndexer { if (tokenized.title && this.exactMatch(tokenized.title, token)) { score += this.SCORING_WEIGHTS.TITLE; } + if (tokenized.firstParagraph.toLowerCase().includes(token.toLowerCase())) { + score += this.SCORING_WEIGHTS.CONTENT; + } } - return { score }; + + // Exact phrase bonus + for (const phrase of exactPhrases) { + if (fullText.includes(phrase.toLowerCase())) { + score += this.SCORING_WEIGHTS.EXACT_PHRASE; + } + } + + return score; } async getVaultEntries(): Promise { @@ -190,7 +223,7 @@ export class VaultIndexer { : await this.vault.read(file); const parsed = this.parseMarkdown(content); entries.push({ - file: file, + file, title: parsed.frontmatter.title || file.basename, frontmatter: parsed.frontmatter, headings: parsed.headings, @@ -206,29 +239,28 @@ export class VaultIndexer { return entries; } - async searchVault(query: string, limit = 3): Promise { + async searchVault(query: string, limit = 3, options?: SearchOptions): Promise { if (!query || !query.trim()) { return []; } - // Try semantic search first if vector store is available + const exactPhrases = + options?.includeExactPhrase !== false ? this.extractExactPhrases(query) : []; + const queryTokens = this.tokenize(query); + + // Try hybrid search: semantic + keyword + let semanticResults: VaultIndexEntry[] = []; if (this.vectorStore) { try { - const semanticResults = await this.vectorStore.search(query, limit); - if (semanticResults.length > 0) { - return semanticResults; - } + semanticResults = await this.vectorStore.search(query, limit * 3); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); - Logger.warn( - `Semantic search failed, falling back to keyword search: ${errorMessage}`, - 'vault-indexer' - ); + Logger.warn(`Semantic search failed: ${errorMessage}`, 'vault-indexer'); } } - const cacheKey = `query:${query.trim()}:limit:${limit}`; - if (this.cache) { + const cacheKey = this.buildCacheKey(query, limit, options); + if (this.cache && semanticResults.length === 0) { let cachedResults: string | null = null; try { cachedResults = await this.cache.get(cacheKey); @@ -246,15 +278,22 @@ export class VaultIndexer { } } - const queryTokens = this.tokenize(query); - if (queryTokens.length === 0) { + if (queryTokens.length === 0 && exactPhrases.length === 0) { + // Only non-token words (e.g. "a", "the") — try exact match fallback + if (semanticResults.length > 0) return semanticResults.slice(0, limit); return []; } const entries = await this.getVaultEntries(); + const now = Date.now(); + const halfLife = (options?.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE) * DAYS_TO_MS; + + // Precompute exact phrases lowercased + const lowerExactPhrases = exactPhrases.map((p) => p.toLowerCase()); + const scored = entries .map((entry) => { - const { score } = this.calculateWeightedScore( + const keywordScore = this.calculateWeightedScore( { title: entry.title, headings: entry.headings, @@ -263,11 +302,45 @@ export class VaultIndexer { content: entry.content, basename: entry.basename, }, - queryTokens + queryTokens, + lowerExactPhrases ); + + // Semantic score + const semanticEntry = semanticResults.find((s) => s.path === entry.file.path); + const semanticScore = semanticEntry ? (semanticEntry.score || 0) * 0.3 : 0; + + // Hybrid score: keyword dominates, semantic adds bonus + let score = keywordScore + semanticScore; + + // Folder filter: penalize non-matches + if (options?.folder) { + const folderLower = options.folder.toLowerCase().replace(/\/$/, ''); + const entryFolder = entry.file.path.toLowerCase().split('/').slice(0, -1).join('/'); + if (!entryFolder.startsWith(folderLower) && entryFolder !== folderLower) { + score *= 0.1; // Heavy penalty + } + } + + // Tag filter + if (options?.tag) { + const tagLower = options.tag.toLowerCase(); + const entryTags = (entry.frontmatter.tags ?? '').toLowerCase(); + if (!entryTags.includes(tagLower)) { + score *= 0.1; + } + } + + // Recency boost + if (options?.recencyBoost !== false && entry.file.stat?.mtime) { + const age = now - entry.file.stat.mtime; + const recencyMultiplier = 1 + this.SCORING_WEIGHTS.RECENT * Math.exp(-age / halfLife); + score *= recencyMultiplier; + } + return { ...entry, score }; }) - .filter((e) => e.score > 0); + .filter((e) => e.score > 0.01); scored.sort((a, b) => b.score - a.score); const results: VaultIndexEntry[] = scored.slice(0, limit).map((e) => ({ @@ -276,9 +349,10 @@ export class VaultIndexer { content: e.content, score: e.score, tags: e.frontmatter?.tags, + mtime: e.file.stat?.mtime, })); - if (this.cache) { + if (this.cache && semanticResults.length === 0) { try { await this.cache.put(cacheKey, JSON.stringify(results)); } catch (error) { @@ -293,6 +367,28 @@ export class VaultIndexer { return results; } + /** + * Extracts quoted exact phrases from a query. + */ + private extractExactPhrases(query: string): string[] { + const phrases: string[] = []; + const quoteRegex = /"([^"]+)"/g; + let match: RegExpExecArray | null; + while ((match = quoteRegex.exec(query)) !== null) { + phrases.push(match[1]); + } + return phrases; + } + + private buildCacheKey(query: string, limit: number, options?: SearchOptions): string { + const parts = [`query:${query.trim()}:limit:${limit}`]; + if (options?.folder) parts.push(`folder:${options.folder}`); + if (options?.tag) parts.push(`tag:${options.tag}`); + if (options?.recencyBoost === false) parts.push('norecency'); + if (options?.includeExactPhrase === false) parts.push('noexact'); + return parts.join(':'); + } + private stemToken(token: string): string { if (token.endsWith('ing') && token.length > 4) return token.slice(0, -3); if (token.endsWith('ed') && token.length > 3) return token.slice(0, -2); diff --git a/tests/vault-indexer.test.ts b/tests/vault-indexer.test.ts index f6431ed..15bd1d3 100755 --- a/tests/vault-indexer.test.ts +++ b/tests/vault-indexer.test.ts @@ -286,8 +286,8 @@ describe('VaultIndexer', () => { basename: 'test', path: 'test.md', } as any); - const score = (indexer as any).calculateWeightedScore(tokenized, queryTokens); - expect(score.score).toBe(0); + const score = (indexer as any).calculateWeightedScore(tokenized, queryTokens, []); + expect(score).toBe(0); }); it('should score higher when more tokens match', () => { @@ -300,13 +300,15 @@ describe('VaultIndexer', () => { const query2 = 'algorithm design pattern'; const score1 = (indexer as any).calculateWeightedScore( tokenized, - (indexer as any).tokenize(query1) + (indexer as any).tokenize(query1), + [] ); const score2 = (indexer as any).calculateWeightedScore( tokenized, - (indexer as any).tokenize(query2) + (indexer as any).tokenize(query2), + [] ); - expect(score2.score).toBeGreaterThan(score1.score); + expect(score2).toBeGreaterThan(score1); }); it('should be case insensitive', () => { @@ -318,9 +320,10 @@ describe('VaultIndexer', () => { const query = 'important algorithm'; const score = (indexer as any).calculateWeightedScore( tokenized, - (indexer as any).tokenize(query) + (indexer as any).tokenize(query), + [] ); - expect(score.score).toBeGreaterThan(0); + expect(score).toBeGreaterThan(0); }); it('should handle word boundary matching', () => { @@ -332,9 +335,10 @@ describe('VaultIndexer', () => { const query = 'algorithm'; const score = (indexer as any).calculateWeightedScore( tokenized, - (indexer as any).tokenize(query) + (indexer as any).tokenize(query), + [] ); - expect(score.score).toBeGreaterThan(0); + expect(score).toBeGreaterThan(0); }); }); @@ -392,4 +396,97 @@ describe('VaultIndexer', () => { expect(tokenized.firstParagraph).not.toContain('Second'); }); }); + + describe('extractExactPhrases', () => { + it('should extract quoted phrases', () => { + const phrases = (indexer as any).extractExactPhrases('search "exact phrase" here'); + expect(phrases).toEqual(['exact phrase']); + }); + + it('should extract multiple quoted phrases', () => { + const phrases = (indexer as any).extractExactPhrases('"phrase one" and "phrase two"'); + expect(phrases).toEqual(['phrase one', 'phrase two']); + }); + + it('should return empty array when no quotes', () => { + const phrases = (indexer as any).extractExactPhrases('no quotes here'); + expect(phrases).toEqual([]); + }); + }); + + describe('searchVault with options', () => { + it('should boost exact phrase matches', async () => { + const file1: MockTFile = { basename: 'a', path: 'a.md' }; + const file2: MockTFile = { basename: 'b', path: 'b.md' }; + mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]); + mockVault.read = jest + .fn() + .mockResolvedValueOnce('The quick brown fox jumps over the lazy dog') + .mockResolvedValueOnce('The quick brown fox'); + + const results = await indexer.searchVault('"lazy dog"', 5); + // The file with the exact phrase should rank higher + expect(results.length).toBeGreaterThan(0); + if (results.length >= 2) { + expect(results[0].score).toBeGreaterThanOrEqual(results[1].score); + } + }); + + it('should filter by folder', async () => { + const file1: MockTFile = { basename: 'a', path: 'Projects/a.md' }; + const file2: MockTFile = { basename: 'b', path: 'Archive/b.md' }; + mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]); + mockVault.read = jest + .fn() + .mockResolvedValueOnce('important keyword here') + .mockResolvedValueOnce('completely unrelated text'); + + const results = await indexer.searchVault('important keyword', 5, { folder: 'Projects' }); + expect(results.length).toBe(1); + expect(results[0].path).toBe('Projects/a.md'); + }); + + it('should filter by tag', async () => { + const file1: MockTFile = { basename: 'a', path: 'a.md' }; + const file2: MockTFile = { basename: 'b', path: 'b.md' }; + mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]); + mockVault.read = jest + .fn() + .mockResolvedValueOnce('---\ntags: idea\n---\nimportant keyword') + .mockResolvedValueOnce('---\ntags: done\n---\nother unrelated content'); + + const results = await indexer.searchVault('important keyword', 5, { tag: 'idea' }); + expect(results.length).toBe(1); + expect(results[0].path).toBe('a.md'); + }); + + it('should apply recency boost', async () => { + const now = Date.now(); + const file1: MockTFile = { basename: 'a', path: 'a.md' }; + const file2: MockTFile = { basename: 'b', path: 'b.md' }; + mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]); + mockVault.read = jest + .fn() + .mockResolvedValueOnce('important keyword') + .mockResolvedValueOnce('important keyword'); + + // Mock stat with different mtimes + (file1 as any).stat = { mtime: now - 86400000 }; // 1 day ago + (file2 as any).stat = { mtime: now - 86400000 * 100 }; // 100 days ago + + const results = await indexer.searchVault('important keyword', 5, { recencyBoost: true }); + expect(results.length).toBe(2); + // The more recent file should have a higher score + expect(results[0].score).toBeGreaterThanOrEqual(results[1].score); + }); + + it('should build a cache key with options', () => { + const key1 = (indexer as any).buildCacheKey('test', 5, { folder: 'Projects', tag: 'idea' }); + expect(key1).toContain('folder:Projects'); + expect(key1).toContain('tag:idea'); + + const key2 = (indexer as any).buildCacheKey('test', 5, { recencyBoost: false }); + expect(key2).toContain('norecency'); + }); + }); });