Add hybrid search with filtering and recency boost

Introduces SearchOptions interface with folder/tag filters, exact phrase
matching via quoted queries, and optional recency boosting with configurable
half-life. Replaces pure semantic or keyword search with a combined scoring
model: keyword scores are blended with semantic results, then adjusted by
filters and recency. Adds mtime to vault index entries and expands test
coverage for the new options.

src/types.ts

@@ -157,6 +157,15 @@ export interface VaultIndexEntry {
   content: string;
   score: number;
   tags?: string;
+  mtime?: number;
+}
+
+export interface SearchOptions {
+  folder?: string;
+  tag?: string;
+  includeExactPhrase?: boolean;
+  recencyBoost?: boolean;
+  recencyHalfLifeDays?: number;
 }
 
 export interface ChatMessage {

src/vault-indexer.ts

@@ -4,7 +4,7 @@ import { Vault, TFile } from 'obsidian';
 import { Logger } from './utils';
 import { Cache } from './cache';
 import { VaultVectorStore } from './vault-vector-store';
-import { VaultIndexEntry } from './types';
+import { VaultIndexEntry, SearchOptions } from './types';
 
 interface ParsedFrontmatter {
   title?: string;
@@ -105,6 +105,8 @@ const STOP_WORDS = new Set([
 ]);
 
 const CONTENT_PREVIEW_LENGTH = 500;
+const DAYS_TO_MS = 86400000;
+const DEFAULT_RECENCY_HALF_LIFE = 30; // 30 days
 
 export class VaultIndexer {
   private vault: Vault;
@@ -116,6 +118,10 @@ export class VaultIndexer {
     FRONTMATTER_TAGS: 3,
     HEADINGS: 2,
     CONTENT: 1,
+    FILENAME: 3,
+    EXACT_PHRASE: 8,
+    LINKED: 2,
+    RECENT: 0.5, // multiplier, not additive
   };
 
   constructor(vault: Vault, cache?: Cache, vectorStore?: VaultVectorStore) {
@@ -154,14 +160,30 @@ export class VaultIndexer {
     };
   }
 
-  calculateWeightedScore(tokenized: TokenizedContent, queryTokens: string[]): { score: number } {
+  calculateWeightedScore(
+    tokenized: TokenizedContent,
+    queryTokens: string[],
+    exactPhrases: string[]
+  ): number {
     let score = 0;
+    const fullText = [
+      tokenized.title,
+      tokenized.headings.join(' '),
+      tokenized.frontmatter.title ?? '',
+      tokenized.frontmatter.tags ?? '',
+      tokenized.content,
+      tokenized.firstParagraph,
+      tokenized.basename,
+    ]
+      .join(' ')
+      .toLowerCase();
+
     for (const token of queryTokens) {
       if (tokenized.frontmatter?.title && this.exactMatch(tokenized.frontmatter.title, token)) {
         score += this.SCORING_WEIGHTS.FRONTMATTER_TITLE;
       }
       if (tokenized.basename && this.exactMatch(tokenized.basename, token)) {
-        score += this.SCORING_WEIGHTS.FRONTMATTER_TITLE;
+        score += this.SCORING_WEIGHTS.FILENAME;
       }
       if (tokenized.frontmatter?.tags && this.exactMatch(tokenized.frontmatter.tags, token)) {
         score += this.SCORING_WEIGHTS.FRONTMATTER_TAGS;
@@ -175,8 +197,19 @@ export class VaultIndexer {
       if (tokenized.title && this.exactMatch(tokenized.title, token)) {
         score += this.SCORING_WEIGHTS.TITLE;
       }
+      if (tokenized.firstParagraph.toLowerCase().includes(token.toLowerCase())) {
+        score += this.SCORING_WEIGHTS.CONTENT;
+      }
     }
-    return { score };
+
+    // Exact phrase bonus
+    for (const phrase of exactPhrases) {
+      if (fullText.includes(phrase.toLowerCase())) {
+        score += this.SCORING_WEIGHTS.EXACT_PHRASE;
+      }
+    }
+
+    return score;
   }
 
   async getVaultEntries(): Promise<VaultEntry[]> {
@@ -190,7 +223,7 @@ export class VaultIndexer {
             : await this.vault.read(file);
         const parsed = this.parseMarkdown(content);
         entries.push({
-          file: file,
+          file,
           title: parsed.frontmatter.title || file.basename,
           frontmatter: parsed.frontmatter,
           headings: parsed.headings,
@@ -206,29 +239,28 @@ export class VaultIndexer {
     return entries;
   }
 
-  async searchVault(query: string, limit = 3): Promise<VaultIndexEntry[]> {
+  async searchVault(query: string, limit = 3, options?: SearchOptions): Promise<VaultIndexEntry[]> {
     if (!query || !query.trim()) {
       return [];
     }
 
-    // Try semantic search first if vector store is available
+    const exactPhrases =
+      options?.includeExactPhrase !== false ? this.extractExactPhrases(query) : [];
+    const queryTokens = this.tokenize(query);
+
+    // Try hybrid search: semantic + keyword
+    let semanticResults: VaultIndexEntry[] = [];
     if (this.vectorStore) {
       try {
-        const semanticResults = await this.vectorStore.search(query, limit);
-        if (semanticResults.length > 0) {
-          return semanticResults;
-        }
+        semanticResults = await this.vectorStore.search(query, limit * 3);
       } catch (error) {
         const errorMessage = error instanceof Error ? error.message : String(error);
-        Logger.warn(
-          `Semantic search failed, falling back to keyword search: ${errorMessage}`,
-          'vault-indexer'
-        );
+        Logger.warn(`Semantic search failed: ${errorMessage}`, 'vault-indexer');
       }
     }
 
-    const cacheKey = `query:${query.trim()}:limit:${limit}`;
-    if (this.cache) {
+    const cacheKey = this.buildCacheKey(query, limit, options);
+    if (this.cache && semanticResults.length === 0) {
       let cachedResults: string | null = null;
       try {
         cachedResults = await this.cache.get(cacheKey);
@@ -246,15 +278,22 @@ export class VaultIndexer {
       }
     }
 
-    const queryTokens = this.tokenize(query);
-    if (queryTokens.length === 0) {
+    if (queryTokens.length === 0 && exactPhrases.length === 0) {
+      // Only non-token words (e.g. "a", "the") — try exact match fallback
+      if (semanticResults.length > 0) return semanticResults.slice(0, limit);
       return [];
     }
 
     const entries = await this.getVaultEntries();
+    const now = Date.now();
+    const halfLife = (options?.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE) * DAYS_TO_MS;
+
+    // Precompute exact phrases lowercased
+    const lowerExactPhrases = exactPhrases.map((p) => p.toLowerCase());
+
     const scored = entries
       .map((entry) => {
-        const { score } = this.calculateWeightedScore(
+        const keywordScore = this.calculateWeightedScore(
           {
             title: entry.title,
             headings: entry.headings,
@@ -263,11 +302,45 @@ export class VaultIndexer {
             content: entry.content,
             basename: entry.basename,
           },
-          queryTokens
+          queryTokens,
+          lowerExactPhrases
         );
+
+        // Semantic score
+        const semanticEntry = semanticResults.find((s) => s.path === entry.file.path);
+        const semanticScore = semanticEntry ? (semanticEntry.score || 0) * 0.3 : 0;
+
+        // Hybrid score: keyword dominates, semantic adds bonus
+        let score = keywordScore + semanticScore;
+
+        // Folder filter: penalize non-matches
+        if (options?.folder) {
+          const folderLower = options.folder.toLowerCase().replace(/\/$/, '');
+          const entryFolder = entry.file.path.toLowerCase().split('/').slice(0, -1).join('/');
+          if (!entryFolder.startsWith(folderLower) && entryFolder !== folderLower) {
+            score *= 0.1; // Heavy penalty
+          }
+        }
+
+        // Tag filter
+        if (options?.tag) {
+          const tagLower = options.tag.toLowerCase();
+          const entryTags = (entry.frontmatter.tags ?? '').toLowerCase();
+          if (!entryTags.includes(tagLower)) {
+            score *= 0.1;
+          }
+        }
+
+        // Recency boost
+        if (options?.recencyBoost !== false && entry.file.stat?.mtime) {
+          const age = now - entry.file.stat.mtime;
+          const recencyMultiplier = 1 + this.SCORING_WEIGHTS.RECENT * Math.exp(-age / halfLife);
+          score *= recencyMultiplier;
+        }
+
         return { ...entry, score };
       })
-      .filter((e) => e.score > 0);
+      .filter((e) => e.score > 0.01);
 
     scored.sort((a, b) => b.score - a.score);
     const results: VaultIndexEntry[] = scored.slice(0, limit).map((e) => ({
@@ -276,9 +349,10 @@ export class VaultIndexer {
       content: e.content,
       score: e.score,
       tags: e.frontmatter?.tags,
+      mtime: e.file.stat?.mtime,
     }));
 
-    if (this.cache) {
+    if (this.cache && semanticResults.length === 0) {
       try {
         await this.cache.put(cacheKey, JSON.stringify(results));
       } catch (error) {
@@ -293,6 +367,28 @@ export class VaultIndexer {
     return results;
   }
 
+  /**
+   * Extracts quoted exact phrases from a query.
+   */
+  private extractExactPhrases(query: string): string[] {
+    const phrases: string[] = [];
+    const quoteRegex = /"([^"]+)"/g;
+    let match: RegExpExecArray | null;
+    while ((match = quoteRegex.exec(query)) !== null) {
+      phrases.push(match[1]);
+    }
+    return phrases;
+  }
+
+  private buildCacheKey(query: string, limit: number, options?: SearchOptions): string {
+    const parts = [`query:${query.trim()}:limit:${limit}`];
+    if (options?.folder) parts.push(`folder:${options.folder}`);
+    if (options?.tag) parts.push(`tag:${options.tag}`);
+    if (options?.recencyBoost === false) parts.push('norecency');
+    if (options?.includeExactPhrase === false) parts.push('noexact');
+    return parts.join(':');
+  }
+
   private stemToken(token: string): string {
     if (token.endsWith('ing') && token.length > 4) return token.slice(0, -3);
     if (token.endsWith('ed') && token.length > 3) return token.slice(0, -2);

tests/vault-indexer.test.ts

@@ -286,8 +286,8 @@ describe('VaultIndexer', () => {
         basename: 'test',
         path: 'test.md',
       } as any);
-      const score = (indexer as any).calculateWeightedScore(tokenized, queryTokens);
-      expect(score.score).toBe(0);
+      const score = (indexer as any).calculateWeightedScore(tokenized, queryTokens, []);
+      expect(score).toBe(0);
     });
 
     it('should score higher when more tokens match', () => {
@@ -300,13 +300,15 @@ describe('VaultIndexer', () => {
       const query2 = 'algorithm design pattern';
       const score1 = (indexer as any).calculateWeightedScore(
         tokenized,
-        (indexer as any).tokenize(query1)
+        (indexer as any).tokenize(query1),
+        []
       );
       const score2 = (indexer as any).calculateWeightedScore(
         tokenized,
-        (indexer as any).tokenize(query2)
+        (indexer as any).tokenize(query2),
+        []
       );
-      expect(score2.score).toBeGreaterThan(score1.score);
+      expect(score2).toBeGreaterThan(score1);
     });
 
     it('should be case insensitive', () => {
@@ -318,9 +320,10 @@ describe('VaultIndexer', () => {
       const query = 'important algorithm';
       const score = (indexer as any).calculateWeightedScore(
         tokenized,
-        (indexer as any).tokenize(query)
+        (indexer as any).tokenize(query),
+        []
       );
-      expect(score.score).toBeGreaterThan(0);
+      expect(score).toBeGreaterThan(0);
     });
 
     it('should handle word boundary matching', () => {
@@ -332,9 +335,10 @@ describe('VaultIndexer', () => {
       const query = 'algorithm';
       const score = (indexer as any).calculateWeightedScore(
         tokenized,
-        (indexer as any).tokenize(query)
+        (indexer as any).tokenize(query),
+        []
       );
-      expect(score.score).toBeGreaterThan(0);
+      expect(score).toBeGreaterThan(0);
     });
   });
 
@@ -392,4 +396,97 @@ describe('VaultIndexer', () => {
       expect(tokenized.firstParagraph).not.toContain('Second');
     });
   });
+
+  describe('extractExactPhrases', () => {
+    it('should extract quoted phrases', () => {
+      const phrases = (indexer as any).extractExactPhrases('search "exact phrase" here');
+      expect(phrases).toEqual(['exact phrase']);
+    });
+
+    it('should extract multiple quoted phrases', () => {
+      const phrases = (indexer as any).extractExactPhrases('"phrase one" and "phrase two"');
+      expect(phrases).toEqual(['phrase one', 'phrase two']);
+    });
+
+    it('should return empty array when no quotes', () => {
+      const phrases = (indexer as any).extractExactPhrases('no quotes here');
+      expect(phrases).toEqual([]);
+    });
+  });
+
+  describe('searchVault with options', () => {
+    it('should boost exact phrase matches', async () => {
+      const file1: MockTFile = { basename: 'a', path: 'a.md' };
+      const file2: MockTFile = { basename: 'b', path: 'b.md' };
+      mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]);
+      mockVault.read = jest
+        .fn()
+        .mockResolvedValueOnce('The quick brown fox jumps over the lazy dog')
+        .mockResolvedValueOnce('The quick brown fox');
+
+      const results = await indexer.searchVault('"lazy dog"', 5);
+      // The file with the exact phrase should rank higher
+      expect(results.length).toBeGreaterThan(0);
+      if (results.length >= 2) {
+        expect(results[0].score).toBeGreaterThanOrEqual(results[1].score);
+      }
+    });
+
+    it('should filter by folder', async () => {
+      const file1: MockTFile = { basename: 'a', path: 'Projects/a.md' };
+      const file2: MockTFile = { basename: 'b', path: 'Archive/b.md' };
+      mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]);
+      mockVault.read = jest
+        .fn()
+        .mockResolvedValueOnce('important keyword here')
+        .mockResolvedValueOnce('completely unrelated text');
+
+      const results = await indexer.searchVault('important keyword', 5, { folder: 'Projects' });
+      expect(results.length).toBe(1);
+      expect(results[0].path).toBe('Projects/a.md');
+    });
+
+    it('should filter by tag', async () => {
+      const file1: MockTFile = { basename: 'a', path: 'a.md' };
+      const file2: MockTFile = { basename: 'b', path: 'b.md' };
+      mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]);
+      mockVault.read = jest
+        .fn()
+        .mockResolvedValueOnce('---\ntags: idea\n---\nimportant keyword')
+        .mockResolvedValueOnce('---\ntags: done\n---\nother unrelated content');
+
+      const results = await indexer.searchVault('important keyword', 5, { tag: 'idea' });
+      expect(results.length).toBe(1);
+      expect(results[0].path).toBe('a.md');
+    });
+
+    it('should apply recency boost', async () => {
+      const now = Date.now();
+      const file1: MockTFile = { basename: 'a', path: 'a.md' };
+      const file2: MockTFile = { basename: 'b', path: 'b.md' };
+      mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]);
+      mockVault.read = jest
+        .fn()
+        .mockResolvedValueOnce('important keyword')
+        .mockResolvedValueOnce('important keyword');
+
+      // Mock stat with different mtimes
+      (file1 as any).stat = { mtime: now - 86400000 }; // 1 day ago
+      (file2 as any).stat = { mtime: now - 86400000 * 100 }; // 100 days ago
+
+      const results = await indexer.searchVault('important keyword', 5, { recencyBoost: true });
+      expect(results.length).toBe(2);
+      // The more recent file should have a higher score
+      expect(results[0].score).toBeGreaterThanOrEqual(results[1].score);
+    });
+
+    it('should build a cache key with options', () => {
+      const key1 = (indexer as any).buildCacheKey('test', 5, { folder: 'Projects', tag: 'idea' });
+      expect(key1).toContain('folder:Projects');
+      expect(key1).toContain('tag:idea');
+
+      const key2 = (indexer as any).buildCacheKey('test', 5, { recencyBoost: false });
+      expect(key2).toContain('norecency');
+    });
+  });
 });