Add hybrid search with filtering and recency boost
Introduces SearchOptions interface with folder/tag filters, exact phrase matching via quoted queries, and optional recency boosting with configurable half-life. Replaces pure semantic or keyword search with a combined scoring model: keyword scores are blended with semantic results, then adjusted by filters and recency. Adds mtime to vault index entries and expands test coverage for the new options.
This commit is contained in:
+106
-9
@@ -286,8 +286,8 @@ describe('VaultIndexer', () => {
|
||||
basename: 'test',
|
||||
path: 'test.md',
|
||||
} as any);
|
||||
const score = (indexer as any).calculateWeightedScore(tokenized, queryTokens);
|
||||
expect(score.score).toBe(0);
|
||||
const score = (indexer as any).calculateWeightedScore(tokenized, queryTokens, []);
|
||||
expect(score).toBe(0);
|
||||
});
|
||||
|
||||
it('should score higher when more tokens match', () => {
|
||||
@@ -300,13 +300,15 @@ describe('VaultIndexer', () => {
|
||||
const query2 = 'algorithm design pattern';
|
||||
const score1 = (indexer as any).calculateWeightedScore(
|
||||
tokenized,
|
||||
(indexer as any).tokenize(query1)
|
||||
(indexer as any).tokenize(query1),
|
||||
[]
|
||||
);
|
||||
const score2 = (indexer as any).calculateWeightedScore(
|
||||
tokenized,
|
||||
(indexer as any).tokenize(query2)
|
||||
(indexer as any).tokenize(query2),
|
||||
[]
|
||||
);
|
||||
expect(score2.score).toBeGreaterThan(score1.score);
|
||||
expect(score2).toBeGreaterThan(score1);
|
||||
});
|
||||
|
||||
it('should be case insensitive', () => {
|
||||
@@ -318,9 +320,10 @@ describe('VaultIndexer', () => {
|
||||
const query = 'important algorithm';
|
||||
const score = (indexer as any).calculateWeightedScore(
|
||||
tokenized,
|
||||
(indexer as any).tokenize(query)
|
||||
(indexer as any).tokenize(query),
|
||||
[]
|
||||
);
|
||||
expect(score.score).toBeGreaterThan(0);
|
||||
expect(score).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should handle word boundary matching', () => {
|
||||
@@ -332,9 +335,10 @@ describe('VaultIndexer', () => {
|
||||
const query = 'algorithm';
|
||||
const score = (indexer as any).calculateWeightedScore(
|
||||
tokenized,
|
||||
(indexer as any).tokenize(query)
|
||||
(indexer as any).tokenize(query),
|
||||
[]
|
||||
);
|
||||
expect(score.score).toBeGreaterThan(0);
|
||||
expect(score).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -392,4 +396,97 @@ describe('VaultIndexer', () => {
|
||||
expect(tokenized.firstParagraph).not.toContain('Second');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractExactPhrases', () => {
|
||||
it('should extract quoted phrases', () => {
|
||||
const phrases = (indexer as any).extractExactPhrases('search "exact phrase" here');
|
||||
expect(phrases).toEqual(['exact phrase']);
|
||||
});
|
||||
|
||||
it('should extract multiple quoted phrases', () => {
|
||||
const phrases = (indexer as any).extractExactPhrases('"phrase one" and "phrase two"');
|
||||
expect(phrases).toEqual(['phrase one', 'phrase two']);
|
||||
});
|
||||
|
||||
it('should return empty array when no quotes', () => {
|
||||
const phrases = (indexer as any).extractExactPhrases('no quotes here');
|
||||
expect(phrases).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchVault with options', () => {
|
||||
it('should boost exact phrase matches', async () => {
|
||||
const file1: MockTFile = { basename: 'a', path: 'a.md' };
|
||||
const file2: MockTFile = { basename: 'b', path: 'b.md' };
|
||||
mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]);
|
||||
mockVault.read = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce('The quick brown fox jumps over the lazy dog')
|
||||
.mockResolvedValueOnce('The quick brown fox');
|
||||
|
||||
const results = await indexer.searchVault('"lazy dog"', 5);
|
||||
// The file with the exact phrase should rank higher
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
if (results.length >= 2) {
|
||||
expect(results[0].score).toBeGreaterThanOrEqual(results[1].score);
|
||||
}
|
||||
});
|
||||
|
||||
it('should filter by folder', async () => {
|
||||
const file1: MockTFile = { basename: 'a', path: 'Projects/a.md' };
|
||||
const file2: MockTFile = { basename: 'b', path: 'Archive/b.md' };
|
||||
mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]);
|
||||
mockVault.read = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce('important keyword here')
|
||||
.mockResolvedValueOnce('completely unrelated text');
|
||||
|
||||
const results = await indexer.searchVault('important keyword', 5, { folder: 'Projects' });
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].path).toBe('Projects/a.md');
|
||||
});
|
||||
|
||||
it('should filter by tag', async () => {
|
||||
const file1: MockTFile = { basename: 'a', path: 'a.md' };
|
||||
const file2: MockTFile = { basename: 'b', path: 'b.md' };
|
||||
mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]);
|
||||
mockVault.read = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce('---\ntags: idea\n---\nimportant keyword')
|
||||
.mockResolvedValueOnce('---\ntags: done\n---\nother unrelated content');
|
||||
|
||||
const results = await indexer.searchVault('important keyword', 5, { tag: 'idea' });
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].path).toBe('a.md');
|
||||
});
|
||||
|
||||
it('should apply recency boost', async () => {
|
||||
const now = Date.now();
|
||||
const file1: MockTFile = { basename: 'a', path: 'a.md' };
|
||||
const file2: MockTFile = { basename: 'b', path: 'b.md' };
|
||||
mockVault.getMarkdownFiles = jest.fn().mockReturnValue([file1, file2]);
|
||||
mockVault.read = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce('important keyword')
|
||||
.mockResolvedValueOnce('important keyword');
|
||||
|
||||
// Mock stat with different mtimes
|
||||
(file1 as any).stat = { mtime: now - 86400000 }; // 1 day ago
|
||||
(file2 as any).stat = { mtime: now - 86400000 * 100 }; // 100 days ago
|
||||
|
||||
const results = await indexer.searchVault('important keyword', 5, { recencyBoost: true });
|
||||
expect(results.length).toBe(2);
|
||||
// The more recent file should have a higher score
|
||||
expect(results[0].score).toBeGreaterThanOrEqual(results[1].score);
|
||||
});
|
||||
|
||||
it('should build a cache key with options', () => {
|
||||
const key1 = (indexer as any).buildCacheKey('test', 5, { folder: 'Projects', tag: 'idea' });
|
||||
expect(key1).toContain('folder:Projects');
|
||||
expect(key1).toContain('tag:idea');
|
||||
|
||||
const key2 = (indexer as any).buildCacheKey('test', 5, { recencyBoost: false });
|
||||
expect(key2).toContain('norecency');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user