fegger/obsidian_ollama
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add hybrid search with filtering and recency boost

Browse Source 
Introduces SearchOptions interface with folder/tag filters, exact phrase
matching via quoted queries, and optional recency boosting with configurable
half-life. Replaces pure semantic or keyword search with a combined scoring
model: keyword scores are blended with semantic results, then adjusted by
filters and recency. Adds mtime to vault index entries and expands test
coverage for the new options.
This commit is contained in:
Florian Egger
2026-05-20 19:52:49 +02:00
parent 3abacb5d6e
commit 3c7c4d58bb
3 changed files with 234 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/types.ts
+9
View File
@@ -157,6 +157,15 @@ export interface VaultIndexEntry {
content: string;
score: number;
tags?: string;
mtime?: number;
}
export interface SearchOptions {
folder?: string;
tag?: string;
includeExactPhrase?: boolean;
recencyBoost?: boolean;
recencyHalfLifeDays?: number;
}
export interface ChatMessage {
src/vault-indexer.ts
+119 -23
View File
@@ -4,7 +4,7 @@ import { Vault, TFile } from 'obsidian';
import { Logger } from './utils';
import { Cache } from './cache';
import { VaultVectorStore } from './vault-vector-store';
import { VaultIndexEntry } from './types';
import { VaultIndexEntry, SearchOptions } from './types';
interface ParsedFrontmatter {
title?: string;
@@ -105,6 +105,8 @@ const STOP_WORDS = new Set([
]);
const CONTENT_PREVIEW_LENGTH = 500;
const DAYS_TO_MS = 86400000;
const DEFAULT_RECENCY_HALF_LIFE = 30; // 30 days
export class VaultIndexer {
private vault: Vault;
@@ -116,6 +118,10 @@ export class VaultIndexer {
FRONTMATTER_TAGS: 3,
HEADINGS: 2,
CONTENT: 1,
FILENAME: 3,
EXACT_PHRASE: 8,
LINKED: 2,
RECENT: 0.5, // multiplier, not additive
};
constructor(vault: Vault, cache?: Cache, vectorStore?: VaultVectorStore) {
@@ -154,14 +160,30 @@ export class VaultIndexer {
};
}
calculateWeightedScore(tokenized: TokenizedContent, queryTokens: string[]): { score: number } {
calculateWeightedScore(
tokenized: TokenizedContent,
queryTokens: string[],
exactPhrases: string[]
): number {
let score = 0;
const fullText = [
tokenized.title,
tokenized.headings.join(' '),
tokenized.frontmatter.title ?? '',
tokenized.frontmatter.tags ?? '',
tokenized.content,
tokenized.firstParagraph,
tokenized.basename,
]
.join(' ')
.toLowerCase();
for (const token of queryTokens) {
if (tokenized.frontmatter?.title && this.exactMatch(tokenized.frontmatter.title, token)) {
score += this.SCORING_WEIGHTS.FRONTMATTER_TITLE;
}
if (tokenized.basename && this.exactMatch(tokenized.basename, token)) {
score += this.SCORING_WEIGHTS.FRONTMATTER_TITLE;
score += this.SCORING_WEIGHTS.FILENAME;
}
if (tokenized.frontmatter?.tags && this.exactMatch(tokenized.frontmatter.tags, token)) {
score += this.SCORING_WEIGHTS.FRONTMATTER_TAGS;
@@ -175,8 +197,19 @@ export class VaultIndexer {
if (tokenized.title && this.exactMatch(tokenized.title, token)) {
score += this.SCORING_WEIGHTS.TITLE;
}
if (tokenized.firstParagraph.toLowerCase().includes(token.toLowerCase())) {
score += this.SCORING_WEIGHTS.CONTENT;
}
}
return { score };
// Exact phrase bonus
for (const phrase of exactPhrases) {
if (fullText.includes(phrase.toLowerCase())) {
score += this.SCORING_WEIGHTS.EXACT_PHRASE;
}
}
return score;
}
async getVaultEntries(): Promise<VaultEntry[]> {
@@ -190,7 +223,7 @@ export class VaultIndexer {
: await this.vault.read(file);
const parsed = this.parseMarkdown(content);
entries.push({
file: file,
file,
title: parsed.frontmatter.title || file.basename,
frontmatter: parsed.frontmatter,
headings: parsed.headings,
@@ -206,29 +239,28 @@ export class VaultIndexer {
return entries;
}
async searchVault(query: string, limit = 3): Promise<VaultIndexEntry[]> {
async searchVault(query: string, limit = 3, options?: SearchOptions): Promise<VaultIndexEntry[]> {
if (!query || !query.trim()) {
return [];
}
// Try semantic search first if vector store is available
const exactPhrases =
options?.includeExactPhrase !== false ? this.extractExactPhrases(query) : [];
const queryTokens = this.tokenize(query);
// Try hybrid search: semantic + keyword
let semanticResults: VaultIndexEntry[] = [];
if (this.vectorStore) {
try {
const semanticResults = await this.vectorStore.search(query, limit);
if (semanticResults.length > 0) {
return semanticResults;
}
semanticResults = await this.vectorStore.search(query, limit * 3);
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
Logger.warn(
`Semantic search failed, falling back to keyword search: ${errorMessage}`,
'vault-indexer'
);
Logger.warn(`Semantic search failed: ${errorMessage}`, 'vault-indexer');
}
}
const cacheKey = `query:${query.trim()}:limit:${limit}`;
if (this.cache) {
const cacheKey = this.buildCacheKey(query, limit, options);
if (this.cache && semanticResults.length === 0) {
let cachedResults: string | null = null;
try {
cachedResults = await this.cache.get(cacheKey);
@@ -246,15 +278,22 @@ export class VaultIndexer {
}
}
const queryTokens = this.tokenize(query);
if (queryTokens.length === 0) {
if (queryTokens.length === 0 && exactPhrases.length === 0) {
// Only non-token words (e.g. "a", "the") — try exact match fallback
if (semanticResults.length > 0) return semanticResults.slice(0, limit);
return [];
}
const entries = await this.getVaultEntries();
const now = Date.now();
const halfLife = (options?.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE) * DAYS_TO_MS;
// Precompute exact phrases lowercased
const lowerExactPhrases = exactPhrases.map((p) => p.toLowerCase());
const scored = entries
.map((entry) => {
const { score } = this.calculateWeightedScore(
const keywordScore = this.calculateWeightedScore(
{
title: entry.title,
headings: entry.headings,
@@ -263,11 +302,45 @@ export class VaultIndexer {
content: entry.content,
basename: entry.basename,
},
queryTokens
queryTokens,
lowerExactPhrases
);
// Semantic score
const semanticEntry = semanticResults.find((s) => s.path === entry.file.path);
const semanticScore = semanticEntry ? (semanticEntry.score || 0) * 0.3 : 0;
// Hybrid score: keyword dominates, semantic adds bonus
let score = keywordScore + semanticScore;
// Folder filter: penalize non-matches
if (options?.folder) {
const folderLower = options.folder.toLowerCase().replace(/\/$/, '');
const entryFolder = entry.file.path.toLowerCase().split('/').slice(0, -1).join('/');
if (!entryFolder.startsWith(folderLower) && entryFolder !== folderLower) {
score *= 0.1; // Heavy penalty
}
}
// Tag filter
if (options?.tag) {
const tagLower = options.tag.toLowerCase();
const entryTags = (entry.frontmatter.tags ?? '').toLowerCase();
if (!entryTags.includes(tagLower)) {
score *= 0.1;
}
}
// Recency boost
if (options?.recencyBoost !== false && entry.file.stat?.mtime) {
const age = now - entry.file.stat.mtime;
const recencyMultiplier = 1 + this.SCORING_WEIGHTS.RECENT * Math.exp(-age / halfLife);
score *= recencyMultiplier;
}
return { ...entry, score };
})
.filter((e) => e.score > 0);
.filter((e) => e.score > 0.01);
scored.sort((a, b) => b.score - a.score);
const results: VaultIndexEntry[] = scored.slice(0, limit).map((e) => ({
@@ -276,9 +349,10 @@ export class VaultIndexer {
content: e.content,
score: e.score,
tags: e.frontmatter?.tags,
mtime: e.file.stat?.mtime,
}));
if (this.cache) {
if (this.cache && semanticResults.length === 0) {
try {
await this.cache.put(cacheKey, JSON.stringify(results));
} catch (error) {
@@ -293,6 +367,28 @@ export class VaultIndexer {
return results;
}
/**
* Extracts quoted exact phrases from a query.
*/
private extractExactPhrases(query: string): string[] {
const phrases: string[] = [];
const quoteRegex = /"([^"]+)"/g;
let match: RegExpExecArray | null;
while ((match = quoteRegex.exec(query)) !== null) {
phrases.push(match[1]);
}
return phrases;
}
private buildCacheKey(query: string, limit: number, options?: SearchOptions): string {
const parts = [`query:${query.trim()}:limit:${limit}`];
if (options?.folder) parts.push(`folder:${options.folder}`);
if (options?.tag) parts.push(`tag:${options.tag}`);
if (options?.recencyBoost === false) parts.push('norecency');
if (options?.includeExactPhrase === false) parts.push('noexact');
return parts.join(':');
}
private stemToken(token: string): string {
if (token.endsWith('ing') && token.length > 4) return token.slice(0, -3);
if (token.endsWith('ed') && token.length > 3) return token.slice(0, -2);