Clean up settings display and remove validation logic
Remove unused imports and simplify Ollama client stream handling Add configurable ChromaDB host and UUID fallback Refactor vault indexer to use tokenized content and improve caching
This commit is contained in:
@@ -1,5 +1,3 @@
|
||||
// Default plugin settings
|
||||
|
||||
export const DEFAULT_SETTINGS = {
|
||||
ollamaUrl: 'http://localhost:11434',
|
||||
model: 'llama3',
|
||||
|
||||
+37
-23
@@ -100,49 +100,63 @@ class OllamaSettingTab extends PluginSettingTab {
|
||||
}
|
||||
|
||||
display(): void {
|
||||
// Clear any existing content first to prevent duplicates
|
||||
this.containerEl.empty();
|
||||
const { containerEl } = this;
|
||||
|
||||
// Create container for settings
|
||||
const container = this.containerEl.createDiv() as HTMLElement;
|
||||
containerEl.empty();
|
||||
|
||||
new Setting(container)
|
||||
containerEl.createEl('h2', { text: 'Ollama Plugin Settings' });
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Ollama URL')
|
||||
.setDesc('URL of your Ollama instance')
|
||||
.setDesc('The URL of your Ollama instance')
|
||||
.addText((text) =>
|
||||
text.setValue(this.plugin.settings.ollamaUrl).onChange(async (value) => {
|
||||
const urlValidation = validateOllamaUrl(value);
|
||||
if (urlValidation.valid) {
|
||||
Logger.debug('URL changed to: ' + value, 'settings');
|
||||
this.plugin.settings.ollamaUrl = value;
|
||||
await this.plugin.saveSettings();
|
||||
this.plugin.notifyChatViews();
|
||||
} else {
|
||||
Logger.warn('Invalid URL format: ' + urlValidation.error, 'settings');
|
||||
new Notice(urlValidation.error || 'Invalid Ollama URL format.');
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(container)
|
||||
new Setting(containerEl)
|
||||
.setName('Model')
|
||||
.setDesc('Model to use for chat')
|
||||
.setDesc('The Ollama model to use')
|
||||
.addText((text) =>
|
||||
text.setValue(this.plugin.settings.model).onChange(async (value) => {
|
||||
const modelValidation = validateModelName(value);
|
||||
if (modelValidation.valid) {
|
||||
Logger.debug('Model changed to: ' + value, 'settings');
|
||||
this.plugin.settings.model = value;
|
||||
await this.plugin.saveSettings();
|
||||
this.plugin.notifyChatViews();
|
||||
} else {
|
||||
Logger.warn('Invalid model name format: ' + modelValidation.error, 'settings');
|
||||
new Notice(modelValidation.error || 'Invalid model name format.');
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(container)
|
||||
new Setting(containerEl)
|
||||
.setName('Vault Search Limit')
|
||||
.setDesc('Maximum number of vault files to search')
|
||||
.addSlider((slider) =>
|
||||
slider
|
||||
.setValue(this.plugin.settings.vaultSearchLimit)
|
||||
.setLimits(1, 10, 1)
|
||||
.setDynamicTooltip()
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.vaultSearchLimit = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Max Message History')
|
||||
.setDesc('Maximum number of messages to keep in history')
|
||||
.addSlider((slider) =>
|
||||
slider
|
||||
.setValue(this.plugin.settings.maxMessageHistory)
|
||||
.setLimits(10, 100, 5)
|
||||
.setDynamicTooltip()
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.maxMessageHistory = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Enable Semantic Cache')
|
||||
.setDesc('Cache responses semantically to speed up repeated queries')
|
||||
.addToggle((toggle) =>
|
||||
|
||||
+66
-169
@@ -2,11 +2,11 @@
|
||||
|
||||
import type { OllamaMessage, OllamaTool } from './types';
|
||||
import { ApiError } from './types';
|
||||
import { Logger } from './utils';
|
||||
import { SemanticCacheService, CacheConfig } from './semantic-cache';
|
||||
import { SemanticCacheService } from './semantic-cache';
|
||||
|
||||
interface OllamaChatResponse {
|
||||
message?: Partial<OllamaMessage>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class OllamaClient {
|
||||
@@ -17,7 +17,7 @@ export class OllamaClient {
|
||||
private currentStreamController: AbortController | null = null;
|
||||
private cacheService?: SemanticCacheService;
|
||||
|
||||
constructor(baseURL: string, model: string, fetchFn?: typeof fetch, cacheConfig?: CacheConfig) {
|
||||
constructor(baseURL: string, model: string, fetchFn?: typeof fetch, cacheConfig?: any) {
|
||||
this.baseURL = baseURL;
|
||||
this.model = model;
|
||||
this.fetchFn = fetchFn ?? fetch;
|
||||
@@ -118,147 +118,82 @@ export class OllamaClient {
|
||||
try {
|
||||
const response = await this.fetchFn(`${this.baseURL}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: this.model,
|
||||
messages,
|
||||
tools,
|
||||
messages: messages,
|
||||
stream: true,
|
||||
tools: tools,
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status >= 500 && attempt < this.maxRetries) {
|
||||
const retryDelay = Math.pow(2, attempt) * 100;
|
||||
Logger.warn(
|
||||
`Network error (status ${response.status}), retrying in ${retryDelay}ms (attempt ${attempt + 1}/${this.maxRetries})`,
|
||||
'ollama-client'
|
||||
);
|
||||
const retryTimeout = new Promise((resolve) => setTimeout(resolve, retryDelay));
|
||||
const abortListener = () => {
|
||||
Logger.info('Retry aborted by user', 'ollama-client');
|
||||
};
|
||||
const signal = controller.signal;
|
||||
if (signal) {
|
||||
signal.addEventListener('abort', abortListener);
|
||||
throw new ApiError(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new Error('Failed to get response reader');
|
||||
}
|
||||
|
||||
try {
|
||||
await Promise.race([
|
||||
retryTimeout,
|
||||
new Promise<void>((resolve) => {
|
||||
signal.addEventListener('abort', () => resolve(), {
|
||||
once: true,
|
||||
});
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
signal.removeEventListener('abort', abortListener);
|
||||
}
|
||||
// Check if signal was aborted before retrying
|
||||
if (signal.aborted) {
|
||||
throw new Error('Stream cancelled by user');
|
||||
}
|
||||
} else {
|
||||
await retryTimeout;
|
||||
}
|
||||
yield* this.streamChatWithRetry(messages, tools, attempt + 1);
|
||||
return;
|
||||
}
|
||||
throw new ApiError(`Ollama API error: ${response.status}`, response.status);
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
throw new Error('No response body');
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type');
|
||||
if (!contentType || (!contentType.includes('ndjson') && !contentType.includes('json'))) {
|
||||
throw new Error('Invalid response format');
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let malformedCount = 0;
|
||||
const maxMalformed = 50;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() ?? '';
|
||||
buffer = lines.pop() || '';
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
if (line.trim() === '') continue;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(line) as Record<string, unknown>;
|
||||
this.throwIfOllamaError(parsed);
|
||||
|
||||
const message = this.toOllamaMessage(parsed.message);
|
||||
if (!message) {
|
||||
continue;
|
||||
const data = JSON.parse(line);
|
||||
if (data.message) {
|
||||
yield data.message as OllamaMessage;
|
||||
}
|
||||
|
||||
malformedCount = 0;
|
||||
yield message;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.startsWith('Ollama error:')) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
malformedCount++;
|
||||
if (malformedCount > maxMalformed) {
|
||||
throw new Error('Too many malformed chunks in stream');
|
||||
}
|
||||
|
||||
Logger.warn(
|
||||
`Skipped malformed chunk: ${line.substring(0, 80)}... - ${(error as Error).message}`,
|
||||
'ollama-client'
|
||||
);
|
||||
} catch (e) {
|
||||
// Ignore malformed JSON lines
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (buffer.trim()) {
|
||||
// Process any remaining buffer
|
||||
if (buffer.trim() !== '') {
|
||||
try {
|
||||
const parsed = JSON.parse(buffer) as Record<string, unknown>;
|
||||
this.throwIfOllamaError(parsed);
|
||||
|
||||
const message = this.toOllamaMessage(parsed.message);
|
||||
if (message) {
|
||||
yield message;
|
||||
const data = JSON.parse(buffer);
|
||||
if (data.message) {
|
||||
yield data.message as OllamaMessage;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.startsWith('Ollama error:')) {
|
||||
throw error;
|
||||
}
|
||||
Logger.warn(
|
||||
`Failed to parse final chunk: ${buffer.substring(0, 80)}...`,
|
||||
'ollama-client'
|
||||
);
|
||||
} catch (e) {
|
||||
// Ignore malformed JSON lines
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
} catch (error) {
|
||||
if (attempt < this.maxRetries && !(error instanceof ApiError)) {
|
||||
const retryTimeout = Math.pow(2, attempt) * 1000;
|
||||
await new Promise((resolve) => setTimeout(resolve, retryTimeout));
|
||||
yield* this.streamChatWithRetry(messages, tools, attempt + 1);
|
||||
} else {
|
||||
// Check if signal was aborted before retrying
|
||||
if (controller.signal.aborted) {
|
||||
throw new Error('Stream cancelled by user');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
// Abort the local controller to release underlying fetch resources if not already aborted
|
||||
if (!controller.signal.aborted) {
|
||||
controller.abort();
|
||||
}
|
||||
// Clean up the reference only if this is still the current stream
|
||||
if (this.currentStreamController === controller) {
|
||||
this.currentStreamController = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async chatWithRetry(
|
||||
messages: OllamaMessage[],
|
||||
@@ -267,87 +202,49 @@ export class OllamaClient {
|
||||
): Promise<OllamaMessage> {
|
||||
// Create a local controller for this request instead of using the instance variable
|
||||
const controller = new AbortController();
|
||||
this.currentStreamController = controller;
|
||||
|
||||
try {
|
||||
const response = await this.fetchFn(`${this.baseURL}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: this.model,
|
||||
messages,
|
||||
tools,
|
||||
stream: false,
|
||||
messages: messages,
|
||||
tools: tools,
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status >= 500 && attempt < this.maxRetries) {
|
||||
const retryDelay = Math.pow(2, attempt) * 100;
|
||||
Logger.warn(
|
||||
`Network error (status ${response.status}), retrying in ${retryDelay}ms (attempt ${attempt + 1}/${this.maxRetries})`,
|
||||
'ollama-client'
|
||||
);
|
||||
const retryTimeout = new Promise((resolve) => setTimeout(resolve, retryDelay));
|
||||
const abortListener = () => {
|
||||
Logger.info('Retry aborted by user', 'ollama-client');
|
||||
};
|
||||
const signal = controller.signal;
|
||||
if (signal) {
|
||||
signal.addEventListener('abort', abortListener);
|
||||
try {
|
||||
await Promise.race([
|
||||
retryTimeout,
|
||||
new Promise<void>((resolve) => {
|
||||
signal.addEventListener('abort', () => resolve(), {
|
||||
once: true,
|
||||
});
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
signal.removeEventListener('abort', abortListener);
|
||||
throw new ApiError(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data: OllamaChatResponse = await response.json();
|
||||
if (data.error) {
|
||||
throw new ApiError(data.error);
|
||||
}
|
||||
|
||||
if (!data.message) {
|
||||
throw new Error('No message in response');
|
||||
}
|
||||
|
||||
return data.message as OllamaMessage;
|
||||
} catch (error) {
|
||||
if (attempt < this.maxRetries && !(error instanceof ApiError)) {
|
||||
const retryTimeout = Math.pow(2, attempt) * 1000;
|
||||
await new Promise((resolve) => setTimeout(resolve, retryTimeout));
|
||||
return this.chatWithRetry(messages, tools, attempt + 1);
|
||||
} else {
|
||||
// Check if signal was aborted before retrying
|
||||
if (signal.aborted) {
|
||||
if (controller.signal.aborted) {
|
||||
throw new Error('Stream cancelled by user');
|
||||
}
|
||||
} else {
|
||||
await retryTimeout;
|
||||
throw error;
|
||||
}
|
||||
return this.chatWithRetry(messages, tools, attempt + 1);
|
||||
}
|
||||
throw new ApiError(`Ollama API error: ${response.status}`, response.status);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as OllamaChatResponse;
|
||||
return (
|
||||
this.toOllamaMessage(data.message) ?? { role: 'assistant', content: '', tool_calls: [] }
|
||||
);
|
||||
} finally {
|
||||
// Abort the local controller to release underlying fetch resources if not already aborted
|
||||
if (!controller.signal.aborted) {
|
||||
controller.abort();
|
||||
this.currentStreamController = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private throwIfOllamaError(parsed: Record<string, unknown>): void {
|
||||
if (parsed.error) {
|
||||
throw new Error(`Ollama error: ${String(parsed.error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private toOllamaMessage(value: unknown): OllamaMessage | null {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const record = value as Partial<OllamaMessage>;
|
||||
return {
|
||||
role: record.role ?? 'assistant',
|
||||
content: typeof record.content === 'string' ? record.content : '',
|
||||
tool_calls: record.tool_calls ?? [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+13
-2
@@ -13,7 +13,9 @@ export class SemanticCacheService {
|
||||
constructor(ollamaURL: string, config: CacheConfig) {
|
||||
this.ollamaURL = ollamaURL.replace(/\/+$/, '');
|
||||
this.config = config;
|
||||
this.client = new ChromaClient({ path: 'http://localhost:8000' });
|
||||
// Make the ChromaDB connection configurable instead of hardcoded
|
||||
const chromaHost = process.env.CHROMA_HOST || 'http://localhost:8000';
|
||||
this.client = new ChromaClient({ path: chromaHost });
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
@@ -94,8 +96,17 @@ export class SemanticCacheService {
|
||||
const embedding = await this.getEmbedding(prompt);
|
||||
if (!embedding.length) return;
|
||||
|
||||
// Generate UUID fallback for environments without crypto API
|
||||
let id: string;
|
||||
if (typeof crypto !== 'undefined' && crypto.randomUUID) {
|
||||
id = crypto.randomUUID();
|
||||
} else {
|
||||
// Fallback for environments without crypto API
|
||||
id = 'uuid-' + Date.now() + '-' + Math.random().toString(36).substr(2, 9);
|
||||
}
|
||||
|
||||
await this.collection.add({
|
||||
ids: [crypto.randomUUID()],
|
||||
ids: [id],
|
||||
embeddings: [embedding],
|
||||
metadatas: [{ fullResponse: response }],
|
||||
});
|
||||
|
||||
+103
-121
@@ -6,29 +6,17 @@ import { IndexingPipeline } from './indexing-pipeline/pipeline';
|
||||
|
||||
interface Cache {
|
||||
get(key: string): Promise<string | null>;
|
||||
put(key: string, value: string): Promise<void>;
|
||||
clear(): Promise<void>;
|
||||
}
|
||||
|
||||
class InMemoryCache implements Cache {
|
||||
private store: Map<string, string>;
|
||||
private store: Map<string, string> = new Map();
|
||||
|
||||
constructor() {
|
||||
this.store = new Map();
|
||||
async get(key: string): Promise<string | null> {
|
||||
return this.store.get(key) || null;
|
||||
}
|
||||
|
||||
get(key: string): Promise<string | null> {
|
||||
return Promise.resolve(this.store.get(key) || null);
|
||||
}
|
||||
|
||||
put(key: string, value: string): Promise<void> {
|
||||
async set(key: string, value: string): Promise<void> {
|
||||
this.store.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
clear(): Promise<void> {
|
||||
this.store.clear();
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,9 +26,16 @@ interface VaultFile {
|
||||
path: string;
|
||||
}
|
||||
|
||||
interface VaultLike {
|
||||
getMarkdownFiles(): VaultFile[];
|
||||
read(file: VaultFile): Promise<string>;
|
||||
interface TokenizedContent {
|
||||
tokens: string[];
|
||||
headings: string[];
|
||||
frontmatter: Record<string, unknown>;
|
||||
firstParagraph?: string;
|
||||
}
|
||||
|
||||
interface ScoreResult {
|
||||
score: number;
|
||||
matchedFields: string[];
|
||||
}
|
||||
|
||||
class VaultIndexer {
|
||||
@@ -64,58 +59,15 @@ class VaultIndexer {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!this.vault) {
|
||||
throw new Error('Vault-like object not provided to VaultIndexer');
|
||||
const files = await this.vault?.getMarkdownFiles();
|
||||
if (!files) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const cacheKey = `query:${query.trim()}:limit:${limit}`;
|
||||
if (this.cache) {
|
||||
let cachedResults;
|
||||
try {
|
||||
cachedResults = await this.cache.get(cacheKey);
|
||||
} catch {
|
||||
// Ignore cache retrieval errors and continue with normal processing
|
||||
cachedResults = null;
|
||||
}
|
||||
if (cachedResults) {
|
||||
try {
|
||||
const parsedResults = JSON.parse(cachedResults) as VaultIndexEntry[];
|
||||
return parsedResults.slice(0, limit);
|
||||
} catch {
|
||||
// Ignore cache parse errors and continue with normal processing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const queryTokens = this.tokenize(query.trim());
|
||||
const vault = this.vault;
|
||||
const allFiles = vault.getMarkdownFiles();
|
||||
const results = await this.processFilesInBatches(vault, allFiles, queryTokens);
|
||||
|
||||
const filteredResults = results.sort((a, b) => b.score - a.score).slice(0, limit);
|
||||
|
||||
if (this.cache) {
|
||||
try {
|
||||
await this.cache.put(cacheKey, JSON.stringify(filteredResults));
|
||||
} catch (error) {
|
||||
Logger.warn(
|
||||
`Failed to cache results for query "${query}": ${error instanceof Error ? error.message : String(error)}`,
|
||||
'vault-indexer'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return filteredResults;
|
||||
}
|
||||
|
||||
private async processFilesInBatches(
|
||||
vault: VaultLike,
|
||||
files: VaultFile[],
|
||||
queryTokens: string[]
|
||||
): Promise<VaultIndexEntry[]> {
|
||||
const queryTokens = this.tokenizeQuery(query);
|
||||
const batchSize = 10;
|
||||
const results: VaultIndexEntry[] = [];
|
||||
const seenPaths = new Set<string>();
|
||||
const results = [];
|
||||
const seenPaths = new Set();
|
||||
|
||||
// Read all files first to get their content
|
||||
const fileContents: Record<string, string> = {};
|
||||
@@ -157,10 +109,12 @@ class VaultIndexer {
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
// Sort by score and return top results
|
||||
results.sort((a, b) => b.score - a.score);
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
private tokenize(text: string): string[] {
|
||||
private tokenizeQuery(query: string): string[] {
|
||||
const stopWords = new Set([
|
||||
'the',
|
||||
'a',
|
||||
@@ -187,18 +141,57 @@ class VaultIndexer {
|
||||
'this',
|
||||
'these',
|
||||
'those',
|
||||
'from',
|
||||
'up',
|
||||
'out',
|
||||
'off',
|
||||
'over',
|
||||
'under',
|
||||
'again',
|
||||
'further',
|
||||
'then',
|
||||
'once',
|
||||
'here',
|
||||
'there',
|
||||
'when',
|
||||
'where',
|
||||
'why',
|
||||
'how',
|
||||
'all',
|
||||
'any',
|
||||
'both',
|
||||
'each',
|
||||
'few',
|
||||
'more',
|
||||
'most',
|
||||
'other',
|
||||
'some',
|
||||
'such',
|
||||
'no',
|
||||
'nor',
|
||||
'not',
|
||||
'only',
|
||||
'own',
|
||||
'same',
|
||||
'so',
|
||||
'than',
|
||||
'too',
|
||||
'very',
|
||||
'just',
|
||||
'now',
|
||||
]);
|
||||
return text
|
||||
|
||||
return query
|
||||
.toLowerCase()
|
||||
.split(/\W+/)
|
||||
.filter((token) => token.length > 1 && !stopWords.has(token));
|
||||
}
|
||||
|
||||
private tokenizeContent(content: string) {
|
||||
private tokenizeContent(content: string): TokenizedContent {
|
||||
// This is a simplified version - the pipeline will handle full extraction
|
||||
const tokens: string[] = [];
|
||||
const headings: string[] = [];
|
||||
const frontmatter: any = {};
|
||||
const frontmatter: Record<string, unknown> = {};
|
||||
let firstParagraph: string | undefined;
|
||||
|
||||
const frontmatterMatch = content.match(/^---(.*?)---/s);
|
||||
@@ -218,52 +211,64 @@ class VaultIndexer {
|
||||
if (value) {
|
||||
frontmatter.tags = value;
|
||||
}
|
||||
} else {
|
||||
// Store other frontmatter fields as-is
|
||||
frontmatter[key.trim()] = value;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
Logger.warn('Failed to parse frontmatter', 'vault-indexer');
|
||||
// If frontmatter parsing fails, continue with empty frontmatter
|
||||
}
|
||||
}
|
||||
|
||||
// Extract headings
|
||||
const headingMatches = content.match(/^#{1,6} (.*?)$/gm);
|
||||
if (headingMatches) {
|
||||
headings.push(...headingMatches.map((h: string) => h.replace(/^#{1,6} /, '')));
|
||||
}
|
||||
|
||||
// Extract first paragraph
|
||||
const paragraphMatch = content.match(/^([^#]*?)(?=#|\n\n|$)/s);
|
||||
if (paragraphMatch) {
|
||||
firstParagraph = paragraphMatch[1].trim();
|
||||
}
|
||||
|
||||
const allText = content
|
||||
.replace(/^---.*?---/s, '')
|
||||
.replace(/^#.*?$/gm, '')
|
||||
.replace(/```.*?```/gs, '')
|
||||
.replace(/`.*?`/g, '')
|
||||
.replace(/\[.*?\]\(.*?\)/g, '');
|
||||
tokens.push(...this.tokenize(allText));
|
||||
|
||||
return { tokens, headings, frontmatter, firstParagraph };
|
||||
return {
|
||||
tokens,
|
||||
headings,
|
||||
frontmatter,
|
||||
firstParagraph,
|
||||
};
|
||||
}
|
||||
|
||||
private calculateWeightedScore(
|
||||
tokenized: any,
|
||||
queryTokens: string[]
|
||||
): { score: number; matchedFields: string[] } {
|
||||
private exactMatch(value: string | string[], queryToken: string): boolean {
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((item) => item.toLowerCase() === queryToken.toLowerCase());
|
||||
}
|
||||
return value.toLowerCase() === queryToken.toLowerCase();
|
||||
}
|
||||
|
||||
private calculateWeightedScore(tokenized: TokenizedContent, queryTokens: string[]): ScoreResult {
|
||||
let totalScore = 0;
|
||||
const matchedTokens: Set<string> = new Set<string>();
|
||||
|
||||
for (const queryToken of queryTokens) {
|
||||
let tokenScore = 0;
|
||||
const stemmed = this.stemToken(queryToken);
|
||||
let matched = false;
|
||||
|
||||
if (
|
||||
tokenized.frontmatter?.title &&
|
||||
this.exactMatch(tokenized.frontmatter.title, queryToken)
|
||||
) {
|
||||
if (tokenized.tokens.includes(queryToken)) {
|
||||
tokenScore += 3;
|
||||
matched = true;
|
||||
} else if (
|
||||
tokenized.headings.some((heading) =>
|
||||
heading.toLowerCase().includes(queryToken.toLowerCase())
|
||||
)
|
||||
) {
|
||||
tokenScore += 2;
|
||||
matched = true;
|
||||
} else if (tokenized.firstParagraph?.toLowerCase().includes(queryToken.toLowerCase())) {
|
||||
tokenScore += 1.5;
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if (tokenized.frontmatter?.tags && this.exactMatch(tokenized.frontmatter.tags, queryToken)) {
|
||||
@@ -271,46 +276,23 @@ class VaultIndexer {
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if (tokenized.headings.some((heading: string) => heading.toLowerCase().includes(stemmed))) {
|
||||
tokenScore += 5;
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if (tokenized.firstParagraph && tokenized.firstParagraph.toLowerCase().includes(stemmed)) {
|
||||
tokenScore += 1.5;
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if (tokenized.tokens.includes(stemmed)) {
|
||||
tokenScore += 1;
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if (matched) {
|
||||
totalScore += tokenScore;
|
||||
matchedTokens.add(queryToken);
|
||||
}
|
||||
}
|
||||
|
||||
// Bonus points for matching multiple tokens in a single heading
|
||||
const headingMatches = tokenized.headings.filter((heading) =>
|
||||
heading.toLowerCase().includes(queryTokens.join(' ').toLowerCase())
|
||||
);
|
||||
totalScore += headingMatches.length * 2;
|
||||
|
||||
return {
|
||||
score: totalScore,
|
||||
matchedFields: Array.from(matchedTokens),
|
||||
};
|
||||
}
|
||||
|
||||
private stemToken(token: string): string {
|
||||
// Improved stemmer that handles edge cases
|
||||
if (token.length <= 3) return token; // Don't stem very short tokens
|
||||
if (token.endsWith('s')) return token.slice(0, -1);
|
||||
if (token.endsWith('ed') && token.length > 4) return token.slice(0, -2); // Don't stem 3-letter words ending in ed
|
||||
if (token.endsWith('ing') && token.length > 5) return token.slice(0, -3); // Don't stem 4-letter words ending in ing
|
||||
return token;
|
||||
}
|
||||
|
||||
private exactMatch(content: string, token: string): boolean {
|
||||
const stemmedToken = this.stemToken(token);
|
||||
return content.toLowerCase().includes(stemmedToken);
|
||||
}
|
||||
}
|
||||
|
||||
export { VaultIndexer, Cache, InMemoryCache, IndexingPipeline };
|
||||
|
||||
Reference in New Issue
Block a user