Add semantic cache support using ChromaDB #3

Open
fegger wants to merge 4 commits from feature/semantic_caching into main
4 changed files with 29 additions and 43 deletions
Showing only changes of commit c598d9cf01 - Show all commits
+1 -1
View File
@@ -11,6 +11,6 @@ export const DEFAULT_SETTINGS = {
similarityThreshold: 0.85, similarityThreshold: 0.85,
collectionName: 'ollama_semantic_cache', collectionName: 'ollama_semantic_cache',
embeddingModel: 'nomic-embed-text', embeddingModel: 'nomic-embed-text',
chromaUrl: 'http://localhost:8000', chromaURL: 'http://localhost:8000',
}, },
}; };
+5 -4
View File
@@ -170,12 +170,13 @@ class OllamaSettingTab extends PluginSettingTab {
new Setting(container) new Setting(container)
.setName('ChromaDB URL') .setName('ChromaDB URL')
.setDesc('URL of your ChromaDB instance (used for semantic cache)') .setDesc('URL for your ChromaDB instance (default: http://localhost:8000)')
.addText((text) => .addText((text) =>
text.setValue(this.plugin.settings.cacheConfig.chromaUrl).onChange(async (value) => { text
this.plugin.settings.cacheConfig.chromaUrl = value; .setValue(this.plugin.settings.cacheConfig.chromaURL || 'http://localhost:8000')
.onChange(async (value) => {
this.plugin.settings.cacheConfig.chromaURL = value;
await this.plugin.saveSettings(); await this.plugin.saveSettings();
this.plugin.notifyChatViews();
}) })
); );
+20 -35
View File
@@ -1,24 +1,25 @@
// src/semantic-cache.ts // src/semantic-cache.ts
import { ChromaClient, Collection, IncludeEnum } from 'chromadb'; import { ChromaClient } from 'chromadb';
import { Logger } from './utils'; import { Logger } from './utils';
import { CacheConfig } from './types'; import { CacheConfig } from './types';
export { CacheConfig } from './types';
export class SemanticCacheService { export class SemanticCacheService {
private client: ChromaClient; private client: ChromaClient;
private collection: Collection | null = null; private collection: ReturnType<ChromaClient['getOrCreateCollection']> | null = null;
private config: CacheConfig; private config: CacheConfig;
private ollamaURL: string; private ollamaURL: string;
private chromaURL: string;
constructor(ollamaURL: string, config: CacheConfig) { constructor(ollamaURL: string, config: CacheConfig) {
this.ollamaURL = ollamaURL.replace(/\/+$/, ''); this.ollamaURL = ollamaURL.replace(/\/+$/, '');
this.config = config; this.config = config;
this.client = new ChromaClient({ path: config.chromaUrl }); // Use configurable ChromaDB URL or default to localhost
this.chromaURL = config.chromaURL || 'http://localhost:8000';
this.client = new ChromaClient({ path: this.chromaURL });
} }
async initialize(): Promise<void> { async initialize() {
if (!this.config.enabled) return; if (!this.config.enabled) return;
try { try {
@@ -29,32 +30,9 @@ export class SemanticCacheService {
Logger.info(`Semantic cache initialized: ${this.config.collectionName}`, 'semantic-cache'); Logger.info(`Semantic cache initialized: ${this.config.collectionName}`, 'semantic-cache');
} catch (error) { } catch (error) {
Logger.error(`Failed to initialize semantic cache: ${String(error)}`, 'semantic-cache'); Logger.error(`Failed to initialize semantic cache: ${String(error)}`, 'semantic-cache');
throw error;
} }
} }
async clearCache(): Promise<void> {
await this.client.deleteCollection({ name: this.config.collectionName });
this.collection = null;
Logger.info('Semantic cache cleared', 'semantic-cache');
try {
await this.initialize();
} catch {
// best-effort re-init — swallow errors
}
}
// FNV-1a 32-bit hash — deterministic and collision-resistant enough for cache keys
private computeId(text: string): string {
let hash = 0x811c9dc5;
for (let i = 0; i < text.length; i++) {
hash ^= text.charCodeAt(i);
hash = Math.imul(hash, 0x01000193) >>> 0;
}
return hash.toString(16).padStart(8, '0');
}
private async getEmbedding(text: string): Promise<number[]> { private async getEmbedding(text: string): Promise<number[]> {
try { try {
const response = await fetch(`${this.ollamaURL}/api/embeddings`, { const response = await fetch(`${this.ollamaURL}/api/embeddings`, {
@@ -70,7 +48,7 @@ export class SemanticCacheService {
throw new Error(`Embedding failed with status ${response.status}`); throw new Error(`Embedding failed with status ${response.status}`);
} }
const data = (await response.json()) as { embedding: number[] }; const data = await response.json();
return data.embedding; return data.embedding;
} catch (error) { } catch (error) {
Logger.warn(`Failed to generate embedding: ${String(error)}`, 'semantic-cache'); Logger.warn(`Failed to generate embedding: ${String(error)}`, 'semantic-cache');
@@ -90,7 +68,7 @@ export class SemanticCacheService {
const results = await this.collection.query({ const results = await this.collection.query({
queryEmbeddings: [embedding], queryEmbeddings: [embedding],
nResults: 1, nResults: 1,
include: [IncludeEnum.Metadatas, IncludeEnum.Distances], include: ['metadatas', 'distances'],
}); });
// Cosine distance = 1 - cosine_similarity // Cosine distance = 1 - cosine_similarity
@@ -101,8 +79,7 @@ export class SemanticCacheService {
results.distances[0][0] < 1 - this.config.similarityThreshold results.distances[0][0] < 1 - this.config.similarityThreshold
) { ) {
Logger.debug('Semantic cache hit', 'semantic-cache'); Logger.debug('Semantic cache hit', 'semantic-cache');
const fullResponse = results.metadatas?.[0]?.[0]?.fullResponse; return results.metadatas?.[0]?.[0]?.fullResponse ?? null;
return typeof fullResponse === 'string' ? fullResponse : null;
} }
} catch (error) { } catch (error) {
Logger.warn(`Cache lookup failed: ${String(error)}`, 'semantic-cache'); Logger.warn(`Cache lookup failed: ${String(error)}`, 'semantic-cache');
@@ -120,8 +97,16 @@ export class SemanticCacheService {
const embedding = await this.getEmbedding(prompt); const embedding = await this.getEmbedding(prompt);
if (!embedding.length) return; if (!embedding.length) return;
const id = this.computeId(prompt); // Fallback for crypto.randomUUID() if not available
await this.collection.upsert({ let id: string;
if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {
id = crypto.randomUUID();
} else {
// Fallback to a simple ID generator if crypto is not available
id = 'cache_' + Date.now() + '_' + Math.random().toString(36).substr(2, 9);
}
await this.collection.add({
ids: [id], ids: [id],
embeddings: [embedding], embeddings: [embedding],
metadatas: [{ fullResponse: response }], metadatas: [{ fullResponse: response }],
+1 -1
View File
@@ -95,7 +95,7 @@ export interface CacheConfig {
similarityThreshold: number; similarityThreshold: number;
collectionName: string; collectionName: string;
embeddingModel: string; embeddingModel: string;
chromaUrl: string; chromaURL?: string;
} }
export interface PluginSettings { export interface PluginSettings {