fegger
cc97d77810
- semantic-cache.clearCache: delete collection instead of client.reset() to avoid 403 Forbidden on newer ChromaDB versions - vault-vector-store.clearIndex: same collection deletion approach - ContentVectorizer: add 3-attempt retry with exponential backoff (1s, 2s, 4s) - main.ts: reduce indexing batch size from 5 to 2, increase delay from 100ms to 500ms - Update semantic-cache tests for deleteCollection
107 lines
3.0 KiB
TypeScript
107 lines
3.0 KiB
TypeScript
// src/indexing-pipeline/vectorization.ts
|
|
|
|
import { ContentChunk } from './normalization';
|
|
import { Logger } from '../utils';
|
|
|
|
interface VectorizationConfig {
|
|
model: string;
|
|
ollamaUrl: string;
|
|
}
|
|
|
|
/**
|
|
* Vectorizes content chunks using Ollama embeddings
|
|
*/
|
|
export class ContentVectorizer {
|
|
private model: string;
|
|
private ollamaUrl: string;
|
|
private fetchFn: typeof fetch;
|
|
|
|
constructor(config: VectorizationConfig, fetchFn?: typeof fetch) {
|
|
this.model = config.model;
|
|
this.ollamaUrl = config.ollamaUrl;
|
|
this.fetchFn = fetchFn ?? ((url, init) => fetch(url, init));
|
|
}
|
|
|
|
/**
|
|
* Generates embeddings for a content chunk with retry logic
|
|
*/
|
|
async vectorize(chunk: ContentChunk): Promise<number[]> {
|
|
const prompt = this.createPrompt(chunk);
|
|
const maxRetries = 3;
|
|
const baseDelay = 1000;
|
|
|
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
try {
|
|
if (attempt > 0) {
|
|
const delay = baseDelay * Math.pow(2, attempt - 1);
|
|
Logger.info(
|
|
`Retrying embedding (attempt ${attempt + 1}/${maxRetries}) after ${delay}ms`,
|
|
'indexing-pipeline'
|
|
);
|
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
}
|
|
|
|
const response = await this.fetchFn(`${this.ollamaUrl}/api/embeddings`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
model: this.model,
|
|
prompt: prompt,
|
|
}),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`Embedding failed with status ${response.status}`);
|
|
}
|
|
|
|
const data: unknown = await response.json();
|
|
if (!this.isEmbeddingResponse(data)) {
|
|
throw new Error('Invalid embedding response');
|
|
}
|
|
|
|
return data.embedding;
|
|
} catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
Logger.warn(
|
|
`Embedding attempt ${attempt + 1} failed: ${errorMessage}`,
|
|
'indexing-pipeline'
|
|
);
|
|
if (attempt === maxRetries - 1) {
|
|
Logger.warn(
|
|
`Failed to generate embedding after ${maxRetries} attempts`,
|
|
'indexing-pipeline'
|
|
);
|
|
return [];
|
|
}
|
|
}
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
private isEmbeddingResponse(data: unknown): data is { embedding: number[] } {
|
|
return (
|
|
typeof data === 'object' &&
|
|
data !== null &&
|
|
Array.isArray((data as { embedding?: unknown }).embedding) &&
|
|
(data as { embedding: unknown[] }).embedding.every((value) => typeof value === 'number')
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Creates a prompt from content chunk for embedding
|
|
*/
|
|
private createPrompt(chunk: ContentChunk): string {
|
|
// Combine important elements for embedding
|
|
const parts = [
|
|
chunk.title,
|
|
chunk.firstParagraph,
|
|
chunk.content.substring(0, 1000), // Limit content to avoid long prompts
|
|
chunk.headings.join(' '),
|
|
JSON.stringify(chunk.frontmatter),
|
|
].filter(Boolean);
|
|
|
|
return parts.join('\n\n');
|
|
}
|
|
}
|