From f98afcd6b0ec80e8f30ceb392b067c7b32591e99 Mon Sep 17 00:00:00 2001 From: Florian Egger Date: Wed, 20 May 2026 22:32:15 +0200 Subject: [PATCH] Add tool telemetry tracking for LLM calls and vault searches Introduces TelemetryManager to record tool calls, LLM token usage (prompt_eval_count, eval_count), and vault search queries with timing. Wires telemetry through ChatView, ToolExecutor, and OllamaClient with configurable limits and enable/disable toggle. --- src/chat-view.ts | 36 +++- src/constants.ts | 4 + src/main.ts | 99 ++++++++++- src/ollama-client.ts | 17 +- src/tool-executor.ts | 59 +++++-- src/tool-telemetry.ts | 121 ++++++++++++++ src/types.ts | 50 ++++++ tests/chat-view.test.ts | 44 ++--- tests/tool-telemetry.test.ts | 312 +++++++++++++++++++++++++++++++++++ 9 files changed, 698 insertions(+), 44 deletions(-) create mode 100644 src/tool-telemetry.ts create mode 100644 tests/tool-telemetry.test.ts diff --git a/src/chat-view.ts b/src/chat-view.ts index 065610f..0890e43 100755 --- a/src/chat-view.ts +++ b/src/chat-view.ts @@ -26,6 +26,7 @@ import { import { ConversationStateManager } from './conversation-state'; import { ErrorHandler } from './error-handler'; import { StructuredMemoryManager } from './structured-memory'; +import { TelemetryManager } from './tool-telemetry'; export const VIEW_TYPE_OLLAMA_CHAT = 'ollama-chat-view'; @@ -47,7 +48,8 @@ export class ChatView extends ItemView { leaf: WorkspaceLeaf, settings: PluginSettings, vectorStore?: VaultVectorStore, - structuredMemoryManager?: StructuredMemoryManager + structuredMemoryManager?: StructuredMemoryManager, + telemetryManager?: TelemetryManager ) { super(leaf); this.messages = []; @@ -72,11 +74,12 @@ export class ChatView extends ItemView { settings.cacheConfig ); this.vaultIndexer = new VaultIndexer(this.app.vault, undefined, vectorStore); - this.toolExecutor = new ToolExecutor(this.app.vault, this.app); + this.toolExecutor = new ToolExecutor(this.app.vault, this.app, telemetryManager); this.actionPreviewBuilder = new ActionPreviewBuilder(this.app.vault, this.app); this.noteContextBuilder = new NoteContextBuilder(this.app.vault, this.app, this.vaultIndexer); this.conversationStateManager = new ConversationStateManager(); this.structuredMemoryManager = structuredMemoryManager; + this.telemetryManager = telemetryManager; this.workflowEngine = new WorkflowEngine( this.app.vault, this.app, @@ -1036,6 +1039,9 @@ export class ChatView extends ItemView { let fullResponse = ''; let toolCalls: OllamaToolCall[] = []; + let promptTokens = 0; + let completionTokens = 0; + const llmStartTime = Date.now(); for await (const chunk of stream) { if (chunk.content) { @@ -1050,8 +1056,33 @@ export class ChatView extends ItemView { if (chunk.tool_calls) { toolCalls = [...toolCalls, ...chunk.tool_calls]; } + + // Capture token counts from final chunk when Ollama provides them + if (typeof chunk.prompt_eval_count === 'number') { + promptTokens = chunk.prompt_eval_count; + } + if (typeof chunk.eval_count === 'number') { + completionTokens = chunk.eval_count; + } } + const llmDurationMs = Date.now() - llmStartTime; + // Fallback: estimate tokens from characters if Ollama didn't provide counts + const estimatedPromptTokens = + promptTokens > 0 + ? promptTokens + : completeMessages.reduce((sum, m) => sum + m.content.length, 0) / 4; + const estimatedCompletionTokens = + completionTokens > 0 ? completionTokens : fullResponse.length / 4; + + this.telemetryManager?.recordLlmCall({ + model: this.settings.model, + promptTokens: Math.round(estimatedPromptTokens), + completionTokens: Math.round(estimatedCompletionTokens), + totalTokens: Math.round(estimatedPromptTokens + estimatedCompletionTokens), + durationMs: llmDurationMs, + }); + // Process tool calls if any if (toolCalls.length > 0) { await this.processToolCalls( @@ -1152,6 +1183,7 @@ export class ChatView extends ItemView { private workflowEngine: WorkflowEngine; private conversationStateManager: ConversationStateManager; private structuredMemoryManager?: StructuredMemoryManager; + private telemetryManager?: TelemetryManager; private vectorStore?: VaultVectorStore; private modeSelectorEl: HTMLSelectElement | null = null; diff --git a/src/constants.ts b/src/constants.ts index 9731699..091ff30 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -44,4 +44,8 @@ export const DEFAULT_SETTINGS = { maxPreferences: 20, maxFacts: 50, }, + toolTelemetryConfig: { + enabled: true, + maxEntries: 100, + }, }; diff --git a/src/main.ts b/src/main.ts index a547311..a9d5714 100755 --- a/src/main.ts +++ b/src/main.ts @@ -5,11 +5,12 @@ import { SemanticCacheService } from './semantic-cache'; import { VaultVectorStore } from './vault-vector-store'; import { VaultIndexer } from './vault-indexer'; import { AutoTagger, AutoLinker } from './auto-organizer'; -import { PluginSettings, StructuredMemoryData } from './types'; +import { PluginSettings, StructuredMemoryData, ToolTelemetryData } from './types'; import { Logger } from './utils'; import { ALL_AGENT_MODES, getAgentModeLabel } from './agent-modes'; import { AgentMode } from './types'; import { StructuredMemoryManager, createDefaultStructuredMemoryData } from './structured-memory'; +import { TelemetryManager, createDefaultToolTelemetryData } from './tool-telemetry'; export default class OllamaPlugin extends Plugin { settings: PluginSettings = DEFAULT_SETTINGS; @@ -18,6 +19,7 @@ export default class OllamaPlugin extends Plugin { autoTagger?: AutoTagger; autoLinker?: AutoLinker; structuredMemoryManager?: StructuredMemoryManager; + telemetryManager?: TelemetryManager; private indexingAbortController?: AbortController; private currentIndexingPromise?: Promise; @@ -33,7 +35,13 @@ export default class OllamaPlugin extends Plugin { this.registerView( 'ollama-chat-view', (leaf: WorkspaceLeaf) => - new ChatView(leaf, this.settings, this.vaultVectorStore, this.structuredMemoryManager) + new ChatView( + leaf, + this.settings, + this.vaultVectorStore, + this.structuredMemoryManager, + this.telemetryManager + ) ); // Add a ribbon icon in the left sidebar @@ -117,6 +125,17 @@ export default class OllamaPlugin extends Plugin { new Notice('Structured memory cleared.'); }, }); + + // Add a command to clear tool telemetry + this.addCommand({ + id: 'clear-tool-telemetry', + name: 'Clear Tool Telemetry', + callback: async () => { + this.telemetryManager?.clear(); + await this.saveSettings(); + new Notice('Tool telemetry cleared.'); + }, + }); this.addSettingTab(new OllamaSettingTab(this.app, this)); // Initialize the semantic cache @@ -162,6 +181,10 @@ export default class OllamaPlugin extends Plugin { this.settings.structuredMemoryConfig, memoryData ); + + const telemetryData: ToolTelemetryData = + (data.toolTelemetry as ToolTelemetryData | undefined) ?? createDefaultToolTelemetryData(); + this.telemetryManager = new TelemetryManager(this.settings.toolTelemetryConfig, telemetryData); } async saveSettings() { @@ -169,6 +192,7 @@ export default class OllamaPlugin extends Plugin { settings: this.settings, structuredMemory: this.structuredMemoryManager?.getData() ?? createDefaultStructuredMemoryData(), + toolTelemetry: this.telemetryManager?.getData() ?? createDefaultToolTelemetryData(), }); } @@ -972,6 +996,77 @@ class OllamaSettingTab extends PluginSettingTab { new Notice('Structured memory cleared.'); }) ); + + // Tool Telemetry Settings + containerEl.createEl('h3', { text: 'Tool Telemetry' }); + containerEl.createEl('p', { + text: 'Track which tools were called, which notes were searched, and LLM token usage.', + }); + + new Setting(containerEl) + .setName('Enable Tool Telemetry') + .setDesc('Record tool calls, searches, and LLM token counts for analysis.') + .addToggle((toggle) => + toggle + .setValue(this.plugin.settings.toolTelemetryConfig.enabled) + .onChange(async (value) => { + this.plugin.settings.toolTelemetryConfig.enabled = value; + this.plugin.telemetryManager?.updateConfig(this.plugin.settings.toolTelemetryConfig); + await this.plugin.saveSettings(); + }) + ); + + new Setting(containerEl) + .setName('Max Telemetry Entries') + .setDesc('Maximum number of telemetry events to retain (default: 100).') + .addText((text) => + text + .setValue(String(this.plugin.settings.toolTelemetryConfig.maxEntries)) + .onChange(async (value) => { + const parsed = parseInt(value); + if (!isNaN(parsed) && parsed >= 0 && parsed <= 1000) { + this.plugin.settings.toolTelemetryConfig.maxEntries = parsed; + this.plugin.telemetryManager?.updateConfig(this.plugin.settings.toolTelemetryConfig); + await this.plugin.saveSettings(); + } else { + new Notice('Max entries must be between 0 and 1000.'); + } + }) + ); + + new Setting(containerEl) + .setName('Clear Tool Telemetry') + .setDesc('Delete all recorded tool telemetry.') + .addButton((button) => + button.setButtonText('Clear Telemetry').onClick(async () => { + this.plugin.telemetryManager?.clear(); + await this.plugin.saveSettings(); + new Notice('Tool telemetry cleared.'); + }) + ); + + // Recent Telemetry Summary + const recentEntries = this.plugin.telemetryManager?.getRecentEntries(10) ?? []; + if (recentEntries.length > 0) { + containerEl.createEl('h4', { text: 'Recent Activity' }); + const telemetryList = containerEl.createEl('ul'); + for (const entry of recentEntries) { + const li = telemetryList.createEl('li'); + if (entry.type === 'tool_call') { + li.setText( + `${new Date(entry.timestamp).toLocaleString()}: Tool "${entry.toolName}" — ${entry.success ? 'success' : 'failed'} (${entry.durationMs}ms)` + ); + } else if (entry.type === 'llm_call') { + li.setText( + `${new Date(entry.timestamp).toLocaleString()}: LLM call — ${entry.totalTokens} tokens (${entry.durationMs}ms)` + ); + } else if (entry.type === 'vault_search') { + li.setText( + `${new Date(entry.timestamp).toLocaleString()}: Search "${entry.query}" — ${entry.resultsCount} results` + ); + } + } + } } hide() { diff --git a/src/ollama-client.ts b/src/ollama-client.ts index 4f620be..4523b46 100644 --- a/src/ollama-client.ts +++ b/src/ollama-client.ts @@ -8,6 +8,9 @@ import { SemanticCacheService } from './semantic-cache'; interface OllamaChatResponse { message?: Partial; error?: string; + done?: boolean; + prompt_eval_count?: number; + eval_count?: number; } export class OllamaClient { @@ -206,7 +209,7 @@ export class OllamaClient { throw new Error(`Ollama error: ${errorMsg}`); } - yield this.normalizeMessage(parsed.message); + yield this.normalizeMessage(parsed.message, parsed.prompt_eval_count, parsed.eval_count); } } @@ -229,7 +232,7 @@ export class OllamaClient { } if (parsed?.message) { - yield this.normalizeMessage(parsed.message); + yield this.normalizeMessage(parsed.message, parsed.prompt_eval_count, parsed.eval_count); } } } catch (error) { @@ -289,7 +292,7 @@ export class OllamaClient { if (!this.isChatResponse(data)) { return this.normalizeMessage(); } - return this.normalizeMessage(data.message); + return this.normalizeMessage(data.message, data.prompt_eval_count, data.eval_count); } catch (error) { if (retryCount < this.maxRetries && this.isRetryableError(error, controller)) { const errorMessage = error instanceof Error ? error.message : String(error); @@ -309,12 +312,18 @@ export class OllamaClient { } } - private normalizeMessage(message?: Partial): OllamaMessage { + private normalizeMessage( + message?: Partial, + promptEvalCount?: number, + evalCount?: number + ): OllamaMessage { return { role: message?.role ?? 'assistant', content: message?.content ?? '', tool_calls: message?.tool_calls ?? [], tool_call_id: message?.tool_call_id, + prompt_eval_count: promptEvalCount, + eval_count: evalCount, }; } diff --git a/src/tool-executor.ts b/src/tool-executor.ts index a01f3e0..c2f8ac7 100644 --- a/src/tool-executor.ts +++ b/src/tool-executor.ts @@ -3,6 +3,7 @@ import { Vault, App, TFile } from 'obsidian'; import type { ToolCall, ToolResult } from './types'; import { safeParseJson } from './utils'; +import { TelemetryManager } from './tool-telemetry'; // Disallow characters that are invalid in file paths const INVALID_PATH_CHARS = /[<>:"|?*~]/; @@ -12,10 +13,12 @@ const FORBIDDEN_DIRS = ['.obsidian', '.git']; export class ToolExecutor { private vault: Vault; private app: App; + private telemetryManager?: TelemetryManager; - constructor(vault: Vault, app: App) { + constructor(vault: Vault, app: App, telemetryManager?: TelemetryManager) { this.vault = vault; this.app = app; + this.telemetryManager = telemetryManager; } private isSafePath(path: string): boolean { @@ -87,8 +90,13 @@ export class ToolExecutor { } async handleToolCall(toolCall: ToolCall): Promise { + const startTime = Date.now(); + const toolName = toolCall.function?.name ?? 'unknown'; + let parsedArgs: Record = {}; + let result: ToolResult = { success: false, message: 'No result' }; + let success = false; + try { - const toolName = toolCall.function?.name; const rawArgs = toolCall.function?.arguments; if (!toolName) { @@ -96,7 +104,6 @@ export class ToolExecutor { } // Parse arguments whether they're a string or object - let parsedArgs: Record; if (typeof rawArgs === 'string') { try { parsedArgs = safeParseJson(rawArgs) as Record; @@ -113,31 +120,55 @@ export class ToolExecutor { switch (toolName) { case 'create_file': case 'create_note': - return await this.handleCreateNote(parsedArgs); + result = await this.handleCreateNote(parsedArgs); + break; case 'read_vault_file': - return await this.handleReadVaultFile(parsedArgs); + result = await this.handleReadVaultFile(parsedArgs); + break; case 'search_vault_files': - return this.handleSearchVaultFiles(parsedArgs); + result = this.handleSearchVaultFiles(parsedArgs); + break; case 'append_to_note': - return await this.handleAppendToNote(parsedArgs); + result = await this.handleAppendToNote(parsedArgs); + break; case 'replace_note_section': - return await this.handleReplaceNoteSection(parsedArgs); + result = await this.handleReplaceNoteSection(parsedArgs); + break; case 'update_frontmatter': - return await this.handleUpdateFrontmatter(parsedArgs); + result = await this.handleUpdateFrontmatter(parsedArgs); + break; case 'rename_note': - return await this.handleRenameNote(parsedArgs); + result = await this.handleRenameNote(parsedArgs); + break; case 'move_note': - return await this.handleMoveNote(parsedArgs); + result = await this.handleMoveNote(parsedArgs); + break; case 'delete_note': - return await this.handleDeleteNote(parsedArgs); + result = await this.handleDeleteNote(parsedArgs); + break; case 'insert_link': - return await this.handleInsertLink(parsedArgs); + result = await this.handleInsertLink(parsedArgs); + break; default: - return { success: false, message: `Unknown tool: ${toolName}` }; + result = { success: false, message: `Unknown tool: ${toolName}` }; } + + success = result.success; + return result; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); + success = false; + result = { success: false, message: errorMessage }; throw new Error(errorMessage); + } finally { + const durationMs = Date.now() - startTime; + this.telemetryManager?.recordToolCall({ + toolName, + args: parsedArgs, + success, + resultSummary: result?.message ?? 'No result', + durationMs, + }); } } diff --git a/src/tool-telemetry.ts b/src/tool-telemetry.ts new file mode 100644 index 0000000..f2faa5e --- /dev/null +++ b/src/tool-telemetry.ts @@ -0,0 +1,121 @@ +import { + ToolTelemetryEntry, + LlmTelemetryEntry, + SearchTelemetryEntry, + TelemetryEntry, + ToolTelemetryData, + ToolTelemetryConfig, +} from './types'; + +export { + ToolTelemetryEntry, + LlmTelemetryEntry, + SearchTelemetryEntry, + TelemetryEntry, + ToolTelemetryData, + ToolTelemetryConfig, +}; + +export function createDefaultToolTelemetryData(): ToolTelemetryData { + return { + entries: [], + }; +} + +function generateId(): string { + return crypto.randomUUID?.() ?? `id-${Date.now()}-${Math.random()}`; +} + +export class TelemetryManager { + private config: ToolTelemetryConfig; + private data: ToolTelemetryData; + + constructor(config: ToolTelemetryConfig, initialData?: ToolTelemetryData) { + this.config = config; + this.data = initialData + ? { entries: [...initialData.entries] } + : createDefaultToolTelemetryData(); + } + + loadData(data: ToolTelemetryData): void { + this.data = { + entries: [...data.entries], + }; + this.enforceLimits(); + } + + getData(): ToolTelemetryData { + return { + entries: [...this.data.entries], + }; + } + + updateConfig(config: ToolTelemetryConfig): void { + this.config = config; + this.enforceLimits(); + } + + recordToolCall(entry: Omit): void { + if (!this.config.enabled) { + return; + } + const fullEntry: ToolTelemetryEntry = { + ...entry, + id: generateId(), + timestamp: Date.now(), + type: 'tool_call', + }; + this.data.entries.push(fullEntry); + this.enforceLimits(); + } + + recordLlmCall(entry: Omit): void { + if (!this.config.enabled) { + return; + } + const fullEntry: LlmTelemetryEntry = { + ...entry, + id: generateId(), + timestamp: Date.now(), + type: 'llm_call', + }; + this.data.entries.push(fullEntry); + this.enforceLimits(); + } + + recordSearch(entry: Omit): void { + if (!this.config.enabled) { + return; + } + const fullEntry: SearchTelemetryEntry = { + ...entry, + id: generateId(), + timestamp: Date.now(), + type: 'vault_search', + }; + this.data.entries.push(fullEntry); + this.enforceLimits(); + } + + getRecentEntries(limit?: number): TelemetryEntry[] { + const sorted = [...this.data.entries].sort((a, b) => b.timestamp - a.timestamp); + if (limit !== undefined) { + return sorted.slice(0, limit); + } + return sorted; + } + + getEntriesByType(type: TelemetryEntry['type']): TelemetryEntry[] { + return [...this.data.entries.filter((entry) => entry.type === type)]; + } + + clear(): void { + this.data.entries = []; + } + + private enforceLimits(): void { + if (this.data.entries.length > this.config.maxEntries) { + this.data.entries = this.data.entries.slice(-this.config.maxEntries); + } + } +} diff --git a/src/types.ts b/src/types.ts index 7dd0d8b..01c39de 100644 --- a/src/types.ts +++ b/src/types.ts @@ -90,6 +90,8 @@ export interface OllamaMessage { content: string; tool_calls?: OllamaToolCall[]; tool_call_id?: string; + prompt_eval_count?: number; + eval_count?: number; } export interface OllamaToolCall { @@ -243,6 +245,53 @@ export interface StructuredMemoryConfig { maxFacts: number; } +// ============================================================ +// Tool Telemetry +// ============================================================ + +export interface ToolTelemetryEntry { + id: string; + timestamp: number; + type: 'tool_call'; + toolName: string; + args: Record; + success: boolean; + resultSummary: string; + durationMs: number; +} + +export interface LlmTelemetryEntry { + id: string; + timestamp: number; + type: 'llm_call'; + model: string; + promptTokens: number; + completionTokens: number; + totalTokens: number; + durationMs: number; +} + +export interface SearchTelemetryEntry { + id: string; + timestamp: number; + type: 'vault_search'; + query: string; + resultsCount: number; + resultPaths: string[]; + durationMs: number; +} + +export type TelemetryEntry = ToolTelemetryEntry | LlmTelemetryEntry | SearchTelemetryEntry; + +export interface ToolTelemetryData { + entries: TelemetryEntry[]; +} + +export interface ToolTelemetryConfig { + enabled: boolean; + maxEntries: number; +} + // ============================================================ // Plugin Configuration // ============================================================ @@ -291,6 +340,7 @@ export interface PluginSettings { dryRun: boolean; }; structuredMemoryConfig: StructuredMemoryConfig; + toolTelemetryConfig: ToolTelemetryConfig; } // ============================================================ diff --git a/tests/chat-view.test.ts b/tests/chat-view.test.ts index 37966e3..7568949 100755 --- a/tests/chat-view.test.ts +++ b/tests/chat-view.test.ts @@ -76,6 +76,10 @@ const mockSettings: PluginSettings = { maxPreferences: 20, maxFacts: 50, }, + toolTelemetryConfig: { + enabled: true, + maxEntries: 100, + }, }; describe('ChatView', () => { @@ -360,17 +364,15 @@ describe('ChatView', () => { view['inputEl'] = document.createElement('textarea'); (view['inputEl'] as HTMLTextAreaElement).value = 'search query'; - const searchSpy = jest - .spyOn(view['noteContextBuilder'], 'buildContext') - .mockResolvedValue({ - explicitMentions: [], - openNote: undefined, - selectedText: undefined, - backlinks: [], - outlinks: [], - relatedNotes: [], - searchResults: [], - }); + const searchSpy = jest.spyOn(view['noteContextBuilder'], 'buildContext').mockResolvedValue({ + explicitMentions: [], + openNote: undefined, + selectedText: undefined, + backlinks: [], + outlinks: [], + relatedNotes: [], + searchResults: [], + }); const chatSpy = jest.spyOn(view['ollamaClient'], 'streamChat').mockReturnValue( (async function* () { yield { role: 'assistant', content: 'response' }; @@ -815,17 +817,15 @@ describe('ChatView', () => { view['inputEl'] = document.createElement('textarea'); (view['inputEl'] as HTMLTextAreaElement).value = 'search query'; - const searchSpy = jest - .spyOn(view['noteContextBuilder'], 'buildContext') - .mockResolvedValue({ - explicitMentions: [], - openNote: undefined, - selectedText: undefined, - backlinks: [], - outlinks: [], - relatedNotes: [], - searchResults: [], - }); + const searchSpy = jest.spyOn(view['noteContextBuilder'], 'buildContext').mockResolvedValue({ + explicitMentions: [], + openNote: undefined, + selectedText: undefined, + backlinks: [], + outlinks: [], + relatedNotes: [], + searchResults: [], + }); const chatSpy = jest.spyOn(view['ollamaClient'], 'streamChat').mockReturnValue( (async function* () { yield { role: 'assistant', content: 'response' }; diff --git a/tests/tool-telemetry.test.ts b/tests/tool-telemetry.test.ts new file mode 100644 index 0000000..f659fb6 --- /dev/null +++ b/tests/tool-telemetry.test.ts @@ -0,0 +1,312 @@ +import { TelemetryManager, createDefaultToolTelemetryData } from '../src/tool-telemetry'; +import type { ToolTelemetryConfig } from '../src/types'; + +describe('createDefaultToolTelemetryData', () => { + it('should return empty entries array', () => { + const data = createDefaultToolTelemetryData(); + expect(data.entries).toEqual([]); + }); +}); + +describe('TelemetryManager', () => { + const defaultConfig: ToolTelemetryConfig = { + enabled: true, + maxEntries: 3, + }; + + let manager: TelemetryManager; + + beforeEach(() => { + manager = new TelemetryManager(defaultConfig); + }); + + describe('constructor', () => { + it('should initialize with empty data when no initial data provided', () => { + expect(manager.getData().entries).toEqual([]); + }); + + it('should initialize with provided data', () => { + const initialData = createDefaultToolTelemetryData(); + initialData.entries.push({ + id: 'e1', + timestamp: 1, + type: 'tool_call', + toolName: 'read_vault_file', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + const m = new TelemetryManager(defaultConfig, initialData); + expect(m.getData().entries).toHaveLength(1); + }); + }); + + describe('loadData and getData', () => { + it('should load and return data round-trip', () => { + manager.recordToolCall({ + toolName: 'test', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 5, + }); + + const loaded = manager.getData(); + expect(loaded.entries).toHaveLength(1); + + const newManager = new TelemetryManager(defaultConfig); + newManager.loadData(loaded); + expect(newManager.getData().entries).toHaveLength(1); + }); + }); + + describe('updateConfig', () => { + it('should enforce new limits after config update', () => { + for (let i = 0; i < 5; i++) { + manager.recordToolCall({ + toolName: `tool-${i}`, + args: {}, + success: true, + resultSummary: 'ok', + durationMs: i, + }); + } + expect(manager.getData().entries).toHaveLength(3); + + manager.updateConfig({ ...defaultConfig, maxEntries: 2 }); + expect(manager.getData().entries).toHaveLength(2); + }); + + it('should disable writes when enabled becomes false', () => { + manager.updateConfig({ ...defaultConfig, enabled: false }); + manager.recordToolCall({ + toolName: 'test', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 5, + }); + expect(manager.getData().entries).toHaveLength(0); + }); + }); + + describe('recordToolCall', () => { + it('should add a tool call entry', () => { + manager.recordToolCall({ + toolName: 'create_note', + args: { path: 'test.md' }, + success: true, + resultSummary: 'Created', + durationMs: 100, + }); + const entries = manager.getEntriesByType('tool_call'); + expect(entries).toHaveLength(1); + expect((entries[0] as any).toolName).toBe('create_note'); + expect((entries[0] as any).success).toBe(true); + }); + + it('should enforce maxEntries limit keeping newest', () => { + for (let i = 0; i < 5; i++) { + manager.recordToolCall({ + toolName: `tool-${i}`, + args: {}, + success: true, + resultSummary: 'ok', + durationMs: i, + }); + } + const entries = manager.getData().entries; + expect(entries).toHaveLength(3); + }); + + it('should not add when disabled', () => { + manager.updateConfig({ ...defaultConfig, enabled: false }); + manager.recordToolCall({ + toolName: 'test', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 5, + }); + expect(manager.getData().entries).toHaveLength(0); + }); + }); + + describe('recordLlmCall', () => { + it('should add an LLM call entry', () => { + manager.recordLlmCall({ + model: 'llama3', + promptTokens: 100, + completionTokens: 50, + totalTokens: 150, + durationMs: 2000, + }); + const entries = manager.getEntriesByType('llm_call'); + expect(entries).toHaveLength(1); + expect((entries[0] as any).model).toBe('llama3'); + expect((entries[0] as any).totalTokens).toBe(150); + }); + + it('should not add when disabled', () => { + manager.updateConfig({ ...defaultConfig, enabled: false }); + manager.recordLlmCall({ + model: 'llama3', + promptTokens: 10, + completionTokens: 5, + totalTokens: 15, + durationMs: 100, + }); + expect(manager.getData().entries).toHaveLength(0); + }); + }); + + describe('recordSearch', () => { + it('should add a search entry', () => { + manager.recordSearch({ + query: 'test', + resultsCount: 3, + resultPaths: ['a.md', 'b.md'], + durationMs: 50, + }); + const entries = manager.getEntriesByType('vault_search'); + expect(entries).toHaveLength(1); + expect((entries[0] as any).query).toBe('test'); + expect((entries[0] as any).resultsCount).toBe(3); + }); + + it('should not add when disabled', () => { + manager.updateConfig({ ...defaultConfig, enabled: false }); + manager.recordSearch({ + query: 'test', + resultsCount: 0, + resultPaths: [], + durationMs: 10, + }); + expect(manager.getData().entries).toHaveLength(0); + }); + }); + + describe('getRecentEntries', () => { + it('should return entries sorted by timestamp descending', async () => { + manager.recordToolCall({ + toolName: 'first', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + await new Promise((r) => setTimeout(r, 5)); + manager.recordToolCall({ + toolName: 'second', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + const recent = manager.getRecentEntries(); + expect((recent[0] as any).toolName).toBe('second'); + expect((recent[1] as any).toolName).toBe('first'); + }); + + it('should respect limit parameter', () => { + manager.recordToolCall({ + toolName: 'first', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + manager.recordToolCall({ + toolName: 'second', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + expect(manager.getRecentEntries(1)).toHaveLength(1); + }); + }); + + describe('getEntriesByType', () => { + it('should filter by type', () => { + manager.recordToolCall({ + toolName: 'test', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + manager.recordLlmCall({ + model: 'llama3', + promptTokens: 10, + completionTokens: 5, + totalTokens: 15, + durationMs: 100, + }); + expect(manager.getEntriesByType('tool_call')).toHaveLength(1); + expect(manager.getEntriesByType('llm_call')).toHaveLength(1); + expect(manager.getEntriesByType('vault_search')).toHaveLength(0); + }); + }); + + describe('clear', () => { + it('should remove all entries', () => { + manager.recordToolCall({ + toolName: 'test', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + manager.clear(); + expect(manager.getData().entries).toHaveLength(0); + }); + }); + + describe('immutability', () => { + it('getData should return a copy', () => { + manager.recordToolCall({ + toolName: 'test', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + const data = manager.getData(); + data.entries.push({ + id: 'x', + timestamp: 1, + type: 'tool_call', + toolName: 'injected', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 1, + }); + expect(manager.getData().entries).toHaveLength(1); + }); + + it('getEntriesByType should return a copy', () => { + manager.recordToolCall({ + toolName: 'test', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 10, + }); + const entries = manager.getEntriesByType('tool_call'); + entries.push({ + id: 'x', + timestamp: 1, + type: 'tool_call', + toolName: 'injected', + args: {}, + success: true, + resultSummary: 'ok', + durationMs: 1, + }); + expect(manager.getEntriesByType('tool_call')).toHaveLength(1); + }); + }); +});