Files
obsidian_ollama/tests/tool-telemetry.test.ts
T
fegger f98afcd6b0 Add tool telemetry tracking for LLM calls and vault searches
Introduces TelemetryManager to record tool calls, LLM token usage
(prompt_eval_count, eval_count), and vault search queries with timing.
Wires telemetry through ChatView, ToolExecutor, and OllamaClient with
configurable limits and enable/disable toggle.
2026-05-20 22:32:15 +02:00

313 lines
8.5 KiB
TypeScript

import { TelemetryManager, createDefaultToolTelemetryData } from '../src/tool-telemetry';
import type { ToolTelemetryConfig } from '../src/types';
describe('createDefaultToolTelemetryData', () => {
it('should return empty entries array', () => {
const data = createDefaultToolTelemetryData();
expect(data.entries).toEqual([]);
});
});
describe('TelemetryManager', () => {
const defaultConfig: ToolTelemetryConfig = {
enabled: true,
maxEntries: 3,
};
let manager: TelemetryManager;
beforeEach(() => {
manager = new TelemetryManager(defaultConfig);
});
describe('constructor', () => {
it('should initialize with empty data when no initial data provided', () => {
expect(manager.getData().entries).toEqual([]);
});
it('should initialize with provided data', () => {
const initialData = createDefaultToolTelemetryData();
initialData.entries.push({
id: 'e1',
timestamp: 1,
type: 'tool_call',
toolName: 'read_vault_file',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
const m = new TelemetryManager(defaultConfig, initialData);
expect(m.getData().entries).toHaveLength(1);
});
});
describe('loadData and getData', () => {
it('should load and return data round-trip', () => {
manager.recordToolCall({
toolName: 'test',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 5,
});
const loaded = manager.getData();
expect(loaded.entries).toHaveLength(1);
const newManager = new TelemetryManager(defaultConfig);
newManager.loadData(loaded);
expect(newManager.getData().entries).toHaveLength(1);
});
});
describe('updateConfig', () => {
it('should enforce new limits after config update', () => {
for (let i = 0; i < 5; i++) {
manager.recordToolCall({
toolName: `tool-${i}`,
args: {},
success: true,
resultSummary: 'ok',
durationMs: i,
});
}
expect(manager.getData().entries).toHaveLength(3);
manager.updateConfig({ ...defaultConfig, maxEntries: 2 });
expect(manager.getData().entries).toHaveLength(2);
});
it('should disable writes when enabled becomes false', () => {
manager.updateConfig({ ...defaultConfig, enabled: false });
manager.recordToolCall({
toolName: 'test',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 5,
});
expect(manager.getData().entries).toHaveLength(0);
});
});
describe('recordToolCall', () => {
it('should add a tool call entry', () => {
manager.recordToolCall({
toolName: 'create_note',
args: { path: 'test.md' },
success: true,
resultSummary: 'Created',
durationMs: 100,
});
const entries = manager.getEntriesByType('tool_call');
expect(entries).toHaveLength(1);
expect((entries[0] as any).toolName).toBe('create_note');
expect((entries[0] as any).success).toBe(true);
});
it('should enforce maxEntries limit keeping newest', () => {
for (let i = 0; i < 5; i++) {
manager.recordToolCall({
toolName: `tool-${i}`,
args: {},
success: true,
resultSummary: 'ok',
durationMs: i,
});
}
const entries = manager.getData().entries;
expect(entries).toHaveLength(3);
});
it('should not add when disabled', () => {
manager.updateConfig({ ...defaultConfig, enabled: false });
manager.recordToolCall({
toolName: 'test',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 5,
});
expect(manager.getData().entries).toHaveLength(0);
});
});
describe('recordLlmCall', () => {
it('should add an LLM call entry', () => {
manager.recordLlmCall({
model: 'llama3',
promptTokens: 100,
completionTokens: 50,
totalTokens: 150,
durationMs: 2000,
});
const entries = manager.getEntriesByType('llm_call');
expect(entries).toHaveLength(1);
expect((entries[0] as any).model).toBe('llama3');
expect((entries[0] as any).totalTokens).toBe(150);
});
it('should not add when disabled', () => {
manager.updateConfig({ ...defaultConfig, enabled: false });
manager.recordLlmCall({
model: 'llama3',
promptTokens: 10,
completionTokens: 5,
totalTokens: 15,
durationMs: 100,
});
expect(manager.getData().entries).toHaveLength(0);
});
});
describe('recordSearch', () => {
it('should add a search entry', () => {
manager.recordSearch({
query: 'test',
resultsCount: 3,
resultPaths: ['a.md', 'b.md'],
durationMs: 50,
});
const entries = manager.getEntriesByType('vault_search');
expect(entries).toHaveLength(1);
expect((entries[0] as any).query).toBe('test');
expect((entries[0] as any).resultsCount).toBe(3);
});
it('should not add when disabled', () => {
manager.updateConfig({ ...defaultConfig, enabled: false });
manager.recordSearch({
query: 'test',
resultsCount: 0,
resultPaths: [],
durationMs: 10,
});
expect(manager.getData().entries).toHaveLength(0);
});
});
describe('getRecentEntries', () => {
it('should return entries sorted by timestamp descending', async () => {
manager.recordToolCall({
toolName: 'first',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
await new Promise((r) => setTimeout(r, 5));
manager.recordToolCall({
toolName: 'second',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
const recent = manager.getRecentEntries();
expect((recent[0] as any).toolName).toBe('second');
expect((recent[1] as any).toolName).toBe('first');
});
it('should respect limit parameter', () => {
manager.recordToolCall({
toolName: 'first',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
manager.recordToolCall({
toolName: 'second',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
expect(manager.getRecentEntries(1)).toHaveLength(1);
});
});
describe('getEntriesByType', () => {
it('should filter by type', () => {
manager.recordToolCall({
toolName: 'test',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
manager.recordLlmCall({
model: 'llama3',
promptTokens: 10,
completionTokens: 5,
totalTokens: 15,
durationMs: 100,
});
expect(manager.getEntriesByType('tool_call')).toHaveLength(1);
expect(manager.getEntriesByType('llm_call')).toHaveLength(1);
expect(manager.getEntriesByType('vault_search')).toHaveLength(0);
});
});
describe('clear', () => {
it('should remove all entries', () => {
manager.recordToolCall({
toolName: 'test',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
manager.clear();
expect(manager.getData().entries).toHaveLength(0);
});
});
describe('immutability', () => {
it('getData should return a copy', () => {
manager.recordToolCall({
toolName: 'test',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
const data = manager.getData();
data.entries.push({
id: 'x',
timestamp: 1,
type: 'tool_call',
toolName: 'injected',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 1,
});
expect(manager.getData().entries).toHaveLength(1);
});
it('getEntriesByType should return a copy', () => {
manager.recordToolCall({
toolName: 'test',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 10,
});
const entries = manager.getEntriesByType('tool_call');
entries.push({
id: 'x',
timestamp: 1,
type: 'tool_call',
toolName: 'injected',
args: {},
success: true,
resultSummary: 'ok',
durationMs: 1,
});
expect(manager.getEntriesByType('tool_call')).toHaveLength(1);
});
});
});