Add tool telemetry tracking for LLM calls and vault searches
Introduces TelemetryManager to record tool calls, LLM token usage (prompt_eval_count, eval_count), and vault search queries with timing. Wires telemetry through ChatView, ToolExecutor, and OllamaClient with configurable limits and enable/disable toggle.
This commit is contained in:
@@ -0,0 +1,312 @@
|
||||
import { TelemetryManager, createDefaultToolTelemetryData } from '../src/tool-telemetry';
|
||||
import type { ToolTelemetryConfig } from '../src/types';
|
||||
|
||||
describe('createDefaultToolTelemetryData', () => {
|
||||
it('should return empty entries array', () => {
|
||||
const data = createDefaultToolTelemetryData();
|
||||
expect(data.entries).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('TelemetryManager', () => {
|
||||
const defaultConfig: ToolTelemetryConfig = {
|
||||
enabled: true,
|
||||
maxEntries: 3,
|
||||
};
|
||||
|
||||
let manager: TelemetryManager;
|
||||
|
||||
beforeEach(() => {
|
||||
manager = new TelemetryManager(defaultConfig);
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should initialize with empty data when no initial data provided', () => {
|
||||
expect(manager.getData().entries).toEqual([]);
|
||||
});
|
||||
|
||||
it('should initialize with provided data', () => {
|
||||
const initialData = createDefaultToolTelemetryData();
|
||||
initialData.entries.push({
|
||||
id: 'e1',
|
||||
timestamp: 1,
|
||||
type: 'tool_call',
|
||||
toolName: 'read_vault_file',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
const m = new TelemetryManager(defaultConfig, initialData);
|
||||
expect(m.getData().entries).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('loadData and getData', () => {
|
||||
it('should load and return data round-trip', () => {
|
||||
manager.recordToolCall({
|
||||
toolName: 'test',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 5,
|
||||
});
|
||||
|
||||
const loaded = manager.getData();
|
||||
expect(loaded.entries).toHaveLength(1);
|
||||
|
||||
const newManager = new TelemetryManager(defaultConfig);
|
||||
newManager.loadData(loaded);
|
||||
expect(newManager.getData().entries).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('updateConfig', () => {
|
||||
it('should enforce new limits after config update', () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
manager.recordToolCall({
|
||||
toolName: `tool-${i}`,
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: i,
|
||||
});
|
||||
}
|
||||
expect(manager.getData().entries).toHaveLength(3);
|
||||
|
||||
manager.updateConfig({ ...defaultConfig, maxEntries: 2 });
|
||||
expect(manager.getData().entries).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should disable writes when enabled becomes false', () => {
|
||||
manager.updateConfig({ ...defaultConfig, enabled: false });
|
||||
manager.recordToolCall({
|
||||
toolName: 'test',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 5,
|
||||
});
|
||||
expect(manager.getData().entries).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('recordToolCall', () => {
|
||||
it('should add a tool call entry', () => {
|
||||
manager.recordToolCall({
|
||||
toolName: 'create_note',
|
||||
args: { path: 'test.md' },
|
||||
success: true,
|
||||
resultSummary: 'Created',
|
||||
durationMs: 100,
|
||||
});
|
||||
const entries = manager.getEntriesByType('tool_call');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect((entries[0] as any).toolName).toBe('create_note');
|
||||
expect((entries[0] as any).success).toBe(true);
|
||||
});
|
||||
|
||||
it('should enforce maxEntries limit keeping newest', () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
manager.recordToolCall({
|
||||
toolName: `tool-${i}`,
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: i,
|
||||
});
|
||||
}
|
||||
const entries = manager.getData().entries;
|
||||
expect(entries).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('should not add when disabled', () => {
|
||||
manager.updateConfig({ ...defaultConfig, enabled: false });
|
||||
manager.recordToolCall({
|
||||
toolName: 'test',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 5,
|
||||
});
|
||||
expect(manager.getData().entries).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('recordLlmCall', () => {
|
||||
it('should add an LLM call entry', () => {
|
||||
manager.recordLlmCall({
|
||||
model: 'llama3',
|
||||
promptTokens: 100,
|
||||
completionTokens: 50,
|
||||
totalTokens: 150,
|
||||
durationMs: 2000,
|
||||
});
|
||||
const entries = manager.getEntriesByType('llm_call');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect((entries[0] as any).model).toBe('llama3');
|
||||
expect((entries[0] as any).totalTokens).toBe(150);
|
||||
});
|
||||
|
||||
it('should not add when disabled', () => {
|
||||
manager.updateConfig({ ...defaultConfig, enabled: false });
|
||||
manager.recordLlmCall({
|
||||
model: 'llama3',
|
||||
promptTokens: 10,
|
||||
completionTokens: 5,
|
||||
totalTokens: 15,
|
||||
durationMs: 100,
|
||||
});
|
||||
expect(manager.getData().entries).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('recordSearch', () => {
|
||||
it('should add a search entry', () => {
|
||||
manager.recordSearch({
|
||||
query: 'test',
|
||||
resultsCount: 3,
|
||||
resultPaths: ['a.md', 'b.md'],
|
||||
durationMs: 50,
|
||||
});
|
||||
const entries = manager.getEntriesByType('vault_search');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect((entries[0] as any).query).toBe('test');
|
||||
expect((entries[0] as any).resultsCount).toBe(3);
|
||||
});
|
||||
|
||||
it('should not add when disabled', () => {
|
||||
manager.updateConfig({ ...defaultConfig, enabled: false });
|
||||
manager.recordSearch({
|
||||
query: 'test',
|
||||
resultsCount: 0,
|
||||
resultPaths: [],
|
||||
durationMs: 10,
|
||||
});
|
||||
expect(manager.getData().entries).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRecentEntries', () => {
|
||||
it('should return entries sorted by timestamp descending', async () => {
|
||||
manager.recordToolCall({
|
||||
toolName: 'first',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
manager.recordToolCall({
|
||||
toolName: 'second',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
const recent = manager.getRecentEntries();
|
||||
expect((recent[0] as any).toolName).toBe('second');
|
||||
expect((recent[1] as any).toolName).toBe('first');
|
||||
});
|
||||
|
||||
it('should respect limit parameter', () => {
|
||||
manager.recordToolCall({
|
||||
toolName: 'first',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
manager.recordToolCall({
|
||||
toolName: 'second',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
expect(manager.getRecentEntries(1)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getEntriesByType', () => {
|
||||
it('should filter by type', () => {
|
||||
manager.recordToolCall({
|
||||
toolName: 'test',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
manager.recordLlmCall({
|
||||
model: 'llama3',
|
||||
promptTokens: 10,
|
||||
completionTokens: 5,
|
||||
totalTokens: 15,
|
||||
durationMs: 100,
|
||||
});
|
||||
expect(manager.getEntriesByType('tool_call')).toHaveLength(1);
|
||||
expect(manager.getEntriesByType('llm_call')).toHaveLength(1);
|
||||
expect(manager.getEntriesByType('vault_search')).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('clear', () => {
|
||||
it('should remove all entries', () => {
|
||||
manager.recordToolCall({
|
||||
toolName: 'test',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
manager.clear();
|
||||
expect(manager.getData().entries).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('immutability', () => {
|
||||
it('getData should return a copy', () => {
|
||||
manager.recordToolCall({
|
||||
toolName: 'test',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
const data = manager.getData();
|
||||
data.entries.push({
|
||||
id: 'x',
|
||||
timestamp: 1,
|
||||
type: 'tool_call',
|
||||
toolName: 'injected',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 1,
|
||||
});
|
||||
expect(manager.getData().entries).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('getEntriesByType should return a copy', () => {
|
||||
manager.recordToolCall({
|
||||
toolName: 'test',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 10,
|
||||
});
|
||||
const entries = manager.getEntriesByType('tool_call');
|
||||
entries.push({
|
||||
id: 'x',
|
||||
timestamp: 1,
|
||||
type: 'tool_call',
|
||||
toolName: 'injected',
|
||||
args: {},
|
||||
success: true,
|
||||
resultSummary: 'ok',
|
||||
durationMs: 1,
|
||||
});
|
||||
expect(manager.getEntriesByType('tool_call')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user