Add structured memory injection and telemetry tracking
- Introduce buildMessagesWithMemory() to prepend memory context as a system message before LLM calls - Record LLM call telemetry (tokens, duration) for follow-up requests in both streaming and non-streaming paths - Add telemetry coverage for tool execution (success/failure, args, duration) - Update tool-executor tests to verify telemetry integration with TelemetryManager
This commit is contained in:
@@ -2,6 +2,7 @@ import { ToolExecutor } from '../src/tool-executor';
|
||||
import { TFile } from 'obsidian';
|
||||
import { ToolCall, ToolResult } from '../src/types';
|
||||
import { ErrorHandler } from '../src/error-handler';
|
||||
import { TelemetryManager } from '../src/tool-telemetry';
|
||||
|
||||
// Mock Obsidian types
|
||||
interface MockVault {
|
||||
@@ -14,9 +15,9 @@ interface MockVault {
|
||||
delete: (file: any) => Promise<void>;
|
||||
}
|
||||
interface MockApp {
|
||||
metadataCache: {
|
||||
getFileCache: jest.Mock;
|
||||
};
|
||||
metadataCache: {
|
||||
getFileCache: jest.Mock;
|
||||
};
|
||||
// Mock app properties if needed
|
||||
}
|
||||
interface MockNotice {
|
||||
@@ -1242,4 +1243,92 @@ describe('ToolExecutor', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('telemetry integration', () => {
|
||||
let telemetryManager: TelemetryManager;
|
||||
let telemetryExecutor: ToolExecutor;
|
||||
|
||||
beforeEach(() => {
|
||||
telemetryManager = new TelemetryManager({ enabled: true, maxEntries: 100 });
|
||||
telemetryExecutor = new ToolExecutor(
|
||||
mockVault as unknown as any,
|
||||
mockApp as unknown as any,
|
||||
telemetryManager
|
||||
);
|
||||
});
|
||||
|
||||
it('should record successful tool calls in telemetry', async () => {
|
||||
mockVault.create = jest.fn().mockResolvedValue(null);
|
||||
|
||||
const call: ToolCall = {
|
||||
id: 'call_t1',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'create_file',
|
||||
arguments: JSON.stringify({ path: 'test.md', content: 'hello' }),
|
||||
},
|
||||
};
|
||||
|
||||
await telemetryExecutor.handleToolCall(call);
|
||||
const entries = telemetryManager.getEntriesByType('tool_call');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect((entries[0] as any).toolName).toBe('create_file');
|
||||
expect((entries[0] as any).success).toBe(true);
|
||||
expect((entries[0] as any).durationMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('should record failed tool calls in telemetry', async () => {
|
||||
const call: ToolCall = {
|
||||
id: 'call_t2',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'create_file',
|
||||
arguments: JSON.stringify({ path: '/invalid/path.md', content: 'hello' }),
|
||||
},
|
||||
};
|
||||
|
||||
await expect(telemetryExecutor.handleToolCall(call)).rejects.toThrow();
|
||||
const entries = telemetryManager.getEntriesByType('tool_call');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect((entries[0] as any).toolName).toBe('create_file');
|
||||
expect((entries[0] as any).success).toBe(false);
|
||||
});
|
||||
|
||||
it('should include parsed args in telemetry', async () => {
|
||||
mockVault.create = jest.fn().mockResolvedValue(null);
|
||||
|
||||
const call: ToolCall = {
|
||||
id: 'call_t3',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'create_file',
|
||||
arguments: JSON.stringify({ path: 'note.md', content: 'data' }),
|
||||
},
|
||||
};
|
||||
|
||||
await telemetryExecutor.handleToolCall(call);
|
||||
const entries = telemetryManager.getEntriesByType('tool_call');
|
||||
expect((entries[0] as any).args).toEqual({ path: 'note.md', content: 'data' });
|
||||
});
|
||||
|
||||
it('should not record telemetry when telemetry manager is undefined', async () => {
|
||||
const noTelemetryExecutor = new ToolExecutor(
|
||||
mockVault as unknown as any,
|
||||
mockApp as unknown as any
|
||||
);
|
||||
mockVault.create = jest.fn().mockResolvedValue(null);
|
||||
|
||||
const call: ToolCall = {
|
||||
id: 'call_t4',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'create_file',
|
||||
arguments: JSON.stringify({ path: 'x.md', content: 'y' }),
|
||||
},
|
||||
};
|
||||
|
||||
// Should not throw
|
||||
await noTelemetryExecutor.handleToolCall(call);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user