tomi/Toak
1
0
Fork 0
Code Activity

feat: Introduce an OpenAI-compatible client to replace the Groq-specific client and enable multiple LLM providers.

Browse Source
This commit is contained in:
Tomi Eckert
2026-02-28 16:09:41 +01:00
parent 3ceecbe5ee
commit 4e04cc6042
7 changed files with 104 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Commands/LatencyTestCommand.cs
View File

@@ -42,7 +42,7 @@ public static class LatencyTestCommand
return;
}
var groq = new GroqApiClient(config.GroqApiKey);
var client = new OpenAiCompatibleClient(config.GroqApiKey);
try
{
@@ -51,13 +51,13 @@ public static class LatencyTestCommand
{
ctx.Status("Testing STT (Whisper)...");
var sttWatch = Stopwatch.StartNew();
var transcript = await groq.TranscribeAsync(testWavPath, config.WhisperLanguage, config.WhisperModel);
var transcript = await client.TranscribeAsync(testWavPath, config.WhisperLanguage, config.WhisperModel);
sttWatch.Stop();
ctx.Status("Testing LLM (Llama)...");
var systemPrompt = PromptBuilder.BuildPrompt(config);
var llmWatch = Stopwatch.StartNew();
var refinedText = await groq.RefineTextAsync("Hello world, this is a latency test.", systemPrompt, config.LlmModel);
var refinedText = await client.RefineTextAsync("Hello world, this is a latency test.", systemPrompt, config.LlmModel);
llmWatch.Stop();
var total = sttWatch.ElapsedMilliseconds + llmWatch.ElapsedMilliseconds;