tomi/Toak
1
0
Fork 0
Code Activity

feat: Introduce an OpenAI-compatible client to replace the Groq-specific client and enable multiple LLM providers.

Browse Source
This commit is contained in:
Tomi Eckert
2026-02-28 16:09:41 +01:00
parent 3ceecbe5ee
commit 4e04cc6042
7 changed files with 104 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Commands/LatencyTestCommand.cs
View File

@@ -42,7 +42,7 @@ public static class LatencyTestCommand
return;
}
var groq = new GroqApiClient(config.GroqApiKey);
var client = new OpenAiCompatibleClient(config.GroqApiKey);
try
{
@@ -51,13 +51,13 @@ public static class LatencyTestCommand
{
ctx.Status("Testing STT (Whisper)...");
var sttWatch = Stopwatch.StartNew();
var transcript = await groq.TranscribeAsync(testWavPath, config.WhisperLanguage, config.WhisperModel);
var transcript = await client.TranscribeAsync(testWavPath, config.WhisperLanguage, config.WhisperModel);
sttWatch.Stop();
ctx.Status("Testing LLM (Llama)...");
var systemPrompt = PromptBuilder.BuildPrompt(config);
var llmWatch = Stopwatch.StartNew();
var refinedText = await groq.RefineTextAsync("Hello world, this is a latency test.", systemPrompt, config.LlmModel);
var refinedText = await client.RefineTextAsync("Hello world, this is a latency test.", systemPrompt, config.LlmModel);
llmWatch.Stop();
var total = sttWatch.ElapsedMilliseconds + llmWatch.ElapsedMilliseconds;

43
Commands/OnboardCommand.cs
View File

@@ -21,17 +21,44 @@ public static class OnboardCommand
AnsiConsole.WriteLine();
config.GroqApiKey = AnsiConsole.Prompt(
new TextPrompt<string>("Groq API Key:")
new TextPrompt<string>("Groq API Key (required for Whisper):")
.DefaultValue(string.IsNullOrWhiteSpace(config.GroqApiKey) ? "" : config.GroqApiKey)
.AllowEmpty());
config.LlmModel = AnsiConsole.Prompt(
config.LlmProvider = AnsiConsole.Prompt(
new SelectionPrompt<string>()
.Title("Select [green]LLM Model[/]:")
.AddChoices(new[] { "openai/gpt-oss-20b", "llama-3.1-8b-instant", "llama-3.3-70b-versatile" })
.UseConverter(c => c == "openai/gpt-oss-20b" ? "openai/gpt-oss-20b (Fastest)" : c == "llama-3.1-8b-instant" ? "llama-3.1-8b-instant (Cheapest)" : "llama-3.3-70b-versatile (More Accurate)"));
if (config.LlmModel.Contains(" ")) config.LlmModel = config.LlmModel.Split(' ')[0];
.Title("Select [green]LLM Provider[/]:")
.AddChoices(new[] { "groq", "together" })
.UseConverter(c => c == "groq" ? "Groq (Default)" : "Together AI"));
if (config.LlmProvider == "together")
{
config.TogetherApiKey = AnsiConsole.Prompt(
new TextPrompt<string>("Together AI API Key:")
.DefaultValue(string.IsNullOrWhiteSpace(config.TogetherApiKey) ? "" : config.TogetherApiKey)
.AllowEmpty());
config.LlmModel = AnsiConsole.Prompt(
new SelectionPrompt<string>()
.Title("Select [green]LLM Model[/]:")
.AddChoices(new[] { "meta-llama/Llama-3.3-70B-Instruct-Turbo", "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "openai/gpt-oss-20b" }));
}
else
{
config.LlmModel = AnsiConsole.Prompt(
new SelectionPrompt<string>()
.Title("Select [green]LLM Model[/]:")
.AddChoices(new[] { "openai/gpt-oss-20b", "llama-3.1-8b-instant", "llama-3.3-70b-versatile" })
.UseConverter(c => c == "openai/gpt-oss-20b" ? "openai/gpt-oss-20b (Fastest)" : c == "llama-3.1-8b-instant" ? "llama-3.1-8b-instant (Cheapest)" : "llama-3.3-70b-versatile (More Accurate)"));
if (config.LlmModel.Contains(" ")) config.LlmModel = config.LlmModel.Split(' ')[0];
}
config.ReasoningEffort = AnsiConsole.Prompt(
new SelectionPrompt<string>()
.Title("Select [green]Reasoning Effort[/]:")
.AddChoices(new[] { "none", "low" })
.UseConverter(c => c == "none" ? "None (Standard)" : "Low (Moderate Reasoning)"));
config.WhisperModel = AnsiConsole.Prompt(
new SelectionPrompt<string>()