tomi/Toak
1
0
Fork 0
Code Activity

feat: Introduce an OpenAI-compatible client to replace the Groq-specific client and enable multiple LLM providers.

Browse Source
This commit is contained in:
Tomi Eckert
2026-02-28 16:09:41 +01:00
parent 3ceecbe5ee
commit 4e04cc6042
7 changed files with 104 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
Api/Models/LlamaModels.cs → Api/Models/OpenAiModels.cs
View File

@@ -2,7 +2,7 @@ using System.Text.Json.Serialization;
namespace Toak.Api.Models;
public class LlamaRequestMessage
public class OpenAiRequestMessage
{
[JsonPropertyName("role")]
public string Role { get; set; } = string.Empty;
@@ -10,43 +10,45 @@ public class LlamaRequestMessage
public string Content { get; set; } = string.Empty;
}
public class LlamaRequest
public class OpenAiRequest
{
[JsonPropertyName("model")]
public string Model { get; set; } = "llama-3.1-8b-instant";
[JsonPropertyName("messages")]
public LlamaRequestMessage[] Messages { get; set; } = Array.Empty<LlamaRequestMessage>();
public OpenAiRequestMessage[] Messages { get; set; } = Array.Empty<OpenAiRequestMessage>();
[JsonPropertyName("temperature")]
public double Temperature { get; set; } = 0.0;
[JsonPropertyName("stream")]
public bool? Stream { get; set; }
[JsonPropertyName("reasoning_effort")]
public string? ReasoningEffort { get; set; }
}
public class LlamaResponse
public class OpenAiResponse
{
[JsonPropertyName("choices")]
public LlamaChoice[] Choices { get; set; } = Array.Empty<LlamaChoice>();
public OpenAiChoice[] Choices { get; set; } = Array.Empty<OpenAiChoice>();
}
public class LlamaChoice
public class OpenAiChoice
{
[JsonPropertyName("message")]
public LlamaRequestMessage Message { get; set; } = new();
public OpenAiRequestMessage Message { get; set; } = new();
}
public class LlamaStreamResponse
public class OpenAiStreamResponse
{
[JsonPropertyName("choices")]
public LlamaStreamChoice[] Choices { get; set; } = Array.Empty<LlamaStreamChoice>();
public OpenAiStreamChoice[] Choices { get; set; } = Array.Empty<OpenAiStreamChoice>();
}
public class LlamaStreamChoice
public class OpenAiStreamChoice
{
[JsonPropertyName("delta")]
public LlamaStreamDelta Delta { get; set; } = new();
public OpenAiStreamDelta Delta { get; set; } = new();
}
public class LlamaStreamDelta
public class OpenAiStreamDelta
{
[JsonPropertyName("content")]
public string? Content { get; set; }

47
Api/GroqApiClient.cs → Api/OpenAiCompatibleClient.cs
View File

@@ -9,19 +9,22 @@ using Toak.Core.Interfaces;
namespace Toak.Api;
public class GroqApiClient : ISpeechClient, ILlmClient
public class OpenAiCompatibleClient : ISpeechClient, ILlmClient
{
private readonly HttpClient _httpClient;
private readonly string? _reasoningEffort;
public GroqApiClient(string apiKey)
public OpenAiCompatibleClient(string apiKey, string baseUrl = "https://api.groq.com/openai/v1/", string? reasoningEffort = null)
{
_httpClient = new HttpClient();
_httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
_httpClient.BaseAddress = new Uri("https://api.groq.com/openai/v1/");
_httpClient.BaseAddress = new Uri(baseUrl);
_reasoningEffort = reasoningEffort == "none" ? null : reasoningEffort;
}
public async Task<string> TranscribeAsync(string filePath, string language = "", string model = Toak.Core.Constants.Defaults.WhisperModel)
{
// ... (TranscribeAsync content remains same except maybe some internal comments or contexts)
using var content = new MultipartFormDataContent();
using var fileStream = File.OpenRead(filePath);
using var streamContent = new StreamContent(fileStream);
@@ -31,8 +34,6 @@ public class GroqApiClient : ISpeechClient, ILlmClient
string modelToUse = string.IsNullOrWhiteSpace(model) ? Toak.Core.Constants.Defaults.WhisperModel : model;
// according to docs whisper-large-v3-turbo requires the language to be provided if it is to be translated later potentially or if we need the most accurate behavior
// Actually, if we want language param, we can pass it to either model
content.Add(new StringContent(modelToUse), "model");
if (!string.IsNullOrWhiteSpace(language))
@@ -58,62 +59,64 @@ public class GroqApiClient : ISpeechClient, ILlmClient
public async Task<string> RefineTextAsync(string rawTranscript, string systemPrompt, string model = Toak.Core.Constants.Defaults.LlmModel)
{
var requestBody = new LlamaRequest
var requestBody = new OpenAiRequest
{
Model = string.IsNullOrWhiteSpace(model) ? Toak.Core.Constants.Defaults.LlmModel : model,
Temperature = 0.0,
ReasoningEffort = _reasoningEffort,
Messages = new[]
{
new LlamaRequestMessage { Role = "system", Content = systemPrompt },
new LlamaRequestMessage { Role = "user", Content = $"<transcript>{rawTranscript}</transcript>" }
new OpenAiRequestMessage { Role = "system", Content = systemPrompt },
new OpenAiRequestMessage { Role = "user", Content = $"<transcript>{rawTranscript}</transcript>" }
}
};
var jsonContent = new StringContent(JsonSerializer.Serialize(requestBody, AppJsonSerializerContext.Default.LlamaRequest), System.Text.Encoding.UTF8, "application/json");
var jsonContent = new StringContent(JsonSerializer.Serialize(requestBody, AppJsonSerializerContext.Default.OpenAiRequest), System.Text.Encoding.UTF8, "application/json");
Logger.LogDebug($"Sending Llama API request (model: {requestBody.Model})...");
Logger.LogDebug($"Sending OpenAi API request (model: {requestBody.Model})...");
var response = await _httpClient.PostAsync("chat/completions", jsonContent);
Logger.LogDebug($"Llama API response status: {response.StatusCode}");
Logger.LogDebug($"OpenAi API response status: {response.StatusCode}");
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
throw new Exception($"Llama API Error: {response.StatusCode} - {error}");
throw new Exception($"OpenAi API Error: {response.StatusCode} - {error}");
}
var json = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize(json, AppJsonSerializerContext.Default.LlamaResponse);
var result = JsonSerializer.Deserialize(json, AppJsonSerializerContext.Default.OpenAiResponse);
return result?.Choices?.FirstOrDefault()?.Message?.Content ?? string.Empty;
}
public async IAsyncEnumerable<string> RefineTextStreamAsync(string rawTranscript, string systemPrompt, string model = Toak.Core.Constants.Defaults.LlmModel)
{
var requestBody = new LlamaRequest
var requestBody = new OpenAiRequest
{
Model = string.IsNullOrWhiteSpace(model) ? Toak.Core.Constants.Defaults.LlmModel : model,
Temperature = 0.0,
Stream = true,
ReasoningEffort = _reasoningEffort,
Messages = new[]
{
new LlamaRequestMessage { Role = "system", Content = systemPrompt },
new LlamaRequestMessage { Role = "user", Content = $"<transcript>{rawTranscript}</transcript>" }
new OpenAiRequestMessage { Role = "system", Content = systemPrompt },
new OpenAiRequestMessage { Role = "user", Content = $"<transcript>{rawTranscript}</transcript>" }
}
};
var jsonContent = new StringContent(JsonSerializer.Serialize(requestBody, AppJsonSerializerContext.Default.LlamaRequest), System.Text.Encoding.UTF8, "application/json");
var jsonContent = new StringContent(JsonSerializer.Serialize(requestBody, AppJsonSerializerContext.Default.OpenAiRequest), System.Text.Encoding.UTF8, "application/json");
using var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions") { Content = jsonContent };
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/event-stream"));
Logger.LogDebug($"Sending Llama Steam API request (model: {requestBody.Model})...");
Logger.LogDebug($"Sending OpenAi Steam API request (model: {requestBody.Model})...");
using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
Logger.LogDebug($"Llama Stream API response status: {response.StatusCode}");
Logger.LogDebug($"OpenAi Stream API response status: {response.StatusCode}");
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
throw new Exception($"Llama API Error: {response.StatusCode} - {error}");
throw new Exception($"OpenAi API Error: {response.StatusCode} - {error}");
}
using var stream = await response.Content.ReadAsStreamAsync();
@@ -128,7 +131,7 @@ public class GroqApiClient : ISpeechClient, ILlmClient
var data = line.Substring("data: ".Length).Trim();
if (data == "[DONE]") break;
var chunk = JsonSerializer.Deserialize(data, AppJsonSerializerContext.Default.LlamaStreamResponse);
var chunk = JsonSerializer.Deserialize(data, AppJsonSerializerContext.Default.OpenAiStreamResponse);
var content = chunk?.Choices?.FirstOrDefault()?.Delta?.Content;
if (!string.IsNullOrEmpty(content))
{
@@ -138,3 +141,5 @@ public class GroqApiClient : ISpeechClient, ILlmClient
}
}
}