diff --git a/Endpoints/ChatEndpoint.cs b/Endpoints/ChatEndpoint.cs index 46905f5..9374e2f 100644 --- a/Endpoints/ChatEndpoint.cs +++ b/Endpoints/ChatEndpoint.cs @@ -254,7 +254,7 @@ public static class ChatEndpoint if (anthropic.IsCliAvailable) { - await anthropic.StreamChatViaCli(res, ctx.ModelId, ctx.SystemPrompt, messages, ctx.ChainId, workspace: ctx.Workspace, allowUserChoice: ctx.Agent?.AllowUserChoice ?? true, planMode: body.PlanMode, webSearch: !string.IsNullOrEmpty(ctx.WebSearchVersion), sessionId: sessionId, agentId: ctx.AgentId, remoteMgr: remoteMgr, unityMcpUrl: ctx.Session?.UnityMcpUrl, unityMcpToken: ctx.Session?.UnityMcpToken, workCt: workCt, sseCt: sseCt); + await anthropic.StreamChatViaCli(res, ctx.ModelId, ctx.SystemPrompt, messages, ctx.ChainId, workspace: ctx.Workspace, allowUserChoice: ctx.Agent?.AllowUserChoice ?? true, planMode: body.PlanMode, webSearch: !string.IsNullOrEmpty(ctx.WebSearchVersion), sessionId: sessionId, agentId: ctx.AgentId, remoteMgr: remoteMgr, unityMcpUrl: ctx.Session?.UnityMcpUrl, unityMcpToken: ctx.Session?.UnityMcpToken, useAutoContext: ctx.Agent?.UseAutoContext ?? true, workCt: workCt, sseCt: sseCt); } else { diff --git a/Endpoints/EmbeddingDebugEndpoint.cs b/Endpoints/EmbeddingDebugEndpoint.cs new file mode 100644 index 0000000..e41aaa5 --- /dev/null +++ b/Endpoints/EmbeddingDebugEndpoint.cs @@ -0,0 +1,198 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Gott.Extensions; +using Gott.Services; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; + +namespace Gott.Endpoints; + +public static class EmbeddingDebugEndpoint +{ + public static void MapEmbeddingDebugEndpoints(this WebApplication app) + { + app.MapGet("/api/admin/embedding-debug", HandleDebugSearch); + app.MapGet("/api/admin/embedding-debug/stats", HandleStats); + app.MapPost("/api/admin/embedding-debug/re-embed", HandleReEmbed); + } + + /// + /// Diagnose-Suche: Zeigt Top-N mit Scores, inkl. Ergebnisse unter Schwellenwert. + /// Query-Params: q (query), agentId, topN (default 20), chainId (zum Simulieren des Filters) + /// + private static async Task HandleDebugSearch(HttpContext ctx, + VectorSearchService vectorSearch, EmbeddingService embeddingService) + { + if (!ctx.IsAdmin()) + { + ctx.Response.StatusCode = 403; + await ctx.Response.WriteAsJsonAsync(new { error = "Admin access required" }); + return; + } + + if (!embeddingService.IsAvailable) + { + await ctx.Response.WriteAsJsonAsync(new { error = "Embedding model not available" }); + return; + } + + var query = ctx.Request.Query["q"].ToString(); + var agentId = ctx.Request.Query["agentId"].ToString(); + var chainId = ctx.Request.Query["chainId"].ToString(); + var topN = 20; + if (int.TryParse(ctx.Request.Query["topN"].ToString(), out var parsedTopN) && parsedTopN > 0) + topN = Math.Min(parsedTopN, 100); + + if (string.IsNullOrWhiteSpace(query)) + { + await ctx.Response.WriteAsJsonAsync(new { error = "Query parameter 'q' is required" }); + return; + } + + // Suche mit SEHR niedrigem Schwellenwert, damit wir alles sehen + var allResults = await vectorSearch.SearchAsync(query, + string.IsNullOrEmpty(agentId) ? null : agentId, + topN: topN, + minScore: 0.1f); + + // Memory-Dateien auch durchsuchen + var memoryResults = new List(); + if (!string.IsNullOrEmpty(agentId)) + { + memoryResults = await vectorSearch.SearchMemoryFilesAsync(query, agentId, topN: 10, minScore: 0.1f); + } + + // Ergebnisse annotieren: Was würde der AutoContext-Filter machen? + // Neuer Threshold ist 0.35 durch Hybrid-Suche + var annotatedMessages = allResults.Select(r => new + { + r.MessageId, + r.ChainId, + r.SessionId, + r.AgentId, + r.Role, + r.TextPreview, + FullText = r.FullText ?? r.TextPreview, + Score = Math.Round(r.Score, 4), + r.CreatedAt, + WouldPassThreshold = r.Score >= 0.35f, + WouldBeFilteredByChain = !string.IsNullOrEmpty(chainId) && r.ChainId == chainId, + WouldBeFilteredByLength = (r.TextPreview?.Length ?? 0) < 20, + WouldAppearInAutoContext = r.Score >= 0.35f + && (string.IsNullOrEmpty(chainId) || r.ChainId != chainId) + && (r.TextPreview?.Length ?? 0) >= 20 + }).ToList(); + + var annotatedMemory = memoryResults.Select(r => new + { + r.AgentId, + r.FilePath, + r.ChunkIndex, + r.TextPreview, + FullText = r.FullText ?? r.TextPreview, + Score = Math.Round(r.Score, 4), + r.UpdatedAt, + WouldPassThreshold = r.Score >= 0.35f + }).ToList(); + + // Zusammenfassung + var summary = new + { + Query = query, + AgentId = agentId, + ChainIdFilter = chainId, + TotalMessageResults = annotatedMessages.Count, + AboveThreshold035 = annotatedMessages.Count(r => r.WouldPassThreshold), + BelowThreshold = annotatedMessages.Count(r => !r.WouldPassThreshold), + WouldAppearInAutoContext = annotatedMessages.Count(r => r.WouldAppearInAutoContext), + FilteredByChain = annotatedMessages.Count(r => r.WouldBeFilteredByChain), + FilteredByLength = annotatedMessages.Count(r => r.WouldBeFilteredByLength), + TotalMemoryResults = annotatedMemory.Count, + MemoryAboveThreshold = annotatedMemory.Count(r => r.WouldPassThreshold) + }; + + await ctx.Response.WriteAsJsonAsync(new + { + summary, + messages = annotatedMessages, + memory = annotatedMemory + }); + } + + /// + /// Statistiken: Anzahl Embeddings pro Agent, Gesamtzahl, etc. + /// + private static async Task HandleStats(HttpContext ctx, + VectorSearchService vectorSearch, DatabaseService db) + { + if (!ctx.IsAdmin()) + { + ctx.Response.StatusCode = 403; + await ctx.Response.WriteAsJsonAsync(new { error = "Admin access required" }); + return; + } + + using var conn = db.GetConnection(); + + var totalMessages = await Dapper.SqlMapper.ExecuteScalarAsync(conn, + "SELECT COUNT(*) FROM message_embeddings"); + var totalMemory = await Dapper.SqlMapper.ExecuteScalarAsync(conn, + "SELECT COUNT(*) FROM memory_file_embeddings"); + + var perAgentDynamic = await Dapper.SqlMapper.QueryAsync(conn, + @"SELECT me.agent_id, + a.name, + COUNT(*) AS message_count, + (SELECT COUNT(*) FROM memory_file_embeddings mfe WHERE mfe.agent_id = me.agent_id) AS memory_chunks + FROM message_embeddings me + LEFT JOIN agents a ON a.id = me.agent_id + GROUP BY me.agent_id + ORDER BY COUNT(*) DESC"); + + var perAgent = perAgentDynamic.Select(r => new + { + AgentId = (string)r.agent_id, + AgentName = (string)r.name, + MessageCount = (long)r.message_count, + MemoryChunks = (long)r.memory_chunks + }).ToList(); + + await ctx.Response.WriteAsJsonAsync(new + { + totalMessages, + totalMemory, + perAgent + }); + } + + /// + /// Triggert Re-Embedding aller Messages und Memory-Dateien. + /// POST /api/admin/embedding-debug/re-embed + /// + private static async Task HandleReEmbed(HttpContext ctx, + EmbeddingBackgroundService embeddingBg, EmbeddingService embeddingService) + { + if (!ctx.IsAdmin()) + { + ctx.Response.StatusCode = 403; + await ctx.Response.WriteAsJsonAsync(new { error = "Admin access required" }); + return; + } + + if (!embeddingService.IsAvailable) + { + await ctx.Response.WriteAsJsonAsync(new { error = "Embedding model not available" }); + return; + } + + embeddingBg.TriggerReEmbed(); + await ctx.Response.WriteAsJsonAsync(new + { + status = "Re-embedding triggered", + model = embeddingService.ModelName, + info = "Background service will re-embed all messages and memory files. Check stats for progress." + }); + } +} diff --git a/Plans/SENTENCEPIECE_TOKENIZER.md b/Plans/SENTENCEPIECE_TOKENIZER.md new file mode 100644 index 0000000..fa85fb5 --- /dev/null +++ b/Plans/SENTENCEPIECE_TOKENIZER.md @@ -0,0 +1,83 @@ +# Plan: SentencePiece-Tokenizer-Support für multilinguales Embedding-Modell + +## Ziel +`paraphrase-multilingual-MiniLM-L12-v2` als alternatives Embedding-Modell ermöglichen, indem der `EmbeddingService` neben WordPiece (BERT) auch SentencePiece (XLM-RoBERTa) Tokenizer unterstützt. + +## Hintergrund +- Aktuell: `all-MiniLM-L6-v2` (BERT, WordPiece, `vocab.txt`, 384 Dims) +- Ziel-Modell: `paraphrase-multilingual-MiniLM-L12-v2` (XLM-RoBERTa, SentencePiece, `sentencepiece.bpe.model`, 384 Dims) +- Problem: Unser Tokenizer ist hardcoded auf WordPiece – Special Tokens (`[CLS]`/`[SEP]`), `token_type_ids`, `vocab.txt` + +## Unterschiede zwischen den Tokenizern + +| Aspekt | WordPiece (BERT) | SentencePiece (XLM-R) | +|---|---|---| +| Vocab-Datei | `vocab.txt` | `sentencepiece.bpe.model` | +| Special Tokens | `[CLS]`=101, `[SEP]`=102, `[PAD]`=0 | ``=0, ``=2, ``=1 | +| token_type_ids | Required (alle 0) | Nicht verwendet | +| Subword-Prefix | `##` | `▁` (U+2581) | +| Vocab-Groesse | ~30.522 | ~250.002 | + +--- + +## Schritte + +### Schritt 1: NuGet-Paket hinzufuegen +- `Microsoft.ML.Tokenizers.SentencePieceBpe` oder `SentencePieceSharp` evaluieren +- Kriterien: .NET 10 Kompatibilitaet, kein native-Dependency-Problem auf Linux, Lizenz +- Paket zu `gott.csproj` hinzufuegen + +### Schritt 2: Tokenizer-Abstraktion in `EmbeddingService.cs` +- Interface oder Strategie-Pattern: `ITextTokenizer` mit Methode `Tokenize(string text, int maxLen) -> (long[] inputIds, long[] attentionMask, long[]? tokenTypeIds)` +- `WordPieceTokenizer` – bestehende Logik extrahieren (Zeilen 127-161) +- `SentencePieceTokenizer` – neue Implementierung +- Constructor-Logik: Wenn `vocab.txt` existiert -> WordPiece; wenn `sentencepiece.bpe.model` existiert -> SentencePiece +- Property `TokenizerType` fuer Debug/Logging + +### Schritt 3: ONNX-Session Input anpassen +- WordPiece: 3 Inputs (`input_ids`, `attention_mask`, `token_type_ids`) +- SentencePiece: 2 Inputs (`input_ids`, `attention_mask`) – kein `token_type_ids` +- `GenerateEmbedding()` muss dynamisch die richtigen Inputs erstellen basierend auf Tokenizer-Typ +- ONNX Input-Names aus dem Modell lesen statt hardcoded (robuster) + +### Schritt 4: Special Token IDs konfigurierbar machen +- `[CLS]`/`[SEP]` IDs vs. ``/`` IDs +- Am besten aus `special_tokens_map.json` oder `tokenizer_config.json` lesen, falls vorhanden +- Fallback: Hardcoded Mapping pro Tokenizer-Typ + +### Schritt 5: Modell-Dateien bereitstellen +- Von HuggingFace herunterladen: + - `model.onnx` (ONNX-Export des Modells) + - `sentencepiece.bpe.model` +- Ablegen in `data/models/paraphrase-multilingual-MiniLM-L12-v2/` +- Server-Setting `EmbeddingModelDir` auf den neuen Ordner setzen + +### Schritt 6: Re-Embed-Kompatibilitaet +- Dimensionen sind identisch (384) -> Keine DB-Schema-Aenderung noetig +- Re-Embed-Endpoint muss weiterhin funktionieren +- Beim Modell-Wechsel: **Alle** Embeddings muessen neu berechnet werden (altes Modell-Vektoren sind inkompatibel mit neuem Modell) +- Warnung im Admin-UI wenn Modell gewechselt wurde aber Re-Embed noch nicht gelaufen ist + +### Schritt 7: Tests & Verifikation +- Embedding-Debug-Page nutzen: Gleiche Queries mit altem vs. neuem Modell vergleichen +- Testcases: + - Deutsche Query -> deutsche Messages ("weisst du was zum thema autocontext") + - Englische Query -> englische Messages ("skills") + - Gemischt -> gemischte Messages ("otter karten") + - Pruefen ob Scores sich verbessern (besonders fuer deutsche Queries) + +--- + +## Betroffene Dateien +1. `gott.csproj` – NuGet-Paket +2. `Services/EmbeddingService.cs` – Tokenizer-Abstraktion, SentencePiece-Support +3. `wwwroot/js/embedding-debug.js` – ggf. Anzeige des aktiven Modells/Tokenizers + +## Risiken +- **SentencePiece NuGet auf Linux**: Native Dependencies koennten Probleme machen -> vorher testen +- **Re-Embed Dauer**: 14k Messages mit 256 Tokens -> je nach CPU 5-30 Minuten +- **Dimensionen-Mismatch**: Falls ein zukuenftiges Modell andere Dims hat -> DB-Migration noetig (aktuell nicht, beide 384) + +## Aufwand +- Geschaetzt: ~2-3 Stunden Implementierung +- Groesster Block: Tokenizer-Abstraktion sauber bauen diff --git a/Program.cs b/Program.cs index d5a4b2c..556a54e 100644 --- a/Program.cs +++ b/Program.cs @@ -65,7 +65,14 @@ var chatService = new ChatService(dbService, toolService, memoryService, skillSe builder.Services.AddSingleton(chatService); // Embedding-Services (lokal, ONNX) -var embeddingService = new Gott.Services.EmbeddingService(); +var embeddingModelDir = serverSettings.Get("EmbeddingModelDir"); +var embeddingMaxSeq = serverSettings.GetInt("EmbeddingMaxSeqLen", 256); +var embeddingService = new Gott.Services.EmbeddingService( + string.IsNullOrEmpty(embeddingModelDir) + ? null + : Path.Combine(dataDir, "models", embeddingModelDir), + embeddingMaxSeq +); builder.Services.AddSingleton(embeddingService); var vectorSearchService = new Gott.Services.VectorSearchService(dbPath, embeddingService); builder.Services.AddSingleton(vectorSearchService); @@ -179,6 +186,7 @@ app.MapApiKeysEndpoints(); app.MapOpenClawEndpoints(); app.MapRemoteEndpoints(); app.MapDatabaseExplorerEndpoints(); +app.MapEmbeddingDebugEndpoints(); app.MapInternalEndpoints(); var port = app.Configuration.GetValue("Port", 19000); diff --git a/Services/AnthropicService.Cli.cs b/Services/AnthropicService.Cli.cs index 3443bae..55a5e24 100644 --- a/Services/AnthropicService.Cli.cs +++ b/Services/AnthropicService.Cli.cs @@ -33,6 +33,7 @@ public partial class AnthropicService RemoteConnectionManager remoteMgr = null, string unityMcpUrl = null, string unityMcpToken = null, + bool useAutoContext = true, CancellationToken workCt = default, CancellationToken sseCt = default) { @@ -77,6 +78,21 @@ public partial class AnthropicService } + // ── Automatische Kontext-Injektion via Embedding-Suche (wie in StreamChat) ── + if (useAutoContext && messages.Count > 0) + { + try + { + var autoCtx = await BuildAutoContextAsync(messages, chainId, agentId); + if (autoCtx != null) + systemText.AppendLine(autoCtx); + } + catch (Exception ex) + { + Console.WriteLine($"[AutoContext/CLI] Error: {ex.Message}"); + } + } + // Letzte User-Message als Prompt extrahieren var prompt = ExtractLastUserText(messages) ?? ""; @@ -118,14 +134,16 @@ public partial class AnthropicService // Text-Puffer flushen bevor Tool-Use gespeichert wird (korrekte Reihenfolge in DB) if (fullText.Length > 0) { - await _db.SaveMessage(new MessageRecord + var textForEmbed = fullText.ToString(); + var msgId = await _db.SaveMessage(new MessageRecord { ChainId = chainId, Iteration = 1, Role = "assistant", Type = "text", - Content = fullText.ToString() + Content = textForEmbed }); + _embeddingBg?.Enqueue(msgId, chainId, sessionId, agentId, "assistant", textForEmbed); fullText.Clear(); } var mappedName = MapCliToolName(evt.ToolName); @@ -235,14 +253,16 @@ public partial class AnthropicService // Antwort in DB speichern if (fullText.Length > 0) { - await _db.SaveMessage(new MessageRecord + var finalText = fullText.ToString(); + var msgId = await _db.SaveMessage(new MessageRecord { ChainId = chainId, Iteration = 1, Role = "assistant", Type = "text", - Content = fullText.ToString() + Content = finalText }); + _embeddingBg?.Enqueue(msgId, chainId, sessionId, agentId, "assistant", finalText); } // Token-Usage vom CLI-Result übernehmen diff --git a/Services/AnthropicService.Streaming.cs b/Services/AnthropicService.Streaming.cs index bb95e21..61a0eda 100644 --- a/Services/AnthropicService.Streaming.cs +++ b/Services/AnthropicService.Streaming.cs @@ -58,38 +58,13 @@ public partial class AnthropicService try { // ── Automatische Kontext-Injektion via Embedding-Suche ── - if (useAutoContext && _vectorSearch != null && systemPrompt != null && messages.Count > 0) + if (useAutoContext) { try { - // Letzte User-Message als Query extrahieren - var lastUserText = ExtractLastUserText(messages); - if (!string.IsNullOrWhiteSpace(lastUserText)) - { - var contextResults = await _vectorSearch.SearchAsync(lastUserText, agentId, topN: 5, minScore: 0.45f); - // Nur Ergebnisse aus anderen Sessions (nicht aktuelle Chain) und nicht zu kurze - contextResults.RemoveAll(r => r.ChainId == chainId || (r.TextPreview?.Length ?? 0) < 20); - // Memory-Dateien durchsuchen - var memoryResults = await _vectorSearch.SearchMemoryFilesAsync(lastUserText, agentId, topN: 3, minScore: 0.45f); - - if (contextResults.Count > 0 || memoryResults.Count > 0) - { - var ctxSb = new StringBuilder(); - ctxSb.AppendLine("\n"); - ctxSb.AppendLine("Die folgende Information wurde automatisch abgerufen und könnte relevant sein:"); - foreach (var r in contextResults) - { - var date = DateTime.TryParse(r.CreatedAt, out var dt) ? dt.ToString("yyyy-MM-dd HH:mm") : r.CreatedAt; - ctxSb.AppendLine($"- [Gespräch, {r.Role}, {date}]: {r.TextPreview}"); - } - foreach (var r in memoryResults) - { - ctxSb.AppendLine($"- [Memory, {r.FilePath}]: {r.TextPreview}"); - } - ctxSb.AppendLine(""); - systemPrompt.Add(new { type = "text", text = ctxSb.ToString() }); - } - } + var autoCtx = await BuildAutoContextAsync(messages, chainId, agentId); + if (autoCtx != null) + systemPrompt.Add(new { type = "text", text = autoCtx }); } catch (Exception ex) { diff --git a/Services/AnthropicService.cs b/Services/AnthropicService.cs index 443d527..2c03d33 100644 --- a/Services/AnthropicService.cs +++ b/Services/AnthropicService.cs @@ -136,6 +136,79 @@ public partial class AnthropicService return null; } + /// Extrahiert die letzten N User-Texte für Query-Erweiterung. + private static string ExtractRecentUserTexts(List messages, int count = 3, int maxLength = 500) + { + var texts = new List(); + for (int i = messages.Count - 1; i >= 0 && texts.Count < count; i--) + { + var json = JsonSerializer.Serialize(messages[i]); + var el = JsonSerializer.Deserialize(json); + if (!el.TryGetProperty("role", out var role) || role.GetString() != "user") continue; + if (!el.TryGetProperty("content", out var content)) continue; + + string text = null; + if (content.ValueKind == JsonValueKind.String) + text = content.GetString(); + else if (content.ValueKind == JsonValueKind.Array) + { + foreach (var block in content.EnumerateArray()) + { + var type = block.TryGetProperty("type", out var t) ? t.GetString() : ""; + if (type == "text" && block.TryGetProperty("text", out var txt)) + { text = txt.GetString(); break; } + } + } + if (!string.IsNullOrWhiteSpace(text)) + texts.Add(text); + } + + texts.Reverse(); // chronological order + var combined = string.Join(" ", texts); + return combined.Length > maxLength ? combined.Substring(0, maxLength) : combined; + } + + /// + /// Baut den AutoContext-String aus Embedding-Suche. + /// Shared zwischen StreamChat und StreamChatViaCli. + /// + private async Task BuildAutoContextAsync(List messages, string chainId, string agentId) + { + if (_vectorSearch == null || messages.Count == 0) return null; + + // Query-Erweiterung: letzte 2-3 User-Messages statt nur die letzte + var queryText = ExtractRecentUserTexts(messages, count: 3, maxLength: 500); + if (string.IsNullOrWhiteSpace(queryText)) return null; + + var contextResults = await _vectorSearch.SearchAsync(queryText, agentId, topN: 5, minScore: 0.35f); + // Note: threshold lowered from 0.45 to 0.35 because hybrid search is now more precise + contextResults.RemoveAll(r => r.ChainId == chainId || (r.TextPreview?.Length ?? 0) < 20); + + var memoryResults = await _vectorSearch.SearchMemoryFilesAsync(queryText, agentId, topN: 3, minScore: 0.35f); + + if (contextResults.Count == 0 && memoryResults.Count == 0) return null; + + var ctxSb = new StringBuilder(); + ctxSb.AppendLine("\n"); + ctxSb.AppendLine("Die folgende Information wurde automatisch abgerufen und könnte relevant sein:"); + foreach (var r in contextResults) + { + var date = DateTime.TryParse(r.CreatedAt, out var dt) ? dt.ToString("yyyy-MM-dd HH:mm") : r.CreatedAt; + // Use FullText (up to 800 chars) if available, otherwise TextPreview + var displayText = r.FullText ?? r.TextPreview ?? ""; + if (displayText.Length > 800) displayText = displayText.Substring(0, 800); + ctxSb.AppendLine($"- [Gespräch, {r.Role}, {date}]: {displayText}"); + } + foreach (var r in memoryResults) + { + var displayText = r.FullText ?? r.TextPreview ?? ""; + if (displayText.Length > 800) displayText = displayText.Substring(0, 800); + ctxSb.AppendLine($"- [Memory, {r.FilePath}]: {displayText}"); + } + ctxSb.AppendLine(""); + return ctxSb.ToString(); + } + /// /// Extrahiert Bild-Blöcke aus der letzten User-Message, speichert sie als Temp-Dateien /// und gibt Datei-Referenz-Hinweise zurück, die der CLI-Prompt vorangestellt werden. diff --git a/Services/EmbeddingBackgroundService.cs b/Services/EmbeddingBackgroundService.cs index d1d8cb6..0ae9945 100644 --- a/Services/EmbeddingBackgroundService.cs +++ b/Services/EmbeddingBackgroundService.cs @@ -23,6 +23,7 @@ public class EmbeddingBackgroundService : BackgroundService readonly ConcurrentQueue _queue = new(); readonly SemaphoreSlim _signal = new(0); readonly string _dataPath; + volatile bool _reEmbedRequested; public EmbeddingBackgroundService(VectorSearchService vectorSearch, EmbeddingService embedding, string dbPath) { @@ -32,6 +33,13 @@ public class EmbeddingBackgroundService : BackgroundService _dataPath = Path.GetDirectoryName(dbPath); } + /// Triggers a full re-embedding of all data. + public void TriggerReEmbed() + { + _reEmbedRequested = true; + _signal.Release(); + } + /// Enqueue eine neue Message zum Embedden. public void Enqueue(long messageId, string chainId, string sessionId, string agentId, string role, string text) { @@ -85,6 +93,17 @@ public class EmbeddingBackgroundService : BackgroundService Console.WriteLine($"[EmbeddingBG] Error embedding job: {ex.Message}"); } } + + // Check for re-embed request + if (_reEmbedRequested) + { + _reEmbedRequested = false; + Console.WriteLine("[EmbeddingBG] Re-embedding triggered..."); + await _vectorSearch.ReEmbedAllAsync(); + await BackfillAsync(ct); + await BackfillMemoryFilesAsync(ct); + Console.WriteLine("[EmbeddingBG] Re-embedding complete."); + } } catch (OperationCanceledException) { break; } } diff --git a/Services/EmbeddingService.cs b/Services/EmbeddingService.cs index 25382cf..02692d3 100644 --- a/Services/EmbeddingService.cs +++ b/Services/EmbeddingService.cs @@ -10,8 +10,9 @@ using Microsoft.ML.OnnxRuntime.Tensors; namespace Gott.Services; /// -/// Lokaler Embedding-Service basierend auf all-MiniLM-L6-v2 (ONNX). +/// Lokaler Embedding-Service (ONNX). Standardmäßig all-MiniLM-L6-v2. /// Generiert 384-dimensionale Sentence-Embeddings für semantische Suche. +/// Das Modellverzeichnis und die maximale Sequenzlänge sind konfigurierbar. /// public sealed class EmbeddingService : IDisposable { @@ -20,15 +21,20 @@ public sealed class EmbeddingService : IDisposable readonly int _clsId; readonly int _sepId; readonly int _unkId; - readonly int _maxSeqLen = 128; + readonly int _maxSeqLen; bool _disposed; public bool IsAvailable { get; } public int Dimensions => 384; + public string ModelName { get; } + public bool CrossEncoderAvailable => false; - public EmbeddingService() + public EmbeddingService(string modelDir = null, int maxSeqLen = 256) { - var modelDir = Path.Combine(AppContext.BaseDirectory, "data", "models", "all-MiniLM-L6-v2"); + _maxSeqLen = maxSeqLen; + if (modelDir == null) + modelDir = Path.Combine(AppContext.BaseDirectory, "data", "models", "all-MiniLM-L6-v2"); + ModelName = Path.GetFileName(modelDir); var modelPath = Path.Combine(modelDir, "model.onnx"); var vocabPath = Path.Combine(modelDir, "vocab.txt"); diff --git a/Services/VectorSearchService.cs b/Services/VectorSearchService.cs index 8a2f84c..939ca45 100644 --- a/Services/VectorSearchService.cs +++ b/Services/VectorSearchService.cs @@ -1,5 +1,6 @@ using System; using System.Text; +using System.Text.RegularExpressions; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -11,6 +12,7 @@ namespace Gott.Services; /// /// Verwaltet Message-Embeddings in SQLite und bietet semantische Suche. +/// Hybrid-Suche: Vektor-Cosine-Similarity + FTS5-Keyword-Matching mit Recency-Boost. /// public class VectorSearchService { @@ -32,6 +34,12 @@ public class VectorSearchService return conn; } + void TryExecuteSql(SqliteConnection conn, string sql) + { + try { conn.Execute(sql); } + catch { /* Column/table may already exist — idempotent migration */ } + } + void InitializeSchema() { using var conn = CreateConnection(); @@ -62,6 +70,17 @@ public class VectorSearchService ); CREATE INDEX IF NOT EXISTS idx_memfile_agent ON memory_file_embeddings(agent_id); "); + + // Schema migration: add new columns (idempotent) + TryExecuteSql(conn, "ALTER TABLE message_embeddings ADD COLUMN full_text TEXT"); + TryExecuteSql(conn, "ALTER TABLE message_embeddings ADD COLUMN embedding_model TEXT"); + TryExecuteSql(conn, "ALTER TABLE memory_file_embeddings ADD COLUMN full_text TEXT"); + TryExecuteSql(conn, "ALTER TABLE memory_file_embeddings ADD COLUMN embedding_model TEXT"); + + // FTS5 virtual tables for keyword search + TryExecuteSql(conn, "CREATE VIRTUAL TABLE IF NOT EXISTS message_embeddings_fts USING fts5(full_text, content='', content_rowid='rowid')"); + TryExecuteSql(conn, "CREATE VIRTUAL TABLE IF NOT EXISTS memory_embeddings_fts USING fts5(full_text, content='', content_rowid='rowid')"); + Console.WriteLine("[VectorSearch] Schema initialized."); } @@ -100,9 +119,9 @@ public class VectorSearchService await conn.ExecuteAsync(@" INSERT OR IGNORE INTO message_embeddings - (message_id, chain_id, session_id, agent_id, role, text_preview, vector, created_at) + (message_id, chain_id, session_id, agent_id, role, text_preview, vector, created_at, full_text, embedding_model) VALUES - (@messageId, @chainId, @sessionId, @agentId, @role, @preview, @vector, @createdAt)", + (@messageId, @chainId, @sessionId, @agentId, @role, @preview, @vector, @createdAt, @fullText, @embeddingModel)", new { messageId, @@ -112,13 +131,28 @@ public class VectorSearchService role, preview, vector = EmbeddingService.ToBytes(vector), - createdAt = DateTime.UtcNow.ToString("o") + createdAt = DateTime.UtcNow.ToString("o"), + fullText = text, + embeddingModel = _embedding.ModelName }); + + // Insert into FTS5 index + try + { + var lastId = await conn.ExecuteScalarAsync("SELECT last_insert_rowid()"); + if (lastId > 0) + { + await conn.ExecuteAsync( + "INSERT INTO message_embeddings_fts(rowid, full_text) VALUES (@id, @fullText)", + new { id = lastId, fullText = text }); + } + } + catch { /* FTS5 insert failure is non-critical */ } } /// /// Semantische Suche: Findet die N ähnlichsten Messages zu einer Query. - /// Durchsucht alle Sessions eines Agents (oder alle wenn agentId null). + /// Hybrid: Vektor-Cosine-Similarity + FTS5-Keyword + Recency-Boost. /// public async Task> SearchAsync(string query, string agentId = null, int topN = 10, float minScore = 0.3f) @@ -146,23 +180,107 @@ public class VectorSearchService if (!string.IsNullOrEmpty(agentId)) { - sql = "SELECT message_id, chain_id, session_id, agent_id, role, text_preview, vector, created_at FROM message_embeddings WHERE agent_id = @agentId"; + sql = "SELECT id, message_id, chain_id, session_id, agent_id, role, text_preview, full_text, vector, created_at FROM message_embeddings WHERE agent_id = @agentId"; param = new { agentId }; } else { - sql = "SELECT message_id, chain_id, session_id, agent_id, role, text_preview, vector, created_at FROM message_embeddings"; + sql = "SELECT id, message_id, chain_id, session_id, agent_id, role, text_preview, full_text, vector, created_at FROM message_embeddings"; param = new { }; } var rows = await conn.QueryAsync(sql, param); - var results = new List(); + var rowList = rows.ToList(); - foreach (var row in rows) + // --- Vector scores --- + var vectorScores = new Dictionary(); // keyed by id + var rowMap = new Dictionary(); + foreach (var row in rowList) { var vec = EmbeddingService.FromBytes(row.Vector); var score = EmbeddingService.CosineSimilarity(queryVec, vec); - if (score >= minScore) + vectorScores[row.Id] = score; + rowMap[row.Id] = row; + } + + // --- FTS5 keyword scores --- + var ftsScores = new Dictionary(); + try + { + var sanitized = SanitizeFtsQuery(query); + if (!string.IsNullOrWhiteSpace(sanitized)) + { + string ftsSql; + object ftsParam; + if (!string.IsNullOrEmpty(agentId)) + { + ftsSql = @"SELECT me.id, f.rank + FROM message_embeddings_fts f + JOIN message_embeddings me ON me.rowid = f.rowid + WHERE message_embeddings_fts MATCH @query AND me.agent_id = @agentId + ORDER BY f.rank LIMIT @limit"; + ftsParam = new { query = sanitized, agentId, limit = topN * 3 }; + } + else + { + ftsSql = @"SELECT me.id, f.rank + FROM message_embeddings_fts f + JOIN message_embeddings me ON me.rowid = f.rowid + WHERE message_embeddings_fts MATCH @query + ORDER BY f.rank LIMIT @limit"; + ftsParam = new { query = sanitized, limit = topN * 3 }; + } + + var ftsRows = await conn.QueryAsync(ftsSql, ftsParam); + var ftsList = ftsRows.ToList(); + + if (ftsList.Count > 0) + { + // FTS5 rank is negative (lower = better). Normalize to 0-1. + var minRank = ftsList.Min(r => r.Rank); + var maxRank = ftsList.Max(r => r.Rank); + var range = maxRank - minRank; + foreach (var r in ftsList) + { + float normalized = range == 0 ? 1.0f : (float)((maxRank - r.Rank) / range); + ftsScores[r.Id] = normalized; + // Ensure row is in rowMap (it should be, but load if needed) + if (!rowMap.ContainsKey(r.Id)) + { + var row = await conn.QueryFirstOrDefaultAsync( + "SELECT id, message_id, chain_id, session_id, agent_id, role, text_preview, full_text, vector, created_at FROM message_embeddings WHERE id = @id", + new { id = r.Id }); + if (row != null) rowMap[r.Id] = row; + } + } + } + } + } + catch { /* FTS5 query failure — fall back to vector-only */ } + + // --- Score fusion --- + var allIds = new HashSet(vectorScores.Keys); + foreach (var id in ftsScores.Keys) allIds.Add(id); + + var results = new List(); + foreach (var id in allIds) + { + if (!rowMap.ContainsKey(id)) continue; + var row = rowMap[id]; + + float vecScore = vectorScores.ContainsKey(id) ? vectorScores[id] : 0f; + float ftsScore = ftsScores.ContainsKey(id) ? ftsScores[id] : 0f; + float fusedScore = 0.6f * vecScore + 0.4f * ftsScore; + + // Recency boost: up to 15% for messages from last 30 days + if (DateTime.TryParse(row.CreatedAt, out var created)) + { + double hoursSince = (DateTime.UtcNow - created).TotalHours; + double recencyBoost = Math.Max(0, 1.0 - hoursSince / (30.0 * 24.0)) * 0.15; + fusedScore *= (float)(1.0 + recencyBoost); + } + + if (fusedScore >= minScore) { results.Add(new SearchResult { @@ -172,7 +290,8 @@ public class VectorSearchService AgentId = row.AgentId, Role = row.Role, TextPreview = row.TextPreview, - Score = score, + FullText = row.FullText ?? row.TextPreview, + Score = fusedScore, CreatedAt = row.CreatedAt }); } @@ -208,8 +327,8 @@ public class VectorSearchService if (!_embedding.IsAvailable || string.IsNullOrWhiteSpace(text)) return; if (text.Length < 10) return; - // Chunks erstellen: Memory-Dateien können lang sein → in 500-Zeichen-Chunks aufteilen - var chunks = ChunkText(text, 500); + // Chunks erstellen mit Overlap + var chunks = ChunkText(text, 500, 100); using var conn = CreateConnection(); @@ -218,6 +337,19 @@ public class VectorSearchService "DELETE FROM memory_file_embeddings WHERE agent_id = @agentId AND file_path = @filePath", new { agentId, filePath }); + // Also clean up old FTS entries for this file's chunks + try + { + var oldIds = await conn.QueryAsync( + "SELECT rowid FROM memory_file_embeddings WHERE agent_id = @agentId AND file_path = @filePath", + new { agentId, filePath }); + foreach (var oldId in oldIds) + { + try { await conn.ExecuteAsync("INSERT INTO memory_embeddings_fts(memory_embeddings_fts, rowid, full_text) VALUES('delete', @id, '')", new { id = oldId }); } catch { } + } + } + catch { } + for (int i = 0; i < chunks.Count; i++) { float[] vector; @@ -229,9 +361,9 @@ public class VectorSearchService var preview = chunks[i].Length > 200 ? chunks[i].Substring(0, 200) : chunks[i]; await conn.ExecuteAsync(@" INSERT INTO memory_file_embeddings - (agent_id, file_path, chunk_index, text_preview, vector, updated_at) + (agent_id, file_path, chunk_index, text_preview, vector, updated_at, full_text, embedding_model) VALUES - (@agentId, @filePath, @chunkIndex, @preview, @vector, @updatedAt)", + (@agentId, @filePath, @chunkIndex, @preview, @vector, @updatedAt, @fullText, @embeddingModel)", new { agentId, @@ -239,13 +371,29 @@ public class VectorSearchService chunkIndex = i, preview, vector = EmbeddingService.ToBytes(vector), - updatedAt = DateTime.UtcNow.ToString("o") + updatedAt = DateTime.UtcNow.ToString("o"), + fullText = chunks[i], + embeddingModel = _embedding.ModelName }); + + // Insert into FTS5 index + try + { + var lastId = await conn.ExecuteScalarAsync("SELECT last_insert_rowid()"); + if (lastId > 0) + { + await conn.ExecuteAsync( + "INSERT INTO memory_embeddings_fts(rowid, full_text) VALUES (@id, @fullText)", + new { id = lastId, fullText = chunks[i] }); + } + } + catch { /* FTS5 insert failure is non-critical */ } } } /// /// Sucht über Memory-Dateien eines Agents. + /// Hybrid: Vektor-Cosine-Similarity + FTS5-Keyword + Recency-Boost. /// public async Task> SearchMemoryFilesAsync(string query, string agentId, int topN = 5, float minScore = 0.4f) { @@ -259,15 +407,83 @@ public class VectorSearchService using var conn = CreateConnection(); var rows = await conn.QueryAsync( - "SELECT agent_id, file_path, chunk_index, text_preview, vector, updated_at FROM memory_file_embeddings WHERE agent_id = @agentId", + "SELECT id, agent_id, file_path, chunk_index, text_preview, full_text, vector, updated_at FROM memory_file_embeddings WHERE agent_id = @agentId", new { agentId }); + var rowList = rows.ToList(); - var results = new List(); - foreach (var row in rows) + // --- Vector scores --- + var vectorScores = new Dictionary(); + var rowMap = new Dictionary(); + foreach (var row in rowList) { var vec = EmbeddingService.FromBytes(row.Vector); var score = EmbeddingService.CosineSimilarity(queryVec, vec); - if (score >= minScore) + vectorScores[row.Id] = score; + rowMap[row.Id] = row; + } + + // --- FTS5 keyword scores --- + var ftsScores = new Dictionary(); + try + { + var sanitized = SanitizeFtsQuery(query); + if (!string.IsNullOrWhiteSpace(sanitized)) + { + var ftsSql = @"SELECT me.id, f.rank + FROM memory_embeddings_fts f + JOIN memory_file_embeddings me ON me.rowid = f.rowid + WHERE memory_embeddings_fts MATCH @query AND me.agent_id = @agentId + ORDER BY f.rank LIMIT @limit"; + var ftsParam = new { query = sanitized, agentId, limit = topN * 3 }; + + var ftsRows = await conn.QueryAsync(ftsSql, ftsParam); + var ftsList = ftsRows.ToList(); + + if (ftsList.Count > 0) + { + var minRank = ftsList.Min(r => r.Rank); + var maxRank = ftsList.Max(r => r.Rank); + var range = maxRank - minRank; + foreach (var r in ftsList) + { + float normalized = range == 0 ? 1.0f : (float)((maxRank - r.Rank) / range); + ftsScores[r.Id] = normalized; + if (!rowMap.ContainsKey(r.Id)) + { + var row = await conn.QueryFirstOrDefaultAsync( + "SELECT id, agent_id, file_path, chunk_index, text_preview, full_text, vector, updated_at FROM memory_file_embeddings WHERE id = @id", + new { id = r.Id }); + if (row != null) rowMap[r.Id] = row; + } + } + } + } + } + catch { /* FTS5 query failure — fall back to vector-only */ } + + // --- Score fusion --- + var allIds = new HashSet(vectorScores.Keys); + foreach (var id in ftsScores.Keys) allIds.Add(id); + + var results = new List(); + foreach (var id in allIds) + { + if (!rowMap.ContainsKey(id)) continue; + var row = rowMap[id]; + + float vecScore = vectorScores.ContainsKey(id) ? vectorScores[id] : 0f; + float ftsScore = ftsScores.ContainsKey(id) ? ftsScores[id] : 0f; + float fusedScore = 0.6f * vecScore + 0.4f * ftsScore; + + // Recency boost + if (DateTime.TryParse(row.UpdatedAt, out var updated)) + { + double hoursSince = (DateTime.UtcNow - updated).TotalHours; + double recencyBoost = Math.Max(0, 1.0 - hoursSince / (30.0 * 24.0)) * 0.15; + fusedScore *= (float)(1.0 + recencyBoost); + } + + if (fusedScore >= minScore) { results.Add(new MemorySearchResult { @@ -275,7 +491,8 @@ public class VectorSearchService FilePath = row.FilePath, ChunkIndex = row.ChunkIndex, TextPreview = row.TextPreview, - Score = score, + FullText = row.FullText ?? row.TextPreview, + Score = fusedScore, UpdatedAt = row.UpdatedAt }); } @@ -284,7 +501,34 @@ public class VectorSearchService return results.OrderByDescending(r => r.Score).Take(topN).ToList(); } - static List ChunkText(string text, int chunkSize) + /// + /// Löscht alle Embeddings zur Neugenerierung. + /// + public async Task ReEmbedAllAsync() + { + using var conn = CreateConnection(); + await conn.ExecuteAsync("DELETE FROM message_embeddings"); + await conn.ExecuteAsync("DELETE FROM memory_file_embeddings"); + // Reset FTS5 tables + try { await conn.ExecuteAsync("DELETE FROM message_embeddings_fts"); } catch { } + try { await conn.ExecuteAsync("DELETE FROM memory_embeddings_fts"); } catch { } + Console.WriteLine("[VectorSearch] All embeddings cleared for re-embedding."); + } + + /// + /// Sanitizes a query string for FTS5 MATCH by removing special operators. + /// + static string SanitizeFtsQuery(string query) + { + if (string.IsNullOrWhiteSpace(query)) return ""; + // Remove FTS5 special characters: * " ( ) : ^ - + AND OR NOT NEAR + var sanitized = Regex.Replace(query, @"[""()*:^+\-]", " "); + sanitized = Regex.Replace(sanitized, @"\b(AND|OR|NOT|NEAR)\b", " ", RegexOptions.IgnoreCase); + sanitized = Regex.Replace(sanitized, @"\s+", " ").Trim(); + return sanitized; + } + + static List ChunkText(string text, int chunkSize, int overlap = 0) { var chunks = new List(); if (string.IsNullOrEmpty(text)) return chunks; @@ -293,12 +537,42 @@ public class VectorSearchService var paragraphs = text.Split(new[] { "\n\n", "\r\n\r\n" }, StringSplitOptions.RemoveEmptyEntries); var current = new StringBuilder(); + // Track paragraph positions for overlap + var paragraphStarts = new List(); + var paragraphTexts = new List(); + int pos = 0; foreach (var para in paragraphs) { + paragraphStarts.Add(pos); + paragraphTexts.Add(para); + pos += para.Length + 2; // approximate separator length + } + + for (int pi = 0; pi < paragraphTexts.Count; pi++) + { + var para = paragraphTexts[pi]; if (current.Length + para.Length > chunkSize && current.Length > 0) { chunks.Add(current.ToString().Trim()); + + // Apply overlap: backtrack to include some text from end of current chunk current.Clear(); + if (overlap > 0 && pi > 0) + { + // Collect paragraphs from the end until we have ~overlap chars + int overlapChars = 0; + int startPi = pi; + for (int bi = pi - 1; bi >= 0 && overlapChars < overlap; bi--) + { + overlapChars += paragraphTexts[bi].Length; + startPi = bi; + } + for (int bi = startPi; bi < pi; bi++) + { + if (current.Length > 0) current.AppendLine(); + current.Append(paragraphTexts[bi]); + } + } } if (current.Length > 0) current.AppendLine(); current.Append(para); @@ -309,12 +583,20 @@ public class VectorSearchService return chunks; } + class FtsRow + { + public long Id { get; set; } + public double Rank { get; set; } + } + public class MemoryFileRow { + public long Id { get; set; } public string AgentId { get; set; } public string FilePath { get; set; } public int ChunkIndex { get; set; } public string TextPreview { get; set; } + public string FullText { get; set; } public byte[] Vector { get; set; } public string UpdatedAt { get; set; } } @@ -325,18 +607,21 @@ public class VectorSearchService public string FilePath { get; set; } public int ChunkIndex { get; set; } public string TextPreview { get; set; } + public string FullText { get; set; } public float Score { get; set; } public string UpdatedAt { get; set; } } public class EmbeddingRow { + public long Id { get; set; } public long MessageId { get; set; } public string ChainId { get; set; } public string SessionId { get; set; } public string AgentId { get; set; } public string Role { get; set; } public string TextPreview { get; set; } + public string FullText { get; set; } public byte[] Vector { get; set; } public string CreatedAt { get; set; } } @@ -349,6 +634,7 @@ public class VectorSearchService public string AgentId { get; set; } public string Role { get; set; } public string TextPreview { get; set; } + public string FullText { get; set; } public float Score { get; set; } public string CreatedAt { get; set; } } diff --git a/wwwroot/css/form.css b/wwwroot/css/form.css index 453e123..81a5e80 100644 --- a/wwwroot/css/form.css +++ b/wwwroot/css/form.css @@ -856,6 +856,139 @@ a.admin-user-meta { color: var(--primary); } +/* ── Embedding Debug ── */ + +.emb-summary { margin-bottom: 0.5rem } + +.emb-summary-grid { + display: flex; + flex-wrap: wrap; + gap: var(--badge-spacing); +} + +.emb-stat { + display: flex; + flex-direction: column; + align-items: center; + padding: var(--box-padding); + background: var(--f1-bg); + border: 1px solid var(--f1-border); + border-radius: var(--border-radius); + min-width: 90px; +} + +.emb-stat-num { + font-size: 1.3em; + font-weight: bold; + font-variant-numeric: tabular-nums; +} + +.emb-stat-label { + opacity: var(--semi-transparent); +} + +.emb-stat.good .emb-stat-num { + color: var(--secondary); +} + +.emb-stat.warn .emb-stat-num { + color: var(--warning); +} + +.emb-results { + display: flex; + flex-direction: column; + gap: var(--badge-spacing); +} + +.emb-result-item { + display: flex; + flex-direction: column; + gap: var(--badge-spacing); + padding: var(--box-padding); + background: var(--f1-bg); + border: 1px solid var(--f1-border); + border-radius: var(--border-radius); +} + +.emb-result-header { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--badge-spacing); +} + +.emb-rank { + font-weight: bold; + opacity: var(--semi-transparent); + min-width: 2rem; +} + +.emb-score { + font-weight: bold; + font-variant-numeric: tabular-nums; + padding: var(--badge-padding); + border-radius: var(--border-radius); + font-size: var(--font-size); +} + +.emb-score.score-good { + background: rgba(76, 175, 80, 0.2); + color: #4caf50; +} + +.emb-score.score-medium { + background: rgba(255, 152, 0, 0.2); + color: #ff9800; +} + +.emb-score.score-low { + background: rgba(244, 67, 54, 0.15); + color: #f44336; +} + +.emb-role { + opacity: var(--semi-transparent); + text-transform: uppercase; +} + +.emb-date { + opacity: var(--semi-transparent); +} + +.emb-tag { + font-size: var(--font-size); + padding: var(--badge-padding); + border-radius: var(--border-radius); + font-weight: bold; + text-transform: uppercase; +} + +.emb-tag.pass { + background: rgba(76, 175, 80, 0.2); + color: #4caf50; +} + +.emb-tag.threshold { + background: rgba(33, 150, 243, 0.2); + color: #2196f3; +} + +.emb-tag.filter { + background: rgba(244, 67, 54, 0.15); + color: #f44336; +} + +.emb-result-text { + word-break: break-word; + white-space: pre-wrap; +} + +.emb-result-meta { + opacity: var(--semi-transparent); + font-variant-numeric: tabular-nums; +} + /* ── Utility Classes (Inline-Style Replacements) ── */ .clickable { cursor: pointer } diff --git a/wwwroot/index.html b/wwwroot/index.html index 21ca93d..a6ba14a 100644 --- a/wwwroot/index.html +++ b/wwwroot/index.html @@ -65,6 +65,7 @@ + @@ -569,6 +570,38 @@ + +