gott/Endpoints/OpenClawEndpoint.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Dapper;
using Gott.Extensions;
using Gott.Services;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using Microsoft.Data.Sqlite;

namespace Gott.Endpoints;

public static class OpenClawEndpoint
{
    private static readonly string OpenClawDbRelativePath = Path.Combine("data", "openclaw-sessions.db");
    private const string ImportTag = "openclaw-import";

    public static void MapOpenClawEndpoints(this IEndpointRouteBuilder app)
    {
        app.MapGet("/api/openclaw/sessions", HandleListSessions);
        app.MapPost("/api/openclaw/import", HandleImport);
    }

    private static string GetOpenClawDbPath()
    {
        var candidate = Path.Combine(AppContext.BaseDirectory, OpenClawDbRelativePath);
        if (File.Exists(candidate)) return candidate;

        return null;
    }

    private static SqliteConnection OpenReadOnly(string dbPath)
    {
        var conn = new SqliteConnection($"Data Source={dbPath};Mode=ReadOnly");
        conn.Open();
        DefaultTypeMap.MatchNamesWithUnderscores = true;
        return conn;
    }

    // ── GET /api/openclaw/sessions ──

    private static async Task HandleListSessions(HttpContext ctx)
    {
        var dbPath = GetOpenClawDbPath();
        if (dbPath == null)
        {
            ctx.Response.StatusCode = 404;
            await ctx.Response.WriteAsJsonAsync(new { error = "OpenClaw-Datenbank nicht gefunden" });
            return;
        }

        using var conn = OpenReadOnly(dbPath);

        // Get all sessions
        var sessions = (await conn.QueryAsync<dynamic>(
            "SELECT id, session, timestamp, substr(data,1,500) as data FROM entries WHERE type='session' ORDER BY timestamp DESC"
        )).ToList();

        var result = new List<object>();

        foreach (var s in sessions)
        {
            string sessionId = s.session;

            // Count user + assistant text messages (skip toolResult)
            var messageCounts = await conn.QuerySingleAsync<dynamic>(@"
                SELECT
                    COUNT(CASE WHEN json_extract(data,'$.message.role')='user' THEN 1 END) as userCount,
                    COUNT(CASE WHEN json_extract(data,'$.message.role')='assistant' THEN 1 END) as assistantCount
                FROM entries
                WHERE type='message' AND session=@sessionId",
                new { sessionId });

            int userCount = (int)(long)messageCounts.userCount;
            int assistantCount = (int)(long)messageCounts.assistantCount;
            int totalMessages = userCount + assistantCount;

            if (totalMessages == 0) continue; // Skip empty sessions

            // Get chain count
            var chainCount = await conn.ExecuteScalarAsync<int>(
                "SELECT COUNT(DISTINCT chainId) FROM entries WHERE type='message' AND session=@sessionId AND chainId IS NOT NULL",
                new { sessionId });

            // Get most common model
            var model = await conn.ExecuteScalarAsync<string>(@"
                SELECT json_extract(data,'$.message.model')
                FROM entries
                WHERE type='message' AND session=@sessionId AND json_extract(data,'$.message.role')='assistant' AND json_extract(data,'$.message.model') IS NOT NULL
                GROUP BY json_extract(data,'$.message.model')
                ORDER BY COUNT(*) DESC LIMIT 1",
                new { sessionId });

            // Get summary from latest compaction
            var summaryRaw = await conn.ExecuteScalarAsync<string>(@"
                SELECT json_extract(data,'$.summary')
                FROM entries
                WHERE type='compaction' AND session=@sessionId
                ORDER BY timestamp DESC LIMIT 1",
                new { sessionId });
            var summary = summaryRaw != null && summaryRaw.Length > 200 ? summaryRaw.Substring(0, 200) + "…" : summaryRaw;

            // Get last 5 user/assistant text messages for preview
            var previewEntries = (await conn.QueryAsync<dynamic>(@"
                SELECT json_extract(data,'$.message.role') as role, data
                FROM entries
                WHERE type='message' AND session=@sessionId
                  AND json_extract(data,'$.message.role') IN ('user','assistant')
                ORDER BY timestamp DESC LIMIT 8",
                new { sessionId })).ToList();

            var previewMessages = new List<object>();
            foreach (var pe in previewEntries)
            {
                string role = pe.role;
                string dataJson = pe.data;

                var text = ExtractTextFromEntryData(dataJson);
                if (string.IsNullOrWhiteSpace(text)) continue;

                if (role == "user")
                    text = CleanUserMessage(text);

                if (text.Length > 120)
                    text = text.Substring(0, 120) + "…";

                previewMessages.Add(new { role, text });
                if (previewMessages.Count >= 5) break;
            }

            previewMessages.Reverse(); // Chronological order

            result.Add(new
            {
                sessionId,
                createdAt = (string)s.timestamp,
                userMessages = userCount,
                assistantMessages = assistantCount,
                totalMessages,
                chainCount,
                model = model ?? "unknown",
                summary,
                previewMessages
            });
        }

        await ctx.Response.WriteAsJsonAsync(result);
    }

    // ── POST /api/openclaw/import ──

    private static async Task HandleImport(HttpContext ctx, DatabaseService db)
    {
        var userId = ctx.GetUserId();
        var body = await JsonSerializer.DeserializeAsync<JsonElement>(ctx.Request.Body);

        var openclawSessionId = body.TryGetProperty("openclawSessionId", out var ocs) ? ocs.GetString() : null;
        var sessionId = body.TryGetProperty("sessionId", out var sid) ? sid.GetString() : null;

        if (string.IsNullOrEmpty(openclawSessionId) || string.IsNullOrEmpty(sessionId))
        {
            ctx.Response.StatusCode = 400;
            await ctx.Response.WriteAsJsonAsync(new { error = "openclawSessionId und sessionId sind erforderlich" });
            return;
        }

        // Validate target session
        var session = await db.GetSession(sessionId);
        if (session == null || session.UserId != userId)
        {
            ctx.Response.StatusCode = 404;
            await ctx.Response.WriteAsJsonAsync(new { error = "Session nicht gefunden" });
            return;
        }

        var dbPath = GetOpenClawDbPath();
        if (dbPath == null)
        {
            ctx.Response.StatusCode = 404;
            await ctx.Response.WriteAsJsonAsync(new { error = "OpenClaw-Datenbank nicht gefunden" });
            return;
        }

        // Delete previous imports for this session (update scenario)
        await db.DeleteOpenClawImport(sessionId);

        // Read all messages from OpenClaw
        List<dynamic> entries;
        using (var conn = OpenReadOnly(dbPath))
        {
            entries = (await conn.QueryAsync<dynamic>(@"
                SELECT id, chainId, timestamp, data
                FROM entries
                WHERE type='message' AND session=@openclawSessionId
                ORDER BY timestamp ASC",
                new { openclawSessionId })).ToList();
        }

        if (entries.Count == 0)
        {
            ctx.Response.StatusCode = 400;
            await ctx.Response.WriteAsJsonAsync(new { error = "Keine Nachrichten in der OpenClaw-Session gefunden" });
            return;
        }

        // Group by chainId
        var chains = new Dictionary<string, List<dynamic>>();
        foreach (var entry in entries)
        {
            string chainId = entry.chainId ?? "__default__";
            if (!chains.ContainsKey(chainId))
                chains[chainId] = new List<dynamic>();
            chains[chainId].Add(entry);
        }

        int totalChains = 0;
        int totalMessages = 0;
        string agentId = session.AgentId;

        foreach (var kvp in chains)
        {
            var chainEntries = kvp.Value;
            var chainMessages = new List<(string role, string type, string content, string createdAt)>();

            string chainModel = "unknown";
            int inputTokens = 0;
            int outputTokens = 0;
            int cacheCreationTokens = 0;
            int cacheReadTokens = 0;
            string firstTimestamp = null;

            foreach (var entry in chainEntries)
            {
                string dataJson = (string)entry.data;
                string timestamp = (string)entry.timestamp;

                if (firstTimestamp == null) firstTimestamp = timestamp;

                JsonElement msgData;
                try
                {
                    msgData = JsonSerializer.Deserialize<JsonElement>(dataJson);
                }
                catch { continue; }

                if (!msgData.TryGetProperty("message", out var message)) continue;

                var role = message.TryGetProperty("role", out var r) ? r.GetString() : null;
                if (role == null) continue;

                // Skip toolResult entirely
                if (role == "toolResult") continue;

                // Extract model and usage from assistant messages
                if (role == "assistant")
                {
                    if (message.TryGetProperty("model", out var modelEl))
                    {
                        var m = modelEl.GetString();
                        if (!string.IsNullOrEmpty(m)) chainModel = m;
                    }
                    if (message.TryGetProperty("usage", out var usage))
                    {
                        if (usage.TryGetProperty("input", out var inp)) inputTokens += inp.GetInt32();
                        if (usage.TryGetProperty("output", out var outp)) outputTokens += outp.GetInt32();
                        if (usage.TryGetProperty("cacheWrite", out var cw)) cacheCreationTokens += cw.GetInt32();
                        if (usage.TryGetProperty("cacheRead", out var cr)) cacheReadTokens += cr.GetInt32();
                    }
                }

                // Extract content
                if (!message.TryGetProperty("content", out var contentArr) || contentArr.ValueKind != JsonValueKind.Array)
                    continue;

                // Process content blocks
                foreach (var block in contentArr.EnumerateArray())
                {
                    var blockType = block.TryGetProperty("type", out var bt) ? bt.GetString() : null;

                    if (blockType == "text")
                    {
                        var text = block.TryGetProperty("text", out var t) ? t.GetString() : null;
                        if (string.IsNullOrWhiteSpace(text)) continue;

                        // Skip tool-use and tool-result blocks
                        if (role == "user")
                            text = CleanUserMessage(text);

                        if (!string.IsNullOrWhiteSpace(text))
                        {
                            chainMessages.Add((
                                role == "user" ? "user" : "assistant",
                                "text",
                                text,
                                timestamp
                            ));
                        }
                    }
                    // Skip thinking, toolCall, toolResult blocks
                }
            }

            // Skip chains with no text messages
            if (chainMessages.Count == 0) continue;

            // Create chain record
            var newChainId = Guid.NewGuid().ToString("N");
            await db.CreateChainForImport(newChainId, sessionId, agentId, chainModel, inputTokens, outputTokens, cacheCreationTokens, cacheReadTokens, firstTimestamp ?? DateTime.UtcNow.ToString("o"));

            // Create messages
            foreach (var (msgRole, msgType, content, createdAt) in chainMessages)
            {
                await db.CreateMessageForImport(newChainId, msgRole, msgType, content, createdAt, ImportTag);
                totalMessages++;
            }

            totalChains++;
        }

        await ctx.Response.WriteAsJsonAsync(new
        {
            imported = true,
            chainCount = totalChains,
            messageCount = totalMessages
        });
    }

    // ── Helper Methods ──

    private static string ExtractTextFromEntryData(string dataJson)
    {
        try
        {
            var data = JsonSerializer.Deserialize<JsonElement>(dataJson);
            if (!data.TryGetProperty("message", out var message)) return null;
            if (!message.TryGetProperty("content", out var contentArr) || contentArr.ValueKind != JsonValueKind.Array) return null;

            foreach (var block in contentArr.EnumerateArray())
            {
                var blockType = block.TryGetProperty("type", out var bt) ? bt.GetString() : null;
                if (blockType == "text")
                {
                    var text = block.TryGetProperty("text", out var t) ? t.GetString() : null;
                    if (!string.IsNullOrWhiteSpace(text)) return text;
                }
            }
        }
        catch { }
        return null;
    }

    private static string CleanUserMessage(string text)
    {
        if (string.IsNullOrEmpty(text)) return text;

        // Remove [Direct 1:1 webchat -- always respond, never use NO_REPLY]
        text = Regex.Replace(text, @"\[Direct 1:1 webchat\s*--\s*always respond,?\s*never use NO_REPLY\]\s*", "", RegexOptions.IgnoreCase);

        // Remove [Current message - respond to this]\nUser:
        text = Regex.Replace(text, @"\[Current message\s*-\s*respond to this\]\s*User:\s*", "", RegexOptions.IgnoreCase);

        // Remove [cron:...] prefix
        text = Regex.Replace(text, @"\[cron:[^\]]*\]\s*", "");

        // Remove "Conversation info (untrusted metadata):\n```json\n{...}\n```" blocks
        text = Regex.Replace(text, @"Conversation info \(untrusted metadata\):\s*```[a-z]*\s*\{.*?\}\s*```\s*", "", RegexOptions.Singleline);

        return text.Trim();
    }
}