feat: OpenClaw session import with sorted history
Backend: - New OpenClawEndpoint with GET /api/openclaw/sessions and POST /api/openclaw/import - Lists all OpenClaw sessions with preview messages, model, summary - Imports user/assistant text messages (skips tool-use/tool-result) - Cleans user messages (removes webchat prefixes, cron tags, conversation info blocks) - Re-import support: deletes previous openclaw-import chains before importing - New DatabaseService methods: CreateChainForImport, CreateMessageForImport, DeleteOpenClawImport - Added SortKey to MessageRecord (computed: openclaw-import gets id-10B, else id) History sorting: - OpenClaw-imported messages always appear before native messages - Sort by CASE WHEN tag='openclaw-import' THEN id-10000000000 ELSE id END - Cursor-based paging updated from beforeId/afterId to beforeSortKey/afterSortKey - All history queries (GetLatestActive, Before, After, CountBefore) updated Frontend: - Dashboard: new 'OpenClaw Import' menu item in session card dropdown - Modal: shows OpenClaw sessions with metadata, summary, preview messages - Imported messages display blue 'OC' badge in chat - API functions: apiGetOpenclawSessions, apiImportOpenclawSession - History paging updated to use sortKey cursors
This commit is contained in:
Meleki committed
1 parent
1597c10a3c
commit
3b7bdb7a20
10 files changed
+750
-38
No files matched your search
@@ -43,16 +43,16 @@ public static class HistoryEndpoint
|
||||
}
|
||||
|
||||
const int messageLimit = 500;
|
||||
var beforeStr = ctx.Request.Query["beforeId"].FirstOrDefault();
|
||||
var beforeId = long.TryParse(beforeStr, out var b) ? b : 0;
|
||||
var beforeStr = ctx.Request.Query["beforeSortKey"].FirstOrDefault();
|
||||
var beforeSortKey = long.TryParse(beforeStr, out var b) ? b : 0;
|
||||
var includeDeleted = ctx.Request.Query["includeDeleted"].FirstOrDefault() == "true";
|
||||
|
||||
var totalCount = await db.GetActiveMessageCount(sessionId, includeDeleted);
|
||||
|
||||
List<MessageRecord> messages;
|
||||
if (beforeId > 0)
|
||||
if (beforeSortKey != 0)
|
||||
{
|
||||
messages = await db.GetActiveMessagesBeforeId(sessionId, beforeId, messageLimit, includeDeleted);
|
||||
messages = await db.GetActiveMessagesBeforeSortKey(sessionId, beforeSortKey, messageLimit, includeDeleted);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -60,9 +60,9 @@ public static class HistoryEndpoint
|
||||
}
|
||||
|
||||
// Zähle wie viele Nachrichten es VOR der ältesten geladenen noch gibt
|
||||
var oldestLoadedId = messages.Count > 0 ? messages[0].Id : 0;
|
||||
var remainingOlder = oldestLoadedId > 0
|
||||
? await db.GetActiveMessageCountBeforeId(sessionId, oldestLoadedId, includeDeleted)
|
||||
var oldestLoadedSortKey = messages.Count > 0 ? messages[0].SortKey : 0;
|
||||
var remainingOlder = oldestLoadedSortKey != 0
|
||||
? await db.GetActiveMessageCountBeforeSortKey(sessionId, oldestLoadedSortKey, includeDeleted)
|
||||
: 0;
|
||||
|
||||
// Nach Chain gruppieren (Reihenfolge beibehalten)
|
||||
@@ -91,8 +91,8 @@ public static class HistoryEndpoint
|
||||
private static async Task HandlePoll(HttpContext ctx, DatabaseService db)
|
||||
{
|
||||
var sessionId = ctx.Request.Query["sessionId"].FirstOrDefault();
|
||||
var afterStr = ctx.Request.Query["afterId"].FirstOrDefault();
|
||||
var afterId = long.TryParse(afterStr, out var a) ? a : 0;
|
||||
var afterStr = ctx.Request.Query["afterSortKey"].FirstOrDefault();
|
||||
var afterSortKey = long.TryParse(afterStr, out var a) ? a : 0;
|
||||
var includeDeleted = ctx.Request.Query["includeDeleted"].FirstOrDefault() == "true";
|
||||
|
||||
if (string.IsNullOrEmpty(sessionId) || !await VerifySessionOwner(ctx, db, sessionId))
|
||||
@@ -101,7 +101,7 @@ public static class HistoryEndpoint
|
||||
return;
|
||||
}
|
||||
|
||||
var messages = await db.GetActiveMessagesAfterId(sessionId, afterId, includeDeleted: includeDeleted);
|
||||
var messages = await db.GetActiveMessagesAfterSortKey(sessionId, afterSortKey, includeDeleted: includeDeleted);
|
||||
await ctx.Response.WriteAsJsonAsync(messages);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using Dapper;
|
||||
using Gott.Extensions;
|
||||
using Gott.Services;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace Gott.Endpoints;
|
||||
|
||||
public static class OpenClawEndpoint
|
||||
{
|
||||
private const string OpenClawDbRelativePath = "publish/data/openclaw-sessions.db";
|
||||
private const string ImportTag = "openclaw-import";
|
||||
|
||||
public static void MapOpenClawEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapGet("/api/openclaw/sessions", HandleListSessions);
|
||||
app.MapPost("/api/openclaw/import", HandleImport);
|
||||
}
|
||||
|
||||
private static string GetOpenClawDbPath()
|
||||
{
|
||||
// Resolve relative to the app base directory
|
||||
var basePath = AppContext.BaseDirectory;
|
||||
// Navigate up from bin/Debug/net10.0 or similar to the project root
|
||||
var candidate = Path.Combine(basePath, OpenClawDbRelativePath);
|
||||
if (File.Exists(candidate)) return candidate;
|
||||
|
||||
// Try from current directory
|
||||
candidate = Path.Combine(Directory.GetCurrentDirectory(), OpenClawDbRelativePath);
|
||||
if (File.Exists(candidate)) return candidate;
|
||||
|
||||
// Try absolute common path
|
||||
candidate = Path.Combine("/app/gott", OpenClawDbRelativePath);
|
||||
if (File.Exists(candidate)) return candidate;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static SqliteConnection OpenReadOnly(string dbPath)
|
||||
{
|
||||
var conn = new SqliteConnection($"Data Source={dbPath};Mode=ReadOnly");
|
||||
conn.Open();
|
||||
DefaultTypeMap.MatchNamesWithUnderscores = true;
|
||||
return conn;
|
||||
}
|
||||
|
||||
// ── GET /api/openclaw/sessions ──
|
||||
|
||||
private static async Task HandleListSessions(HttpContext ctx)
|
||||
{
|
||||
var dbPath = GetOpenClawDbPath();
|
||||
if (dbPath == null)
|
||||
{
|
||||
ctx.Response.StatusCode = 404;
|
||||
await ctx.Response.WriteAsJsonAsync(new { error = "OpenClaw-Datenbank nicht gefunden" });
|
||||
return;
|
||||
}
|
||||
|
||||
using var conn = OpenReadOnly(dbPath);
|
||||
|
||||
// Get all sessions
|
||||
var sessions = (await conn.QueryAsync<dynamic>(
|
||||
"SELECT id, session, timestamp, substr(data,1,500) as data FROM entries WHERE type='session' ORDER BY timestamp DESC"
|
||||
)).ToList();
|
||||
|
||||
var result = new List<object>();
|
||||
|
||||
foreach (var s in sessions)
|
||||
{
|
||||
string sessionId = s.session;
|
||||
|
||||
// Count user + assistant text messages (skip toolResult)
|
||||
var messageCounts = await conn.QuerySingleAsync<dynamic>(@"
|
||||
SELECT
|
||||
COUNT(CASE WHEN json_extract(data,'$.message.role')='user' THEN 1 END) as userCount,
|
||||
COUNT(CASE WHEN json_extract(data,'$.message.role')='assistant' THEN 1 END) as assistantCount
|
||||
FROM entries
|
||||
WHERE type='message' AND session=@sessionId",
|
||||
new { sessionId });
|
||||
|
||||
int userCount = (int)(long)messageCounts.userCount;
|
||||
int assistantCount = (int)(long)messageCounts.assistantCount;
|
||||
int totalMessages = userCount + assistantCount;
|
||||
|
||||
if (totalMessages == 0) continue; // Skip empty sessions
|
||||
|
||||
// Get chain count
|
||||
var chainCount = await conn.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(DISTINCT chainId) FROM entries WHERE type='message' AND session=@sessionId AND chainId IS NOT NULL",
|
||||
new { sessionId });
|
||||
|
||||
// Get most common model
|
||||
var model = await conn.ExecuteScalarAsync<string>(@"
|
||||
SELECT json_extract(data,'$.message.model')
|
||||
FROM entries
|
||||
WHERE type='message' AND session=@sessionId AND json_extract(data,'$.message.role')='assistant' AND json_extract(data,'$.message.model') IS NOT NULL
|
||||
GROUP BY json_extract(data,'$.message.model')
|
||||
ORDER BY COUNT(*) DESC LIMIT 1",
|
||||
new { sessionId });
|
||||
|
||||
// Get summary from latest compaction
|
||||
var summaryRaw = await conn.ExecuteScalarAsync<string>(@"
|
||||
SELECT json_extract(data,'$.summary')
|
||||
FROM entries
|
||||
WHERE type='compaction' AND session=@sessionId
|
||||
ORDER BY timestamp DESC LIMIT 1",
|
||||
new { sessionId });
|
||||
var summary = summaryRaw != null && summaryRaw.Length > 200 ? summaryRaw.Substring(0, 200) + "…" : summaryRaw;
|
||||
|
||||
// Get last 5 user/assistant text messages for preview
|
||||
var previewEntries = (await conn.QueryAsync<dynamic>(@"
|
||||
SELECT json_extract(data,'$.message.role') as role, data
|
||||
FROM entries
|
||||
WHERE type='message' AND session=@sessionId
|
||||
AND json_extract(data,'$.message.role') IN ('user','assistant')
|
||||
ORDER BY timestamp DESC LIMIT 8",
|
||||
new { sessionId })).ToList();
|
||||
|
||||
var previewMessages = new List<object>();
|
||||
foreach (var pe in previewEntries)
|
||||
{
|
||||
string role = pe.role;
|
||||
string dataJson = pe.data;
|
||||
|
||||
var text = ExtractTextFromEntryData(dataJson);
|
||||
if (string.IsNullOrWhiteSpace(text)) continue;
|
||||
|
||||
if (role == "user")
|
||||
text = CleanUserMessage(text);
|
||||
|
||||
if (text.Length > 120)
|
||||
text = text.Substring(0, 120) + "…";
|
||||
|
||||
previewMessages.Add(new { role, text });
|
||||
if (previewMessages.Count >= 5) break;
|
||||
}
|
||||
|
||||
previewMessages.Reverse(); // Chronological order
|
||||
|
||||
result.Add(new
|
||||
{
|
||||
sessionId,
|
||||
createdAt = (string)s.timestamp,
|
||||
userMessages = userCount,
|
||||
assistantMessages = assistantCount,
|
||||
totalMessages,
|
||||
chainCount,
|
||||
model = model ?? "unknown",
|
||||
summary,
|
||||
previewMessages
|
||||
});
|
||||
}
|
||||
|
||||
await ctx.Response.WriteAsJsonAsync(result);
|
||||
}
|
||||
|
||||
// ── POST /api/openclaw/import ──
|
||||
|
||||
private static async Task HandleImport(HttpContext ctx, DatabaseService db)
|
||||
{
|
||||
var userId = ctx.GetUserId();
|
||||
var body = await JsonSerializer.DeserializeAsync<JsonElement>(ctx.Request.Body);
|
||||
|
||||
var openclawSessionId = body.TryGetProperty("openclawSessionId", out var ocs) ? ocs.GetString() : null;
|
||||
var sessionId = body.TryGetProperty("sessionId", out var sid) ? sid.GetString() : null;
|
||||
|
||||
if (string.IsNullOrEmpty(openclawSessionId) || string.IsNullOrEmpty(sessionId))
|
||||
{
|
||||
ctx.Response.StatusCode = 400;
|
||||
await ctx.Response.WriteAsJsonAsync(new { error = "openclawSessionId und sessionId sind erforderlich" });
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate target session
|
||||
var session = await db.GetSession(sessionId);
|
||||
if (session == null || session.UserId != userId)
|
||||
{
|
||||
ctx.Response.StatusCode = 404;
|
||||
await ctx.Response.WriteAsJsonAsync(new { error = "Session nicht gefunden" });
|
||||
return;
|
||||
}
|
||||
|
||||
var dbPath = GetOpenClawDbPath();
|
||||
if (dbPath == null)
|
||||
{
|
||||
ctx.Response.StatusCode = 404;
|
||||
await ctx.Response.WriteAsJsonAsync(new { error = "OpenClaw-Datenbank nicht gefunden" });
|
||||
return;
|
||||
}
|
||||
|
||||
// Delete previous imports for this session (update scenario)
|
||||
await db.DeleteOpenClawImport(sessionId);
|
||||
|
||||
// Read all messages from OpenClaw
|
||||
List<dynamic> entries;
|
||||
using (var conn = OpenReadOnly(dbPath))
|
||||
{
|
||||
entries = (await conn.QueryAsync<dynamic>(@"
|
||||
SELECT id, chainId, timestamp, data
|
||||
FROM entries
|
||||
WHERE type='message' AND session=@openclawSessionId
|
||||
ORDER BY timestamp ASC",
|
||||
new { openclawSessionId })).ToList();
|
||||
}
|
||||
|
||||
if (entries.Count == 0)
|
||||
{
|
||||
ctx.Response.StatusCode = 400;
|
||||
await ctx.Response.WriteAsJsonAsync(new { error = "Keine Nachrichten in der OpenClaw-Session gefunden" });
|
||||
return;
|
||||
}
|
||||
|
||||
// Group by chainId
|
||||
var chains = new Dictionary<string, List<dynamic>>();
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
string chainId = entry.chainId ?? "__default__";
|
||||
if (!chains.ContainsKey(chainId))
|
||||
chains[chainId] = new List<dynamic>();
|
||||
chains[chainId].Add(entry);
|
||||
}
|
||||
|
||||
int totalChains = 0;
|
||||
int totalMessages = 0;
|
||||
string agentId = session.AgentId;
|
||||
|
||||
foreach (var kvp in chains)
|
||||
{
|
||||
var chainEntries = kvp.Value;
|
||||
var chainMessages = new List<(string role, string type, string content, string createdAt)>();
|
||||
|
||||
string chainModel = "unknown";
|
||||
int inputTokens = 0;
|
||||
int outputTokens = 0;
|
||||
int cacheCreationTokens = 0;
|
||||
int cacheReadTokens = 0;
|
||||
string firstTimestamp = null;
|
||||
|
||||
foreach (var entry in chainEntries)
|
||||
{
|
||||
string dataJson = (string)entry.data;
|
||||
string timestamp = (string)entry.timestamp;
|
||||
|
||||
if (firstTimestamp == null) firstTimestamp = timestamp;
|
||||
|
||||
JsonElement msgData;
|
||||
try
|
||||
{
|
||||
msgData = JsonSerializer.Deserialize<JsonElement>(dataJson);
|
||||
}
|
||||
catch { continue; }
|
||||
|
||||
if (!msgData.TryGetProperty("message", out var message)) continue;
|
||||
|
||||
var role = message.TryGetProperty("role", out var r) ? r.GetString() : null;
|
||||
if (role == null) continue;
|
||||
|
||||
// Skip toolResult entirely
|
||||
if (role == "toolResult") continue;
|
||||
|
||||
// Extract model and usage from assistant messages
|
||||
if (role == "assistant")
|
||||
{
|
||||
if (message.TryGetProperty("model", out var modelEl))
|
||||
{
|
||||
var m = modelEl.GetString();
|
||||
if (!string.IsNullOrEmpty(m)) chainModel = m;
|
||||
}
|
||||
if (message.TryGetProperty("usage", out var usage))
|
||||
{
|
||||
if (usage.TryGetProperty("input", out var inp)) inputTokens += inp.GetInt32();
|
||||
if (usage.TryGetProperty("output", out var outp)) outputTokens += outp.GetInt32();
|
||||
if (usage.TryGetProperty("cacheWrite", out var cw)) cacheCreationTokens += cw.GetInt32();
|
||||
if (usage.TryGetProperty("cacheRead", out var cr)) cacheReadTokens += cr.GetInt32();
|
||||
}
|
||||
}
|
||||
|
||||
// Extract content
|
||||
if (!message.TryGetProperty("content", out var contentArr) || contentArr.ValueKind != JsonValueKind.Array)
|
||||
continue;
|
||||
|
||||
// Process content blocks
|
||||
foreach (var block in contentArr.EnumerateArray())
|
||||
{
|
||||
var blockType = block.TryGetProperty("type", out var bt) ? bt.GetString() : null;
|
||||
|
||||
if (blockType == "text")
|
||||
{
|
||||
var text = block.TryGetProperty("text", out var t) ? t.GetString() : null;
|
||||
if (string.IsNullOrWhiteSpace(text)) continue;
|
||||
|
||||
// Skip tool-use and tool-result blocks
|
||||
if (role == "user")
|
||||
text = CleanUserMessage(text);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
chainMessages.Add((
|
||||
role == "user" ? "user" : "assistant",
|
||||
"text",
|
||||
text,
|
||||
timestamp
|
||||
));
|
||||
}
|
||||
}
|
||||
// Skip thinking, toolCall, toolResult blocks
|
||||
}
|
||||
}
|
||||
|
||||
// Skip chains with no text messages
|
||||
if (chainMessages.Count == 0) continue;
|
||||
|
||||
// Create chain record
|
||||
var newChainId = Guid.NewGuid().ToString("N");
|
||||
await db.CreateChainForImport(newChainId, sessionId, agentId, chainModel, inputTokens, outputTokens, cacheCreationTokens, cacheReadTokens, firstTimestamp ?? DateTime.UtcNow.ToString("o"));
|
||||
|
||||
// Create messages
|
||||
foreach (var (msgRole, msgType, content, createdAt) in chainMessages)
|
||||
{
|
||||
await db.CreateMessageForImport(newChainId, msgRole, msgType, content, createdAt, ImportTag);
|
||||
totalMessages++;
|
||||
}
|
||||
|
||||
totalChains++;
|
||||
}
|
||||
|
||||
await ctx.Response.WriteAsJsonAsync(new
|
||||
{
|
||||
imported = true,
|
||||
chainCount = totalChains,
|
||||
messageCount = totalMessages
|
||||
});
|
||||
}
|
||||
|
||||
// ── Helper Methods ──
|
||||
|
||||
private static string ExtractTextFromEntryData(string dataJson)
|
||||
{
|
||||
try
|
||||
{
|
||||
var data = JsonSerializer.Deserialize<JsonElement>(dataJson);
|
||||
if (!data.TryGetProperty("message", out var message)) return null;
|
||||
if (!message.TryGetProperty("content", out var contentArr) || contentArr.ValueKind != JsonValueKind.Array) return null;
|
||||
|
||||
foreach (var block in contentArr.EnumerateArray())
|
||||
{
|
||||
var blockType = block.TryGetProperty("type", out var bt) ? bt.GetString() : null;
|
||||
if (blockType == "text")
|
||||
{
|
||||
var text = block.TryGetProperty("text", out var t) ? t.GetString() : null;
|
||||
if (!string.IsNullOrWhiteSpace(text)) return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string CleanUserMessage(string text)
|
||||
{
|
||||
if (string.IsNullOrEmpty(text)) return text;
|
||||
|
||||
// Remove [Direct 1:1 webchat -- always respond, never use NO_REPLY]
|
||||
text = Regex.Replace(text, @"\[Direct 1:1 webchat\s*--\s*always respond,?\s*never use NO_REPLY\]\s*", "", RegexOptions.IgnoreCase);
|
||||
|
||||
// Remove [Current message - respond to this]\nUser:
|
||||
text = Regex.Replace(text, @"\[Current message\s*-\s*respond to this\]\s*User:\s*", "", RegexOptions.IgnoreCase);
|
||||
|
||||
// Remove [cron:...] prefix
|
||||
text = Regex.Replace(text, @"\[cron:[^\]]*\]\s*", "");
|
||||
|
||||
// Remove "Conversation info (untrusted metadata):\n```json\n{...}\n```" blocks
|
||||
text = Regex.Replace(text, @"Conversation info \(untrusted metadata\):\s*```[a-z]*\s*\{.*?\}\s*```\s*", "", RegexOptions.Singleline);
|
||||
|
||||
return text.Trim();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user