restructure: flatten project, add ARCHITECTURE.md, update DEVY.md
- Moved from session-analytics-web/ subfolder to root - Renamed to token-analytics - Added deploy.sh, ARCHITECTURE.md - Moved export.mjs into jsonl-sync/
This commit is contained in:
Meleki committed
1 parent
1d3070ed4c
commit
dbaad6b372
17 files changed
+340
-270
No files matched your search
+230
@@ -0,0 +1,230 @@
|
||||
using Dapper;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
var dbPath = builder.Configuration["DbPath"]!;
|
||||
|
||||
builder.Services.AddSingleton<Func<SqliteConnection>>(() =>
|
||||
new SqliteConnection($"Data Source={Path.GetFullPath(dbPath)}"));
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
app.UseDefaultFiles();
|
||||
app.UseStaticFiles();
|
||||
|
||||
// GET /api/cost-buckets?from=&to=&bucketMinutes=
|
||||
// Returns N buckets of aggregated cost
|
||||
app.MapGet("/api/cost-buckets", (Func<SqliteConnection> factory, string from, string to, int bucketMinutes) =>
|
||||
{
|
||||
using var db = factory();
|
||||
var rows = db.Query(@"
|
||||
SELECT
|
||||
CAST((strftime('%s', timestamp) - strftime('%s', @from)) / (@bucketSec) AS INTEGER) AS bucket,
|
||||
SUM(CAST(json_extract(data, '$.message.usage.cost.total') AS REAL)) AS cost,
|
||||
COUNT(*) AS count,
|
||||
MIN(timestamp) AS bucketStart,
|
||||
MAX(timestamp) AS bucketEnd
|
||||
FROM entries
|
||||
WHERE json_extract(data, '$.message.usage.cost.total') IS NOT NULL
|
||||
AND timestamp >= @from AND timestamp < @to
|
||||
GROUP BY bucket
|
||||
ORDER BY bucket ASC",
|
||||
new { from, to, bucketSec = bucketMinutes * 60 });
|
||||
|
||||
return Results.Ok(rows);
|
||||
});
|
||||
|
||||
// GET /api/cost-entries?from=&to=
|
||||
app.MapGet("/api/cost-entries", (Func<SqliteConnection> factory, string from, string to) =>
|
||||
{
|
||||
using var db = factory();
|
||||
var rows = db.Query(@"
|
||||
SELECT id, session, type, timestamp,
|
||||
json_extract(data, '$.message.usage.cost.total') AS cost,
|
||||
json_extract(data, '$.message.usage.input') AS inputTokens,
|
||||
json_extract(data, '$.message.usage.output') AS outputTokens,
|
||||
json_extract(data, '$.message.model') AS model,
|
||||
chainId,
|
||||
data
|
||||
FROM entries
|
||||
WHERE json_extract(data, '$.message.usage.cost.total') IS NOT NULL
|
||||
AND timestamp >= @from AND timestamp < @to
|
||||
ORDER BY timestamp ASC",
|
||||
new { from, to });
|
||||
|
||||
return Results.Ok(rows);
|
||||
});
|
||||
|
||||
// GET /api/cost-chains?from=&to=
|
||||
// Returns chains with aggregated cost, sorted by total cost desc
|
||||
app.MapGet("/api/cost-chains", (Func<SqliteConnection> factory, string from, string to, int limit = 50, int offset = 0) =>
|
||||
{
|
||||
using var db = factory();
|
||||
var rows = db.Query(@"
|
||||
SELECT chainId,
|
||||
SUM(CAST(json_extract(data, '$.message.usage.cost.total') AS REAL)) AS totalCost,
|
||||
COUNT(*) AS count,
|
||||
MIN(timestamp) AS first,
|
||||
MAX(timestamp) AS last
|
||||
FROM entries
|
||||
WHERE json_extract(data, '$.message.usage.cost.total') IS NOT NULL
|
||||
AND timestamp >= @from AND timestamp < @to
|
||||
AND chainId IS NOT NULL
|
||||
GROUP BY chainId
|
||||
ORDER BY totalCost DESC
|
||||
LIMIT @limit OFFSET @offset",
|
||||
new { from, to, limit, offset });
|
||||
|
||||
return Results.Ok(rows);
|
||||
});
|
||||
|
||||
// GET /api/cost-summary?from=&to=
|
||||
app.MapGet("/api/cost-summary", (Func<SqliteConnection> factory, string? from, string? to) =>
|
||||
{
|
||||
using var db = factory();
|
||||
var conditions = new List<string> { "json_extract(data, '$.message.usage.cost.total') IS NOT NULL" };
|
||||
if (from != null) conditions.Add("timestamp >= @from");
|
||||
if (to != null) conditions.Add("timestamp < @to");
|
||||
var where = string.Join(" AND ", conditions);
|
||||
|
||||
var row = db.QuerySingle($@"
|
||||
SELECT COALESCE(SUM(CAST(json_extract(data, '$.message.usage.cost.total') AS REAL)), 0) AS totalCost,
|
||||
COUNT(*) AS totalEntries
|
||||
FROM entries
|
||||
WHERE {where}",
|
||||
new { from, to });
|
||||
|
||||
return Results.Ok(row);
|
||||
});
|
||||
|
||||
// GET /api/entry-chain/{id} - full chain: walk back to user message, then forward to end
|
||||
app.MapGet("/api/entry-chain/{id}", (string id, Func<SqliteConnection> factory) =>
|
||||
{
|
||||
using var db = factory();
|
||||
|
||||
// Step 1: Walk backwards to find the user message (chain root)
|
||||
var backChain = new List<dynamic>();
|
||||
var currentId = id;
|
||||
var seen = new HashSet<string>();
|
||||
string? rootId = null;
|
||||
|
||||
while (currentId != null && seen.Add(currentId))
|
||||
{
|
||||
var row = db.QueryFirstOrDefault(
|
||||
"SELECT id, session, type, timestamp, data, json_extract(data, '$.parentId') AS parentId FROM entries WHERE id = @id",
|
||||
new { id = currentId });
|
||||
|
||||
if (row == null) break;
|
||||
|
||||
backChain.Add(row);
|
||||
|
||||
string dataStr = row.data;
|
||||
try
|
||||
{
|
||||
var parsed = System.Text.Json.JsonDocument.Parse(dataStr);
|
||||
var role = parsed.RootElement.TryGetProperty("message", out var msg)
|
||||
&& msg.TryGetProperty("role", out var r) ? r.GetString() : null;
|
||||
if (role == "user")
|
||||
{
|
||||
rootId = (string)row.id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
|
||||
currentId = (string?)row.parentId;
|
||||
}
|
||||
|
||||
backChain.Reverse();
|
||||
|
||||
// Step 2: Walk forwards from the last entry in backChain until we hit the next user message
|
||||
var chainIds = new HashSet<string>(backChain.Select(r => (string)r.id));
|
||||
var forwardEntries = new List<dynamic>();
|
||||
|
||||
var lastId = (string)backChain.Last().id;
|
||||
var maxIter = 100;
|
||||
var currentId2 = lastId;
|
||||
while (maxIter-- > 0)
|
||||
{
|
||||
var child = db.QueryFirstOrDefault(
|
||||
"SELECT id, session, type, timestamp, data, json_extract(data, '$.message.role') AS role FROM entries WHERE json_extract(data, '$.parentId') = @pid",
|
||||
new { pid = currentId2 });
|
||||
|
||||
if (child == null) break;
|
||||
|
||||
string childRole = child.role;
|
||||
// Stop when we reach the next user message (new turn)
|
||||
if (childRole == "user") break;
|
||||
|
||||
string childId = child.id;
|
||||
if (!chainIds.Add(childId)) break;
|
||||
|
||||
forwardEntries.Add(new { id = childId, session = (string)child.session, type = (string)child.type, timestamp = (string)child.timestamp, data = (string)child.data });
|
||||
currentId2 = childId;
|
||||
}
|
||||
|
||||
// Combine and sort chronologically
|
||||
var fullChain = backChain.Concat(forwardEntries)
|
||||
.OrderBy(r => (string)r.timestamp)
|
||||
.Select(r => new { id = (string)r.id, session = (string)r.session, type = (string)r.type, timestamp = (string)r.timestamp, data = (string)r.data })
|
||||
.ToList();
|
||||
|
||||
return Results.Ok(fullChain);
|
||||
});
|
||||
|
||||
// GET /api/entry-chains?from=&to= - returns cost entries with cached chainId
|
||||
app.MapGet("/api/entry-chains", (Func<SqliteConnection> factory, string from, string to) =>
|
||||
{
|
||||
using var db = factory();
|
||||
var rows = db.Query(@"
|
||||
SELECT id, session, type, timestamp,
|
||||
json_extract(data, '$.message.usage.cost.total') AS cost,
|
||||
json_extract(data, '$.message.usage.input') AS inputTokens,
|
||||
json_extract(data, '$.message.usage.output') AS outputTokens,
|
||||
json_extract(data, '$.message.model') AS model,
|
||||
chainId,
|
||||
data
|
||||
FROM entries
|
||||
WHERE json_extract(data, '$.message.usage.cost.total') IS NOT NULL
|
||||
AND timestamp >= @from AND timestamp < @to
|
||||
ORDER BY timestamp ASC",
|
||||
new { from, to }).AsList();
|
||||
|
||||
// Map chainId to sequential index (for coloring)
|
||||
var chainIndex = new Dictionary<string, int>();
|
||||
int nextChain = 0;
|
||||
var result = rows.Select(e =>
|
||||
{
|
||||
string cid = e.chainId ?? e.id;
|
||||
if (!chainIndex.ContainsKey(cid)) chainIndex[cid] = nextChain++;
|
||||
return new
|
||||
{
|
||||
e.id, e.session, e.type, e.timestamp,
|
||||
e.cost, e.inputTokens, e.outputTokens, e.model,
|
||||
e.data,
|
||||
chainId = chainIndex[cid]
|
||||
};
|
||||
});
|
||||
|
||||
return Results.Ok(result);
|
||||
});
|
||||
|
||||
// POST /api/sync - run export.mjs to import new entries
|
||||
app.MapPost("/api/sync", async () =>
|
||||
{
|
||||
var wd = "/home/node/.openclaw/workspace/projects/token-analytics/jsonl-sync";
|
||||
var psi = new System.Diagnostics.ProcessStartInfo("docker", $"exec meleki node {wd}/export.mjs")
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
WorkingDirectory = wd!
|
||||
};
|
||||
var proc = System.Diagnostics.Process.Start(psi)!;
|
||||
var stdout = await proc.StandardOutput.ReadToEndAsync();
|
||||
var stderr = await proc.StandardError.ReadToEndAsync();
|
||||
await proc.WaitForExitAsync();
|
||||
return Results.Ok(new { exitCode = proc.ExitCode, output = (stdout + stderr).Trim() });
|
||||
});
|
||||
|
||||
app.Run();
|
||||
Reference in new issue
Block a user