docs update, fix JSONL import (full-scan instead of line-offset tracking), sync button UX

This commit is contained in:
Meleki committed 2026-02-23 21:54:19 +01:00
1 parent 9aa8b8a1c6
commit 90139430f7
6 files changed
+81 -23

No files matched your search

+5 -14
View File
@@ -24,18 +24,12 @@ db.exec(`
CREATE INDEX IF NOT EXISTS idx_type ON entries (type);
CREATE INDEX IF NOT EXISTS idx_timestamp ON entries (timestamp);
CREATE TABLE IF NOT EXISTS state (
file TEXT PRIMARY KEY,
lines INTEGER NOT NULL DEFAULT 0
);
`);
// Add chainId column if missing (migration)
try { db.exec('ALTER TABLE entries ADD COLUMN chainId TEXT'); } catch {}
try { db.exec('CREATE INDEX IF NOT EXISTS idx_chainId ON entries (chainId)'); } catch {}
const getState = db.prepare('SELECT lines FROM state WHERE file = ?');
const setState = db.prepare('INSERT OR REPLACE INTO state (file, lines) VALUES (?, ?)');
const insertEntry = db.prepare(`
INSERT OR IGNORE INTO entries (id, session, type, timestamp, data)
VALUES (@id, @session, @type, @timestamp, @data)
@@ -48,7 +42,7 @@ const getUnchained = db.prepare(`
ORDER BY timestamp ASC
`);
// --- Step 1: Import new entries ---
// --- Step 1: Import entries (full scan, INSERT OR IGNORE deduplicates) ---
const jsonlFiles = fs.readdirSync(SESSIONS_DIR)
.filter(f => f.endsWith('.jsonl') && !f.includes('.deleted'));
@@ -59,25 +53,22 @@ const runImport = db.transaction(() => {
const filePath = path.join(SESSIONS_DIR, file);
const sessionName = file.replace('.jsonl', '');
const lines = fs.readFileSync(filePath, 'utf-8').split('\n').filter(l => l.trim());
const lastProcessed = getState.get(file)?.lines ?? 0;
for (let i = lastProcessed; i < lines.length; i++) {
for (let i = 0; i < lines.length; i++) {
try {
const entry = JSON.parse(lines[i]);
if (!entry.timestamp) continue;
const id = entry.id || `${sessionName}-${i}`;
insertEntry.run({
const res = insertEntry.run({
id,
session: sessionName,
type: entry.type ?? 'unknown',
timestamp: entry.timestamp,
data: JSON.stringify(entry),
});
exported++;
if (res.changes > 0) exported++;
} catch { /* skip */ }
}
setState.run(file, lines.length);
}
});
@@ -138,4 +129,4 @@ const runChaining = db.transaction(() => {
runChaining();
db.close();
console.log(`Exportiert: ${exported} neue, ${chained} verkettet → ${DB_FILE}`);
console.log(`${exported} neue, ${chained} verkettet`);