docs update, fix JSONL import (full-scan instead of line-offset tracking), sync button UX
This commit is contained in:
1 parent
9aa8b8a1c6
commit
90139430f7
6 files changed
+81
-23
No files matched your search
+5
-14
@@ -24,18 +24,12 @@ db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_type ON entries (type);
|
||||
CREATE INDEX IF NOT EXISTS idx_timestamp ON entries (timestamp);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
file TEXT PRIMARY KEY,
|
||||
lines INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
`);
|
||||
|
||||
// Add chainId column if missing (migration)
|
||||
try { db.exec('ALTER TABLE entries ADD COLUMN chainId TEXT'); } catch {}
|
||||
try { db.exec('CREATE INDEX IF NOT EXISTS idx_chainId ON entries (chainId)'); } catch {}
|
||||
|
||||
const getState = db.prepare('SELECT lines FROM state WHERE file = ?');
|
||||
const setState = db.prepare('INSERT OR REPLACE INTO state (file, lines) VALUES (?, ?)');
|
||||
const insertEntry = db.prepare(`
|
||||
INSERT OR IGNORE INTO entries (id, session, type, timestamp, data)
|
||||
VALUES (@id, @session, @type, @timestamp, @data)
|
||||
@@ -48,7 +42,7 @@ const getUnchained = db.prepare(`
|
||||
ORDER BY timestamp ASC
|
||||
`);
|
||||
|
||||
// --- Step 1: Import new entries ---
|
||||
// --- Step 1: Import entries (full scan, INSERT OR IGNORE deduplicates) ---
|
||||
const jsonlFiles = fs.readdirSync(SESSIONS_DIR)
|
||||
.filter(f => f.endsWith('.jsonl') && !f.includes('.deleted'));
|
||||
|
||||
@@ -59,25 +53,22 @@ const runImport = db.transaction(() => {
|
||||
const filePath = path.join(SESSIONS_DIR, file);
|
||||
const sessionName = file.replace('.jsonl', '');
|
||||
const lines = fs.readFileSync(filePath, 'utf-8').split('\n').filter(l => l.trim());
|
||||
const lastProcessed = getState.get(file)?.lines ?? 0;
|
||||
|
||||
for (let i = lastProcessed; i < lines.length; i++) {
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
try {
|
||||
const entry = JSON.parse(lines[i]);
|
||||
if (!entry.timestamp) continue;
|
||||
const id = entry.id || `${sessionName}-${i}`;
|
||||
insertEntry.run({
|
||||
const res = insertEntry.run({
|
||||
id,
|
||||
session: sessionName,
|
||||
type: entry.type ?? 'unknown',
|
||||
timestamp: entry.timestamp,
|
||||
data: JSON.stringify(entry),
|
||||
});
|
||||
exported++;
|
||||
if (res.changes > 0) exported++;
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
|
||||
setState.run(file, lines.length);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -138,4 +129,4 @@ const runChaining = db.transaction(() => {
|
||||
runChaining();
|
||||
db.close();
|
||||
|
||||
console.log(`Exportiert: ${exported} neue, ${chained} verkettet → ${DB_FILE}`);
|
||||
console.log(`${exported} neue, ${chained} verkettet`);
|
||||
Reference in new issue
Block a user