using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Gott.Models; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace Gott.Services; /// /// Echtzeit-Scanner: Pollt GitHub Public Events API alle 75 Sekunden, /// filtert PushEvents und scannt Commit-Diffs nach geleakten API-Keys. /// public class GitHubEventScannerService : BackgroundService { private readonly DatabaseService _db; private readonly IHttpClientFactory _httpFactory; private readonly ILogger _logger; private readonly string _githubToken; private readonly bool _enabled; private const int PollIntervalMs = 30_000; // 30 Sekunden (häufiger pollen, weniger verpassen) private const int MaxCommitsPerPoll = 30; // Max Commit-Diffs pro Poll-Zyklus private const int MaxDiffSizeBytes = 100_000; // Diffs > 100KB überspringen private const int MaxProcessedIds = 10_000; // Ringbuffer für Event-Dedup // Zentrale Key-Patterns aus KeyPatterns.cs private static readonly Regex[] CompiledPatterns = KeyPatterns.Compiled; // Deduplication Ringbuffer private readonly HashSet _processedEventIds = new(); private readonly Queue _eventIdQueue = new(); // Live-Log Ringbuffer private const int MaxLogEntries = 50; private readonly Queue _logEntries = new(); // Status-Tracking private readonly object _lock = new(); private bool _isRunning; private string _lastPoll; private int _eventsPerHour; private int _keysFoundTotal; private int _pollCount; private string _lastError; // Öffentliche Status-Properties public bool Enabled => _enabled; public bool IsRunning { get { lock (_lock) return _isRunning; } } public string LastPoll { get { lock (_lock) return _lastPoll; } } public int EventsPerHour { get { lock (_lock) return _eventsPerHour; } } public int KeysFoundTotal { get { lock (_lock) return _keysFoundTotal; } } public int PollCount { get { lock (_lock) return _pollCount; } } public string LastError { get { lock (_lock) return _lastError; } } public List GetLog() { lock (_lock) return _logEntries.ToList(); } private void AddLog(string type, string message, string details = null) { lock (_lock) { _logEntries.Enqueue(new EventLogEntry { Time = DateTime.UtcNow.ToString("o"), Type = type, Message = message }); while (_logEntries.Count > MaxLogEntries) _logEntries.Dequeue(); } // Persistieren in DB (fire-and-forget) _ = Task.Run(async () => { try { await _db.CreateScannerLog(type, message, "event", details); } catch { /* DB-Fehler ignorieren, Scanner nicht blockieren */ } }); } public GitHubEventScannerService( DatabaseService db, IHttpClientFactory httpFactory, IConfiguration config, ILogger logger) { _db = db; _httpFactory = httpFactory; _logger = logger; _githubToken = config["GitHubToken"] ?? ""; _enabled = config.GetValue("GitHubEventScanner", false); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { if (!_enabled) { _logger.LogInformation("GitHubEventScannerService deaktiviert (GitHubEventScanner=false)"); return; } if (string.IsNullOrWhiteSpace(_githubToken)) { _logger.LogWarning("GitHubEventScannerService deaktiviert (kein GitHubToken konfiguriert)"); return; } _logger.LogInformation("GitHubEventScannerService gestartet (Poll-Intervall: {Interval}s)", PollIntervalMs / 1000); // Kurze Verzögerung nach App-Start await Task.Delay(5_000, stoppingToken); var eventCounts = new Queue<(DateTime Time, int Count)>(); while (!stoppingToken.IsCancellationRequested) { try { lock (_lock) _isRunning = true; var eventsProcessed = await PollAndScan(stoppingToken); // Events/Stunde berechnen eventCounts.Enqueue((DateTime.UtcNow, eventsProcessed)); while (eventCounts.Count > 0 && eventCounts.Peek().Time < DateTime.UtcNow.AddHours(-1)) eventCounts.Dequeue(); lock (_lock) { _lastPoll = DateTime.UtcNow.ToString("o"); _eventsPerHour = eventCounts.Sum(e => e.Count); _pollCount++; _lastError = null; } } catch (OperationCanceledException) { break; } catch (Exception ex) { _logger.LogError(ex, "GitHubEventScannerService: Fehler beim Poll"); AddLog("error", ex.Message); lock (_lock) _lastError = ex.Message; } finally { lock (_lock) _isRunning = false; } await Task.Delay(PollIntervalMs, stoppingToken); } } /// /// Ein Poll-Zyklus: Events abrufen → PushEvents filtern → Commit-Diffs scannen → Keys speichern. /// Gibt die Anzahl verarbeiteter Events zurück. /// private async Task PollAndScan(CancellationToken ct) { var client = CreateGitHubClient(); var events = await FetchPublicEvents(client, ct); if (events.Count == 0) { AddLog("poll", "0 PushEvents empfangen"); return 0; } // Nur PushEvents, die wir noch nicht verarbeitet haben var pushEvents = new List(); foreach (var ev in events) { if (ev.Type != "PushEvent") continue; if (!TryTrackEvent(ev.Id)) continue; // Bereits verarbeitet pushEvents.Add(ev); } if (pushEvents.Count == 0) { AddLog("poll", events.Count + " Events, 0 neue PushEvents"); return events.Count; } // Max Diffs pro Poll begrenzen var toScan = pushEvents.Count > MaxCommitsPerPoll ? pushEvents.Take(MaxCommitsPerPoll).ToList() : pushEvents; AddLog("poll", events.Count + " Events, " + pushEvents.Count + " neue PushEvents → " + toScan.Count + " scannen"); // Head-Commit-Diffs scannen var newKeysFound = 0; var scannedCount = 0; foreach (var pe in toScan) { ct.ThrowIfCancellationRequested(); try { var keys = await ScanCommitDiff(client, pe.Repo, pe.HeadSha, ct); scannedCount++; foreach (var (key, sourceUrl, fileName) in keys) { if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key)) { AddLog("skip", "Key bereits bekannt: " + key[..Math.Min(20, key.Length)] + "... in " + pe.Repo); continue; } var keyType = DetectKeyType(key); var keyPrefix = key[..Math.Min(20, key.Length)] + "..."; var typeName = KeyPatterns.DisplayName(keyType); newKeysFound++; await _db.CreateScannedKey(new ScannedKeyRecord { ScannedKeyId = Guid.NewGuid().ToString("N"), ApiKey = key, Source = "github_events", SourceUrl = sourceUrl, Type = keyType, Status = "unchecked", Plan = "Unknown", ScannedAt = DateTime.UtcNow.ToString("o") }); var logMsg = $"🔑 Neuer {typeName} Key! Repo: {pe.Repo} · Datei: {fileName} · Commit: {pe.HeadSha[..8]} · Prefix: {keyPrefix}"; _logger.LogInformation(logMsg); // Detailliertes JSON für strukturierte Auswertung var detailsJson = JsonSerializer.Serialize(new { key_type = keyType, key_type_name = typeName, key_prefix = keyPrefix, key_length = key.Length, repo = pe.Repo, commit_sha = pe.HeadSha, file = fileName, source_url = sourceUrl, push_size = pe.Size }); AddLog("key_found", logMsg, detailsJson); } } catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden) { _logger.LogWarning("GitHub Rate-Limit bei Commit-Diff, überspringe restliche Commits"); AddLog("rate_limit", "Rate-Limit bei Commit-Diff, restliche Commits übersprungen"); break; } catch (Exception ex) { _logger.LogDebug("Commit-Diff-Scan fehlgeschlagen ({Repo}/{Sha}): {Msg}", pe.Repo, pe.HeadSha[..Math.Min(8, pe.HeadSha.Length)], ex.Message); } } AddLog("poll", scannedCount + " Diffs gescannt, " + newKeysFound + " neue Keys"); if (newKeysFound > 0) { lock (_lock) _keysFoundTotal += newKeysFound; _logger.LogInformation("GitHubEventScanner: {Count} neue(r) Key(s) in diesem Poll!", newKeysFound); // Neue Keys sofort prüfen await CheckNewKeys(ct); } return events.Count; } /// Ruft die öffentlichen GitHub Events ab (bis zu 3 Seiten à 100). private async Task> FetchPublicEvents(HttpClient client, CancellationToken ct) { var allEvents = new List(); for (int page = 1; page <= 3; page++) { ct.ThrowIfCancellationRequested(); var url = $"https://api.github.com/events?per_page=100&page={page}"; var resp = await client.GetAsync(url, ct); if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden || resp.StatusCode == System.Net.HttpStatusCode.TooManyRequests) { _logger.LogWarning("GitHub Rate-Limit bei Events-Abruf (Seite {Page})", page); AddLog("rate_limit", "Rate-Limit bei Events-Abruf (Seite " + page + ")"); break; } if (!resp.IsSuccessStatusCode) { AddLog("error", "Events-API HTTP " + (int)resp.StatusCode + " (Seite " + page + ")"); break; } var json = await resp.Content.ReadAsStringAsync(ct); var doc = JsonDocument.Parse(json); var totalInPage = doc.RootElement.GetArrayLength(); if (page == 1 && totalInPage == 0) { AddLog("error", "API gab 0 Events zurück – Token ggf. kein Classic PAT?"); break; } foreach (var elem in doc.RootElement.EnumerateArray()) { var type = elem.TryGetProperty("type", out var t) ? t.GetString() : ""; var id = elem.TryGetProperty("id", out var i) ? i.GetString() : ""; var repo = elem.TryGetProperty("repo", out var r) && r.TryGetProperty("name", out var n) ? n.GetString() : ""; if (type != "PushEvent" || string.IsNullOrEmpty(repo)) continue; // Head-SHA aus payload.head – die Events-API liefert keine commits[], aber immer head var headSha = ""; var size = 0; if (elem.TryGetProperty("payload", out var payload)) { headSha = payload.TryGetProperty("head", out var h) ? h.GetString() : ""; size = payload.TryGetProperty("size", out var sz) ? sz.GetInt32() : 0; } if (string.IsNullOrEmpty(headSha)) continue; allEvents.Add(new PushEventData { Id = id, Type = type, Repo = repo, HeadSha = headSha, Size = size }); } // Wenn weniger als 100 Events, gibt es keine weiteren Seiten if (doc.RootElement.GetArrayLength() < 100) break; await Task.Delay(500, ct); // Kurze Pause zwischen Seiten } return allEvents; } /// Scannt den Diff eines einzelnen Commits nach Key-Patterns. Gibt Key, URL und Dateiname zurück. private async Task> ScanCommitDiff(HttpClient client, string repo, string sha, CancellationToken ct) { var keys = new List<(string Key, string Url, string FileName)>(); var detailUrl = $"https://api.github.com/repos/{repo}/commits/{sha}"; var request = new HttpRequestMessage(HttpMethod.Get, detailUrl); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); var resp = await client.SendAsync(request, ct); if (!resp.IsSuccessStatusCode) return keys; var json = await resp.Content.ReadAsStringAsync(ct); // Größe prüfen if (json.Length > MaxDiffSizeBytes) return keys; var doc = JsonDocument.Parse(json); if (!doc.RootElement.TryGetProperty("files", out var files)) return keys; var htmlUrl = $"https://github.com/{repo}/commit/{sha}"; foreach (var file in files.EnumerateArray()) { // Patch kann null sein bei Binärdateien oder zu großen Diffs if (!file.TryGetProperty("patch", out var patchProp)) continue; var patch = patchProp.GetString(); if (string.IsNullOrEmpty(patch)) continue; var fileName = file.TryGetProperty("filename", out var fn) ? fn.GetString() ?? "unknown" : "unknown"; // Nur hinzugefügte Zeilen scannen (beginnen mit "+") foreach (var line in patch.Split('\n')) { if (!line.StartsWith("+")) continue; foreach (var regex in CompiledPatterns) { foreach (Match m in regex.Matches(line)) keys.Add((m.Value, htmlUrl, fileName)); } } } return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList(); } /// Prüft alle unchecked Keys sofort nach Fund. private async Task CheckNewKeys(CancellationToken ct) { try { var allScanned = (await _db.GetScannedKeys()).ToList(); var unchecked_ = allScanned.Where(k => k.Status == "unchecked").ToList(); foreach (var sk in unchecked_) { ct.ThrowIfCancellationRequested(); var type = KeyPatterns.DetectType(sk.ApiKey); var keyPrefix = sk.ApiKey[..Math.Min(20, sk.ApiKey.Length)] + "..."; var typeName = KeyPatterns.DisplayName(type); try { if (KeyPatterns.IsAnthropic(type)) { var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey); var status = (result.Status == "ok" || result.Status == "usage_limited") ? "valid" : result.Status; await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Plan, result.Usage7d, result.Usage5h); var checkMsg = $"🔍 {typeName} geprüft: {status} · Plan: {result.Plan} · {keyPrefix}"; AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, plan = result.Plan, usage_7d = result.Usage7d, usage_5h = result.Usage5h, source_url = sk.SourceUrl })); } else if (type == "openai_project" || type == "openai_service" || type == "openai_legacy") { var result = await OpenAiKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey); var status = result.Status == "ok" ? "valid" : result.Status; await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Message, null, null); var checkMsg = $"🔍 {typeName} geprüft: {status} · {result.Message} · {keyPrefix}"; AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, message = result.Message, source_url = sk.SourceUrl })); } else if (type == "google_api") { var result = await GoogleKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey); var status = result.Status == "ok" ? "valid" : result.Status; await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Message, null, null); var checkMsg = $"🔍 {typeName} geprüft: {status} · {result.Message} · {keyPrefix}"; AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, message = result.Message, source_url = sk.SourceUrl })); } else { AddLog("key_checked", $"⏭️ {typeName} übersprungen (kein Checker) · {keyPrefix}"); } } catch (Exception ex) { _logger.LogWarning("Key-Check fehlgeschlagen: {Msg}", ex.Message); await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null); } await Task.Delay(500, ct); } } catch (Exception ex) { _logger.LogWarning("CheckNewKeys fehlgeschlagen: {Msg}", ex.Message); } } /// Trackt eine Event-ID. Gibt true zurück wenn neu, false wenn bereits verarbeitet. private bool TryTrackEvent(string eventId) { if (string.IsNullOrEmpty(eventId)) return false; lock (_lock) { if (_processedEventIds.Contains(eventId)) return false; _processedEventIds.Add(eventId); _eventIdQueue.Enqueue(eventId); // Ringbuffer: älteste IDs entfernen while (_eventIdQueue.Count > MaxProcessedIds) { var old = _eventIdQueue.Dequeue(); _processedEventIds.Remove(old); } } return true; } private HttpClient CreateGitHubClient() { var client = _httpFactory.CreateClient(); client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottEventScanner", "1.0")); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); if (!string.IsNullOrWhiteSpace(_githubToken)) client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken); return client; } private static string DetectKeyType(string key) => KeyPatterns.DetectType(key); // ── Interne Datenstrukturen ── private class PushEventData { public string Id { get; set; } public string Type { get; set; } public string Repo { get; set; } public string HeadSha { get; set; } // payload.head – Commit-SHA des HEAD public int Size { get; set; } // payload.size – Anzahl Commits im Push } public class EventLogEntry { public string Time { get; set; } public string Type { get; set; } // poll, key, rate_limit, error public string Message { get; set; } } }