using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Gott.Models;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace Gott.Services;
///
/// Echtzeit-Scanner: Pollt GitHub Public Events API alle 75 Sekunden,
/// filtert PushEvents und scannt Commit-Diffs nach geleakten API-Keys.
///
public class GitHubEventScannerService : BackgroundService
{
private readonly DatabaseService _db;
private readonly IHttpClientFactory _httpFactory;
private readonly ILogger _logger;
private readonly ServerSettingsService _settings;
private string _githubToken => _settings.Get("GitHubToken");
private bool _enabled => _settings.GetBool("GitHubEventScanner", false);
private const int PollIntervalMs = 30_000; // 30 Sekunden (häufiger pollen, weniger verpassen)
private const int MaxCommitsPerPoll = 30; // Max Commit-Diffs pro Poll-Zyklus
private const int MaxDiffSizeBytes = 100_000; // Diffs > 100KB überspringen
private const int MaxProcessedIds = 10_000; // Ringbuffer für Event-Dedup
// Zentrale Key-Patterns aus KeyPatterns.cs
private static readonly Regex[] CompiledPatterns = KeyPatterns.Compiled;
// Deduplication Ringbuffer
private readonly HashSet _processedEventIds = new();
private readonly Queue _eventIdQueue = new();
// Live-Log Ringbuffer
private const int MaxLogEntries = 50;
private readonly Queue _logEntries = new();
// Status-Tracking
private readonly object _lock = new();
private bool _isRunning;
private string _lastPoll;
private int _eventsPerHour;
private int _keysFoundTotal;
private int _pollCount;
private string _lastError;
// Öffentliche Status-Properties
public bool Enabled => _enabled;
public bool IsRunning { get { lock (_lock) return _isRunning; } }
public string LastPoll { get { lock (_lock) return _lastPoll; } }
public int EventsPerHour { get { lock (_lock) return _eventsPerHour; } }
public int KeysFoundTotal { get { lock (_lock) return _keysFoundTotal; } }
public int PollCount { get { lock (_lock) return _pollCount; } }
public string LastError { get { lock (_lock) return _lastError; } }
public List GetLog()
{
lock (_lock) return _logEntries.ToList();
}
private void AddLog(string type, string message, string details = null, bool persist = true)
{
lock (_lock)
{
_logEntries.Enqueue(new EventLogEntry
{
Time = DateTime.UtcNow.ToString("o"),
Type = type,
Message = message
});
while (_logEntries.Count > MaxLogEntries)
_logEntries.Dequeue();
}
if (!persist) return;
// Persistieren in DB (fire-and-forget)
_ = Task.Run(async () =>
{
try { await _db.CreateScannerLog(type, message, "event", details); }
catch { /* DB-Fehler ignorieren, Scanner nicht blockieren */ }
});
}
public GitHubEventScannerService(
DatabaseService db,
IHttpClientFactory httpFactory,
ServerSettingsService settings,
ILogger logger)
{
_db = db;
_httpFactory = httpFactory;
_settings = settings;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("GitHubEventScannerService bereit (Enabled wird dynamisch aus Settings gelesen)");
// Kurze Verzögerung nach App-Start
await Task.Delay(5_000, stoppingToken);
var eventCounts = new Queue<(DateTime Time, int Count)>();
while (!stoppingToken.IsCancellationRequested)
{
// Dynamisch prüfen ob aktiviert
if (!_enabled || string.IsNullOrWhiteSpace(_githubToken))
{
lock (_lock) _isRunning = false;
await Task.Delay(10_000, stoppingToken);
continue;
}
try
{
lock (_lock) _isRunning = true;
var eventsProcessed = await PollAndScan(stoppingToken);
// Events/Stunde berechnen
eventCounts.Enqueue((DateTime.UtcNow, eventsProcessed));
while (eventCounts.Count > 0 && eventCounts.Peek().Time < DateTime.UtcNow.AddHours(-1))
eventCounts.Dequeue();
lock (_lock)
{
_lastPoll = DateTime.UtcNow.ToString("o");
_eventsPerHour = eventCounts.Sum(e => e.Count);
_pollCount++;
_lastError = null;
}
}
catch (OperationCanceledException) { break; }
catch (Exception ex)
{
_logger.LogError(ex, "GitHubEventScannerService: Fehler beim Poll");
AddLog("error", ex.Message);
lock (_lock) _lastError = ex.Message;
}
finally
{
lock (_lock) _isRunning = false;
}
await Task.Delay(PollIntervalMs, stoppingToken);
}
}
///
/// Ein Poll-Zyklus: Events abrufen → PushEvents filtern → Commit-Diffs scannen → Keys speichern.
/// Gibt die Anzahl verarbeiteter Events zurück.
///
private async Task PollAndScan(CancellationToken ct)
{
var client = CreateGitHubClient();
var events = await FetchPublicEvents(client, ct);
if (events.Count == 0)
{
AddLog("poll", "0 PushEvents empfangen", persist: false);
return 0;
}
// Nur PushEvents, die wir noch nicht verarbeitet haben
var pushEvents = new List();
foreach (var ev in events)
{
if (ev.Type != "PushEvent") continue;
if (!TryTrackEvent(ev.Id)) continue; // Bereits verarbeitet
pushEvents.Add(ev);
}
if (pushEvents.Count == 0)
{
AddLog("poll", events.Count + " Events, 0 neue PushEvents", persist: false);
return events.Count;
}
// Max Diffs pro Poll begrenzen
var toScan = pushEvents.Count > MaxCommitsPerPoll
? pushEvents.Take(MaxCommitsPerPoll).ToList()
: pushEvents;
AddLog("poll", events.Count + " Events, " + pushEvents.Count + " neue PushEvents → " + toScan.Count + " scannen", persist: false);
// Head-Commit-Diffs scannen
var newKeyIds = new List();
var scannedCount = 0;
foreach (var pe in toScan)
{
ct.ThrowIfCancellationRequested();
try
{
var keys = await ScanCommitDiff(client, pe.Repo, pe.HeadSha, ct);
scannedCount++;
foreach (var (key, sourceUrl, fileName) in keys)
{
if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key))
{
AddLog("skip", "Key bereits bekannt: " + key[..Math.Min(20, key.Length)] + "... in " + pe.Repo);
continue;
}
var keyType = DetectKeyType(key);
var keyPrefix = key[..Math.Min(20, key.Length)] + "...";
var typeName = KeyPatterns.DisplayName(keyType);
var keyId = Guid.NewGuid().ToString("N");
newKeyIds.Add(keyId);
await _db.CreateScannedKey(new ScannedKeyRecord
{
ScannedKeyId = keyId,
ApiKey = key,
Source = "github_events",
SourceUrl = sourceUrl,
Type = keyType,
Status = "unchecked",
Plan = "Unknown",
ScannedAt = DateTime.UtcNow.ToString("o")
});
var logMsg = $"🔑 Neuer {typeName} Key! Repo: {pe.Repo} · Datei: {fileName} · Commit: {pe.HeadSha[..8]} · Prefix: {keyPrefix}";
_logger.LogInformation(logMsg);
// Detailliertes JSON für strukturierte Auswertung
var detailsJson = JsonSerializer.Serialize(new
{
key_type = keyType,
key_type_name = typeName,
key_prefix = keyPrefix,
key_length = key.Length,
repo = pe.Repo,
commit_sha = pe.HeadSha,
file = fileName,
source_url = sourceUrl,
push_size = pe.Size
});
AddLog("key_found", logMsg, detailsJson);
}
}
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
{
_logger.LogWarning("GitHub Rate-Limit bei Commit-Diff, überspringe restliche Commits");
AddLog("rate_limit", "Rate-Limit bei Commit-Diff, restliche Commits übersprungen");
break;
}
catch (Exception ex)
{
_logger.LogDebug("Commit-Diff-Scan fehlgeschlagen ({Repo}/{Sha}): {Msg}",
pe.Repo, pe.HeadSha[..Math.Min(8, pe.HeadSha.Length)], ex.Message);
}
}
AddLog("poll", scannedCount + " Diffs gescannt, " + newKeyIds.Count + " neue Keys", persist: newKeyIds.Count > 0);
if (newKeyIds.Count > 0)
{
lock (_lock) _keysFoundTotal += newKeyIds.Count;
_logger.LogInformation("GitHubEventScanner: {Count} neue(r) Key(s) in diesem Poll!", newKeyIds.Count);
// Nur die gerade gefundenen Keys prüfen (nicht alle unchecked!)
await CheckNewKeys(newKeyIds, ct);
}
return events.Count;
}
/// Ruft die öffentlichen GitHub Events ab (bis zu 3 Seiten à 100).
private async Task> FetchPublicEvents(HttpClient client, CancellationToken ct)
{
var allEvents = new List();
for (int page = 1; page <= 3; page++)
{
ct.ThrowIfCancellationRequested();
var url = $"https://api.github.com/events?per_page=100&page={page}";
var resp = await client.GetAsync(url, ct);
if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden ||
resp.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
{
_logger.LogWarning("GitHub Rate-Limit bei Events-Abruf (Seite {Page})", page);
AddLog("rate_limit", "Rate-Limit bei Events-Abruf (Seite " + page + ")");
break;
}
if (!resp.IsSuccessStatusCode)
{
AddLog("error", "Events-API HTTP " + (int)resp.StatusCode + " (Seite " + page + ")");
break;
}
var json = await resp.Content.ReadAsStringAsync(ct);
var doc = JsonDocument.Parse(json);
var totalInPage = doc.RootElement.GetArrayLength();
if (page == 1 && totalInPage == 0)
{
AddLog("error", "API gab 0 Events zurück – Token ggf. kein Classic PAT?");
break;
}
foreach (var elem in doc.RootElement.EnumerateArray())
{
var type = elem.TryGetProperty("type", out var t) ? t.GetString() : "";
var id = elem.TryGetProperty("id", out var i) ? i.GetString() : "";
var repo = elem.TryGetProperty("repo", out var r) && r.TryGetProperty("name", out var n) ? n.GetString() : "";
if (type != "PushEvent" || string.IsNullOrEmpty(repo)) continue;
// Head-SHA aus payload.head – die Events-API liefert keine commits[], aber immer head
var headSha = "";
var size = 0;
if (elem.TryGetProperty("payload", out var payload))
{
headSha = payload.TryGetProperty("head", out var h) ? h.GetString() : "";
size = payload.TryGetProperty("size", out var sz) ? sz.GetInt32() : 0;
}
if (string.IsNullOrEmpty(headSha)) continue;
allEvents.Add(new PushEventData
{
Id = id,
Type = type,
Repo = repo,
HeadSha = headSha,
Size = size
});
}
// Wenn weniger als 100 Events, gibt es keine weiteren Seiten
if (doc.RootElement.GetArrayLength() < 100) break;
await Task.Delay(500, ct); // Kurze Pause zwischen Seiten
}
return allEvents;
}
/// Scannt den Diff eines einzelnen Commits nach Key-Patterns. Gibt Key, URL und Dateiname zurück.
private async Task> ScanCommitDiff(HttpClient client, string repo, string sha, CancellationToken ct)
{
var keys = new List<(string Key, string Url, string FileName)>();
var detailUrl = $"https://api.github.com/repos/{repo}/commits/{sha}";
var request = new HttpRequestMessage(HttpMethod.Get, detailUrl);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
var resp = await client.SendAsync(request, ct);
if (!resp.IsSuccessStatusCode) return keys;
var json = await resp.Content.ReadAsStringAsync(ct);
// Größe prüfen
if (json.Length > MaxDiffSizeBytes) return keys;
var doc = JsonDocument.Parse(json);
if (!doc.RootElement.TryGetProperty("files", out var files)) return keys;
var htmlUrl = $"https://github.com/{repo}/commit/{sha}";
foreach (var file in files.EnumerateArray())
{
// Patch kann null sein bei Binärdateien oder zu großen Diffs
if (!file.TryGetProperty("patch", out var patchProp)) continue;
var patch = patchProp.GetString();
if (string.IsNullOrEmpty(patch)) continue;
var fileName = file.TryGetProperty("filename", out var fn) ? fn.GetString() ?? "unknown" : "unknown";
// Nur hinzugefügte Zeilen scannen (beginnen mit "+")
foreach (var line in patch.Split('\n'))
{
if (!line.StartsWith("+")) continue;
foreach (var regex in CompiledPatterns)
{
foreach (Match m in regex.Matches(line))
keys.Add((m.Value, htmlUrl, fileName));
}
}
}
return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList();
}
/// Prüft nur die gerade im aktuellen Poll gefundenen Keys.
private async Task CheckNewKeys(List keyIds, CancellationToken ct)
{
try
{
var allScanned = (await _db.GetScannedKeys()).ToList();
var toCheck = allScanned.Where(k => keyIds.Contains(k.ScannedKeyId)).ToList();
foreach (var sk in toCheck)
{
ct.ThrowIfCancellationRequested();
var type = KeyPatterns.DetectType(sk.ApiKey);
var keyPrefix = sk.ApiKey[..Math.Min(20, sk.ApiKey.Length)] + "...";
var typeName = KeyPatterns.DisplayName(type);
try
{
if (KeyPatterns.IsAnthropic(type))
{
var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
var status = (result.Status == "ok" || result.Status == "usage_limited") ? "valid" : result.Status;
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Plan, result.Usage7d, result.Usage5h);
var checkMsg = $"🔍 {typeName} geprüft: {status} · Plan: {result.Plan} · {keyPrefix}";
AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, plan = result.Plan, usage_7d = result.Usage7d, usage_5h = result.Usage5h, source_url = sk.SourceUrl }));
}
else if (type == "openai_project" || type == "openai_service" || type == "openai_legacy")
{
var result = await OpenAiKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
var status = result.Status == "ok" ? "valid" : result.Status;
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Message, null, null);
var checkMsg = $"🔍 {typeName} geprüft: {status} · {result.Message} · {keyPrefix}";
AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, message = result.Message, source_url = sk.SourceUrl }));
}
else if (type == "google_api")
{
var result = await GoogleKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
var status = result.Status == "ok" ? "valid" : result.Status;
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Message, null, null);
var checkMsg = $"🔍 {typeName} geprüft: {status} · {result.Message} · {keyPrefix}";
AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, message = result.Message, source_url = sk.SourceUrl }));
}
else
{
AddLog("key_checked", $"💾 {typeName} gespeichert (kein Checker verfügbar, bleibt unchecked) · {keyPrefix}");
}
}
catch (Exception ex)
{
_logger.LogWarning("Key-Check fehlgeschlagen: {Msg}", ex.Message);
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null);
}
await Task.Delay(500, ct);
}
}
catch (Exception ex)
{
_logger.LogWarning("CheckNewKeys fehlgeschlagen: {Msg}", ex.Message);
}
}
/// Trackt eine Event-ID. Gibt true zurück wenn neu, false wenn bereits verarbeitet.
private bool TryTrackEvent(string eventId)
{
if (string.IsNullOrEmpty(eventId)) return false;
lock (_lock)
{
if (_processedEventIds.Contains(eventId)) return false;
_processedEventIds.Add(eventId);
_eventIdQueue.Enqueue(eventId);
// Ringbuffer: älteste IDs entfernen
while (_eventIdQueue.Count > MaxProcessedIds)
{
var old = _eventIdQueue.Dequeue();
_processedEventIds.Remove(old);
}
}
return true;
}
private HttpClient CreateGitHubClient()
{
var client = _httpFactory.CreateClient();
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottEventScanner", "1.0"));
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
if (!string.IsNullOrWhiteSpace(_githubToken))
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
return client;
}
private static string DetectKeyType(string key) => KeyPatterns.DetectType(key);
// ── Interne Datenstrukturen ──
private class PushEventData
{
public string Id { get; set; }
public string Type { get; set; }
public string Repo { get; set; }
public string HeadSha { get; set; } // payload.head – Commit-SHA des HEAD
public int Size { get; set; } // payload.size – Anzahl Commits im Push
}
public class EventLogEntry
{
public string Time { get; set; }
public string Type { get; set; } // poll, key, rate_limit, error
public string Message { get; set; }
}
}