gott/Services/KeyScannerService.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Gott.Models;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;

namespace Gott.Services;

public class KeyScannerService : BackgroundService
{
    private readonly DatabaseService _db;
    private readonly IHttpClientFactory _httpFactory;
    private readonly ILogger<KeyScannerService> _logger;
    private readonly ServerSettingsService _settings;

    private string _githubToken => _settings.Get("GitHubToken");
    private string _serperApiKey => _settings.Get("SerperApiKey");
    private int _dailyHour => _settings.GetInt("ScanDailyHour", -1);

    // Status-Tracking
    private readonly object _lock = new();
    private bool _isRunning;
    private string _lastRun;
    private string _lastResult;
    private readonly ManualResetEventSlim _triggerNow = new(false);
    private string _scanMode = "full"; // full, quick, web

    // Progress-Tracking
    private string _progressPhase = "";
    private string _progressDetail = "";
    private int _progressCurrent;
    private int _progressTotal;
    private int _progressKeysFound;

    public bool IsRunning { get { lock (_lock) return _isRunning; } }
    public string LastRun { get { lock (_lock) return _lastRun; } }
    public string LastResult { get { lock (_lock) return _lastResult; } }

    public (string Phase, string Detail, int Current, int Total, int KeysFound) Progress
    {
        get { lock (_lock) return (_progressPhase, _progressDetail, _progressCurrent, _progressTotal, _progressKeysFound); }
    }

    private void SetProgress(string phase, string detail = "", int current = 0, int total = 0)
    {
        lock (_lock)
        {
            _progressPhase = phase;
            _progressDetail = detail;
            _progressCurrent = current;
            _progressTotal = total;
        }
    }

    private void IncrementKeysFound()
    {
        lock (_lock) _progressKeysFound++;
    }

    private void AddLog(string type, string message, string details = null)
    {
        _logger.LogInformation("[CodeSearch] [{Type}] {Message}", type, message);
        _ = Task.Run(async () =>
        {
            try { await _db.CreateScannerLog(type, message, "codesearch", details); }
            catch { /* DB-Fehler ignorieren */ }
        });
    }

    // ── GitHub Search Config ──

    /// <summary>Gibt die aktiven Search-Patterns zurück, gefiltert nach ScannerKeyTypes-Setting.</summary>
    private (string Pattern, string SearchTerm, string Type)[] GetActiveSearchPatterns()
    {
        var allowed = new HashSet<string>(_settings.GetStringArray("ScannerKeyTypes"));
        if (allowed.Count == 0) return Array.Empty<(string, string, string)>();
        return KeyPatterns.ForCodeSearch.Where(p => allowed.Contains(p.Type)).ToArray();
    }

    /// <summary>Prüft ob ein Key-Typ in den aktiven Scanner-Typen enthalten ist.</summary>
    private bool IsKeyTypeAllowed(string type)
    {
        var allowed = new HashSet<string>(_settings.GetStringArray("ScannerKeyTypes"));
        return allowed.Contains(type);
    }

    private static readonly string[] SearchLanguages =
    {
        "Dotenv", "Text", "JavaScript", "Python", "TypeScript",
        "Dockerfile", "Markdown", "Jupyter Notebook", "Shell",
        "Java", "Go", "C++", "PHP"
    };

    private static readonly string[] PathExtensions =
    {
        ".env", ".envrc", ".prod", ".secret", ".private", ".key",
        ".xml", ".json", ".properties", ".sql", ".txt", ".log",
        ".yml", ".yaml", ".toml", ".ini", ".config", ".conf", ".cfg",
        ".tmp", ".backup", ".bak", ".enc",
    };

    private const int RateLimitDelayMs = 6_500;
    private const int MaxResultsPerSearch = 1000;
    private const int ResultsPerPage = 100;

    public KeyScannerService(
        DatabaseService db,
        IHttpClientFactory httpFactory,
        ServerSettingsService settings,
        ILogger<KeyScannerService> logger)
    {
        _db = db;
        _httpFactory = httpFactory;
        _settings = settings;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("KeyScannerService gestartet");

        while (!stoppingToken.IsCancellationRequested)
        {
            // Berechne Wartezeit bis zum nächsten Scan oder warte auf manuellen Trigger
            var waitTime = GetTimeUntilNextScan();
            if (waitTime != null)
                _logger.LogInformation("KeyScannerService: Nächster Full-Scan um {Hour}:00 (in {Minutes} Min)", _dailyHour, (int)waitTime.Value.TotalMinutes);

            var triggered = await WaitWithTrigger(waitTime ?? Timeout.InfiniteTimeSpan, stoppingToken);
            if (stoppingToken.IsCancellationRequested) break;

            if (triggered)
            {
                _logger.LogInformation("KeyScannerService: Manueller Scan angefordert");
            }
            else
            {
                // Automatischer täglicher Full-Scan
                lock (_lock) _scanMode = "full";
                _logger.LogInformation("KeyScannerService: Täglicher Full-Scan gestartet ({Hour}:00)", _dailyHour);
            }

            await RunScan(stoppingToken);
        }
    }

    /// <summary>Berechnet die Wartezeit bis zur konfigurierten Scan-Stunde. Null wenn deaktiviert (-1).</summary>
    private TimeSpan? GetTimeUntilNextScan()
    {
        var hour = _dailyHour;
        if (hour < 0 || hour > 23) return null;

        var now = DateTime.Now;
        var nextRun = new DateTime(now.Year, now.Month, now.Day, hour, 0, 0);
        if (nextRun <= now) nextRun = nextRun.AddDays(1);
        return nextRun - now;
    }

    /// <summary>Wartet auf Timeout oder Trigger-Signal. Gibt true zurück wenn getriggert.</summary>
    private Task<bool> WaitWithTrigger(TimeSpan timeout, CancellationToken ct)
    {
        return Task.Run(() =>
        {
            _triggerNow.Reset();
            var waitHandles = new[] { _triggerNow.WaitHandle, ct.WaitHandle };
            var idx = WaitHandle.WaitAny(waitHandles, timeout);
            return idx == 0; // true = triggered
        }, ct);
    }

    /// <summary>Wird vom Endpoint aufgerufen um einen sofortigen Scan auszulösen.</summary>
    public void TriggerScan(string mode = "full")
    {
        lock (_lock) _scanMode = mode;
        _triggerNow.Set();
    }

    private async Task RunScan(CancellationToken ct)
    {
        string mode;
        lock (_lock)
        {
            if (_isRunning) return;
            _isRunning = true;
            mode = _scanMode;
            _scanMode = "full"; // Reset für nächsten Lauf
        }

        var modeLabels = new Dictionary<string, string> { ["quick"] = "Quick", ["web"] = "Web", ["full"] = "Full" };
        var modeLabel = modeLabels.ContainsKey(mode) ? modeLabels[mode] : "Full";
        AddLog("scan", modeLabel + "-Scan gestartet");
        lock (_lock) _progressKeysFound = 0;
        SetProgress("Starte " + modeLabel + "-Scan…");
        var newKeysFound = 0;
        var totalFound = 0;

        try
        {
            // Phase 1: Quellen je nach Scan-Modus scannen
            var allFoundKeys = new List<(string Key, string Url, string Source)>();

            // Quick: nur GitHub mit reduzierten Sprachen/Pfaden
            // Full: GitHub + Serper
            // Web: nur Serper
            if (mode != "web" && !string.IsNullOrWhiteSpace(_githubToken))
            {
                SetProgress("GitHub Code Search", "Starte…");
                var ghKeys = mode == "quick" ? await ScanGitHubQuick(ct) : await ScanGitHub(ct);
                allFoundKeys.AddRange(ghKeys.Select(k => (k.Key, k.Url, "github")));
                AddLog("scan", "GitHub-Scan (" + modeLabel + "): " + ghKeys.Count + " Key(s) gefunden");
            }

            // GitHub Issues & PR Comments (benötigt GitHub Token)
            if (mode != "web" && !string.IsNullOrWhiteSpace(_githubToken))
            {
                SetProgress("GitHub Issues & Comments", "Starte…");
                var issueKeys = await ScanGitHubIssues(mode == "quick", ct);
                allFoundKeys.AddRange(issueKeys.Select(k => (k.Key, k.Url, "github_issues")));
                AddLog("scan", "GitHub Issues (" + modeLabel + "): " + issueKeys.Count + " Key(s) gefunden");
            }

            if (mode != "quick" && !string.IsNullOrWhiteSpace(_serperApiKey))
            {
                SetProgress("Web-Suche (Serper)", "Starte…");
                var spKeys = await ScanSerper(ct);
                allFoundKeys.AddRange(spKeys.Select(k => (k.Key, k.Url, "serper")));
                AddLog("scan", "Serper-Scan: " + spKeys.Count + " Key(s) gefunden");
            }

            if (mode != "quick")
            {
                SetProgress("HuggingFace Hub", "Starte…");
                var hfKeys = await ScanHuggingFace(ct);
                allFoundKeys.AddRange(hfKeys.Select(k => (k.Key, k.Url, "huggingface")));
                AddLog("scan", "HuggingFace-Scan: " + hfKeys.Count + " Key(s) gefunden");
            }

            totalFound = allFoundKeys.Count;

            // Phase 2: Neue Keys in DB speichern (dedupliziert)
            SetProgress("Speichere Keys", totalFound + " gefunden, prüfe Duplikate…", 0, totalFound);
            var seen = new HashSet<string>(StringComparer.Ordinal);
            var saveIdx = 0;
            foreach (var (key, sourceUrl, source) in allFoundKeys)
            {
                saveIdx++;
                SetProgress("Speichere Keys", totalFound + " gefunden, prüfe Duplikate…", saveIdx, totalFound);
                if (!seen.Add(key)) continue;

                var keyType = DetectKeyType(key);

                if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key))
                    continue;
                var keyPrefix = key[..Math.Min(20, key.Length)] + "...";
                var typeName = KeyPatterns.DisplayName(keyType);

                newKeysFound++;
                IncrementKeysFound();
                await _db.CreateScannedKey(new ScannedKeyRecord
                {
                    ScannedKeyId = Guid.NewGuid().ToString("N"),
                    ApiKey = key,
                    Source = source,
                    SourceUrl = sourceUrl,
                    Type = keyType,
                    Status = "unchecked",
                    Plan = "Unknown",
                    ScannedAt = DateTime.UtcNow.ToString("o")
                });

                var logMsg = $"🔑 Neuer {typeName} Key! Quelle: {source} · URL: {sourceUrl} · Prefix: {keyPrefix}";
                var detailsJson = JsonSerializer.Serialize(new
                {
                    key_type = keyType,
                    key_type_name = typeName,
                    key_prefix = keyPrefix,
                    key_length = key.Length,
                    source,
                    source_url = sourceUrl,
                    scan_mode = modeLabel
                });
                AddLog("key_found", logMsg, detailsJson);
            }
            AddLog("scan", newKeysFound + " neue Key(s) gespeichert");

            lock (_lock)
            {
                _lastRun = DateTime.UtcNow.ToString("o");
                _lastResult = $"{modeLabel}: {totalFound} gefunden, {newKeysFound} neu";
            }

            SetProgress("Abgeschlossen", _lastResult);
            AddLog("scan", "Scan abgeschlossen – " + _lastResult);
        }
        catch (OperationCanceledException) { /* shutdown */ }
        catch (Exception ex)
        {
            AddLog("error", "Scan-Fehler: " + ex.Message);
            lock (_lock)
            {
                _lastRun = DateTime.UtcNow.ToString("o");
                _lastResult = $"Fehler: {ex.Message}";
            }
        }
        finally
        {
            lock (_lock) _isRunning = false;
        }
    }

    // ── GitHub Code Search ──

    private static readonly string[] QuickLanguages = { "Dotenv", "Text", "Markdown" };
    private static readonly string[] QuickPathExtensions = { ".env" };

    /// <summary>Quick-Scan: nur die wichtigsten Sprachen/Pfade durchsuchen.</summary>
    private async Task<List<(string Key, string Url)>> ScanGitHubQuick(CancellationToken ct)
    {
        var allKeys = new Dictionary<string, string>(StringComparer.Ordinal);
        var queries = new List<(string Query, Regex Regex)>();
        var activePatterns = GetActiveSearchPatterns();

        foreach (var (pattern, searchTerm, _) in activePatterns)
        {
            var regex = new Regex(pattern, RegexOptions.Compiled);
            foreach (var lang in QuickLanguages)
            {
                var langFilter = lang.Contains(' ') ? $"language:\"{lang}\"" : $"language:{lang}";
                queries.Add(($"{searchTerm} {langFilter}", regex));
            }
            foreach (var pathExt in QuickPathExtensions)
                queries.Add(($"{searchTerm} path:{pathExt}", regex));
        }

        var client = _httpFactory.CreateClient();
        client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
        client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
        client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.text-match+json"));

        for (int i = 0; i < queries.Count; i++)
        {
            ct.ThrowIfCancellationRequested();
            var (query, regex) = queries[i];
            SetProgress("GitHub Code Search (Quick)", query, i + 1, queries.Count);

            try
            {
                var results = await SearchCodeAsync(client, query, regex, ct);
                foreach (var (key, url) in results)
                    if (!allKeys.ContainsKey(key)) allKeys[key] = url;
            }
            catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
            {
                SetProgress("GitHub Code Search (Quick)", "⏳ Rate-Limit, warte 60s…", i + 1, queries.Count);
                AddLog("rate_limit", "GitHub Rate-Limit (Quick), warte 60s");
                await Task.Delay(60_000, ct);
            }
            catch (Exception ex)
            {
                AddLog("error", "GitHub-Suche fehlgeschlagen (Quick): " + ex.Message);
            }
        }

        return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
    }

    private async Task<List<(string Key, string Url)>> ScanGitHub(CancellationToken ct)
    {
        var allKeys = new Dictionary<string, string>(StringComparer.Ordinal); // Key → erste URL
        var queries = BuildSearchQueries();

        var client = _httpFactory.CreateClient();
        client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
        client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
        client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.text-match+json"));

        for (int i = 0; i < queries.Count; i++)
        {
            ct.ThrowIfCancellationRequested();
            var (query, regex) = queries[i];
            SetProgress("GitHub Code Search", query, i + 1, queries.Count);

            try
            {
                var results = await SearchCodeAsync(client, query, regex, ct);
                foreach (var (key, url) in results)
                    if (!allKeys.ContainsKey(key)) allKeys[key] = url;
            }
            catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
            {
                SetProgress("GitHub Code Search", "⏳ Rate-Limit, warte 60s…", i + 1, queries.Count);
                AddLog("rate_limit", "GitHub Rate-Limit erreicht, warte 60s");
                await Task.Delay(60_000, ct);
                try
                {
                    var results = await SearchCodeAsync(client, query, regex, ct);
                    foreach (var (key, url) in results)
                        if (!allKeys.ContainsKey(key)) allKeys[key] = url;
                }
                catch (Exception retryEx)
                {
                    AddLog("error", "Retry fehlgeschlagen: " + retryEx.Message);
                }
            }
            catch (Exception ex)
            {
                AddLog("error", "GitHub-Suche fehlgeschlagen: " + ex.Message);
            }
        }

        return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
    }

    private static async Task<List<(string Key, string Url)>> SearchCodeAsync(HttpClient client, string query, Regex regex, CancellationToken ct)
    {
        var keys = new List<(string Key, string Url)>();
        int page = 1;
        int maxPages = MaxResultsPerSearch / ResultsPerPage;

        while (page <= maxPages)
        {
            ct.ThrowIfCancellationRequested();
            var url = $"https://api.github.com/search/code?q={Uri.EscapeDataString(query)}&per_page={ResultsPerPage}&page={page}";
            await Task.Delay(RateLimitDelayMs, ct);

            var resp = await client.GetAsync(url, ct);
            if (resp.StatusCode == System.Net.HttpStatusCode.UnprocessableEntity) break;
            resp.EnsureSuccessStatusCode();

            var json = await resp.Content.ReadAsStringAsync(ct);
            var doc = JsonDocument.Parse(json);
            var root = doc.RootElement;
            var totalCount = root.GetProperty("total_count").GetInt32();

            if (!root.TryGetProperty("items", out var items)) break;

            foreach (var item in items.EnumerateArray())
            {
                var htmlUrl = BuildHtmlUrl(item);

                if (item.TryGetProperty("text_matches", out var textMatches))
                {
                    foreach (var tm in textMatches.EnumerateArray())
                    {
                        if (tm.TryGetProperty("fragment", out var frag))
                        {
                            var matches = regex.Matches(frag.GetString() ?? "");
                            foreach (Match m in matches) keys.Add((m.Value, htmlUrl));
                        }
                    }
                }

                // Kein text_match → Raw-File fetchen
                var hasMatches = item.TryGetProperty("text_matches", out var tm2) && tm2.EnumerateArray().Any();
                if (!hasMatches)
                {
                    var rawKeys = await TryFetchRawFileKeysAsync(client, item, regex, ct);
                    foreach (var k in rawKeys) keys.Add((k, htmlUrl));
                }
            }

            if (items.GetArrayLength() < ResultsPerPage || page * ResultsPerPage >= totalCount) break;
            page++;
        }

        return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList();
    }

    /// <summary>Baut die GitHub HTML-URL aus einem Search-API-Item.</summary>
    private static string BuildHtmlUrl(JsonElement item)
    {
        if (item.TryGetProperty("html_url", out var htmlUrl))
            return htmlUrl.GetString() ?? "";

        // Fallback: aus repository.full_name + path zusammenbauen
        if (item.TryGetProperty("repository", out var repo) &&
            repo.TryGetProperty("full_name", out var fullName) &&
            item.TryGetProperty("path", out var path))
            return $"https://github.com/{fullName.GetString()}/blob/HEAD/{path.GetString()}";

        return "";
    }

    private static async Task<List<string>> TryFetchRawFileKeysAsync(HttpClient client, JsonElement item, Regex regex, CancellationToken ct)
    {
        var keys = new List<string>();
        try
        {
            if (!item.TryGetProperty("repository", out var repo) ||
                !repo.TryGetProperty("full_name", out var fullName) ||
                !item.TryGetProperty("path", out var path))
                return keys;

            var contentsUrl = $"https://api.github.com/repos/{fullName.GetString()}/contents/{path.GetString()}";
            var request = new HttpRequestMessage(HttpMethod.Get, contentsUrl);
            request.Headers.Accept.Clear();
            request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.raw+json"));

            await Task.Delay(500, ct);
            var resp = await client.SendAsync(request, ct);
            if (!resp.IsSuccessStatusCode) return keys;

            var content = await resp.Content.ReadAsStringAsync(ct);
            var matches = regex.Matches(content);
            foreach (Match m in matches) keys.Add(m.Value);
        }
        catch { /* Datei nicht lesbar – ignorieren */ }
        return keys;
    }

    private List<(string Query, Regex Regex)> BuildSearchQueries()
    {
        var queries = new List<(string, Regex)>();
        var activePatterns = GetActiveSearchPatterns();
        foreach (var (pattern, searchTerm, _) in activePatterns)
        {
            var regex = new Regex(pattern, RegexOptions.Compiled);
            foreach (var lang in SearchLanguages)
            {
                var langFilter = lang.Contains(' ') ? $"language:\"{lang}\"" : $"language:{lang}";
                queries.Add(($"{searchTerm} {langFilter}", regex));
            }
            
            foreach (var pathExt in PathExtensions)
            {
                queries.Add(($"{searchTerm} path:{pathExt}", regex));
            }
        }
        return queries;
    }

    // ── Serper.dev Google Search ──

    // ── GitHub Issues & PR Comments Scanner ──

    private const int IssueSearchDelayMs = 2500; // 30 req/min = 1 req/2s + buffer
    private const int CommentFetchDelayMs = 1000; // 5000 req/h = locker

    /// <summary>
    /// Durchsucht GitHub Issues, PR Bodies und Kommentare nach geleakten API-Keys.
    /// Quick-Mode: Nur Search API (Issue/PR body + comment search).
    /// Full-Mode: Search API + Kommentare der Treffer abrufen und scannen.
    /// </summary>
    private async Task<List<(string Key, string Url)>> ScanGitHubIssues(bool quickMode, CancellationToken ct)
    {
        var allKeys = new Dictionary<string, string>(StringComparer.Ordinal);
        var client = _httpFactory.CreateClient();
        client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
        client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
        client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");

        var activePatterns = GetActiveSearchPatterns();
        // Queries: jeder SearchTerm × (in:body, in:comments)
        var searchScopes = quickMode
            ? new[] { "in:body" }           // Quick: nur Body
            : new[] { "in:body", "in:comments" }; // Full: Body + Comments

        var queries = new List<(string Query, Regex Regex)>();
        foreach (var (pattern, searchTerm, _) in activePatterns)
        {
            var regex = new Regex(pattern, RegexOptions.Compiled);
            foreach (var scope in searchScopes)
                queries.Add(($"\"{searchTerm}\" {scope}", regex));
        }

        // Phase 1: Issue Search API
        var issueUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase); // Deduplizierung
        var allRegexes = activePatterns.Select(p => new Regex(p.Pattern, RegexOptions.Compiled)).ToArray();
        var issuesWithComments = new List<(string Owner, string Repo, int Number)>();

        for (int i = 0; i < queries.Count; i++)
        {
            ct.ThrowIfCancellationRequested();
            var (query, regex) = queries[i];
            SetProgress("GitHub Issues Search", query, i + 1, queries.Count);

            try
            {
                await Task.Delay(IssueSearchDelayMs, ct);
                var url = $"https://api.github.com/search/issues?q={Uri.EscapeDataString(query)}&sort=created&order=desc&per_page=100";
                var resp = await client.GetAsync(url, ct);

                if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden)
                {
                    SetProgress("GitHub Issues Search", "⏳ Rate-Limit, warte 60s…", i + 1, queries.Count);
                    AddLog("rate_limit", "GitHub Issues Rate-Limit, warte 60s");
                    await Task.Delay(60_000, ct);
                    continue;
                }
                if (resp.StatusCode == System.Net.HttpStatusCode.UnprocessableEntity) continue;
                resp.EnsureSuccessStatusCode();

                var json = await resp.Content.ReadAsStringAsync(ct);
                var doc = JsonDocument.Parse(json);

                if (!doc.RootElement.TryGetProperty("items", out var items)) continue;

                foreach (var item in items.EnumerateArray())
                {
                    var htmlUrl = item.TryGetProperty("html_url", out var hu) ? hu.GetString() ?? "" : "";
                    var body = item.TryGetProperty("body", out var b) ? b.GetString() ?? "" : "";

                    // Keys aus Issue/PR Body extrahieren
                    var matches = regex.Matches(body);
                    foreach (Match m in matches)
                        if (!allKeys.ContainsKey(m.Value)) allKeys[m.Value] = htmlUrl;

                    // Für Full-Mode: Issue merken um Kommentare abzurufen
                    if (!quickMode && !string.IsNullOrEmpty(htmlUrl) && issueUrls.Add(htmlUrl))
                    {
                        // Owner/Repo/Number aus URL parsen: https://github.com/{owner}/{repo}/issues/{number}
                        var parts = htmlUrl.Split('/');
                        if (parts.Length >= 7 && int.TryParse(parts[^1], out var number))
                        {
                            var owner = parts[^4];
                            var repo = parts[^3];
                            issuesWithComments.Add((owner, repo, number));
                        }
                    }
                }
            }
            catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
            {
                AddLog("rate_limit", "GitHub Issues Rate-Limit: " + ex.Message);
                await Task.Delay(60_000, ct);
            }
            catch (Exception ex)
            {
                AddLog("error", "GitHub Issues-Suche fehlgeschlagen: " + ex.Message);
            }
        }

        // Phase 2 (nur Full-Mode): Kommentare der Treffer abrufen
        if (!quickMode && issuesWithComments.Count > 0)
        {
            // Max 100 Issues abrufen (Rate-Limit schonen)
            var issuesToFetch = issuesWithComments.Take(100).ToList();
            for (int i = 0; i < issuesToFetch.Count; i++)
            {
                ct.ThrowIfCancellationRequested();
                var (owner, repo, number) = issuesToFetch[i];
                SetProgress("GitHub Issue Comments", $"{owner}/{repo}#{number}", i + 1, issuesToFetch.Count);

                try
                {
                    await Task.Delay(CommentFetchDelayMs, ct);
                    var commentsUrl = $"https://api.github.com/repos/{owner}/{repo}/issues/{number}/comments?per_page=100";
                    var resp = await client.GetAsync(commentsUrl, ct);

                    if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden)
                    {
                        AddLog("rate_limit", "GitHub Comments Rate-Limit, warte 60s");
                        await Task.Delay(60_000, ct);
                        continue;
                    }
                    if (!resp.IsSuccessStatusCode) continue;

                    var json = await resp.Content.ReadAsStringAsync(ct);
                    var doc = JsonDocument.Parse(json);

                    foreach (var comment in doc.RootElement.EnumerateArray())
                    {
                        var commentBody = comment.TryGetProperty("body", out var cb) ? cb.GetString() ?? "" : "";
                        var commentUrl = comment.TryGetProperty("html_url", out var cu) ? cu.GetString() ?? "" : "";
                        if (string.IsNullOrEmpty(commentUrl))
                            commentUrl = $"https://github.com/{owner}/{repo}/issues/{number}";

                        foreach (var regex in allRegexes)
                        {
                            var matches = regex.Matches(commentBody);
                            foreach (Match m in matches)
                                if (!allKeys.ContainsKey(m.Value)) allKeys[m.Value] = commentUrl;
                        }
                    }
                }
                catch (Exception ex)
                {
                    AddLog("error", $"GitHub Comments fehlgeschlagen ({owner}/{repo}#{number}): {ex.Message}");
                }
            }
        }

        AddLog("scan", $"GitHub Issues: {issueUrls.Count} Issues durchsucht, {allKeys.Count} Key(s) gefunden");
        return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
    }

    // ── Serper.dev Google Search ──

    /// <summary>
    /// Durchsucht Google via Serper.dev nach geleakten API-Keys.
    /// Jeder KeyPattern-SearchTerm wird als exakte Phrase gesucht.
    /// Snippets werden nach dem vollständigen Key-Pattern geparst.
    /// </summary>
    private async Task<List<(string Key, string Url)>> ScanSerper(CancellationToken ct)
    {
        var allKeys = new Dictionary<string, string>(StringComparer.Ordinal);
        var client = _httpFactory.CreateClient();
        client.DefaultRequestHeaders.Add("X-API-KEY", _serperApiKey);
        var activePatterns = GetActiveSearchPatterns();

        for (int i = 0; i < activePatterns.Length; i++)
        {
            var (pattern, searchTerm, _) = activePatterns[i];
            ct.ThrowIfCancellationRequested();
            SetProgress("Web-Suche (Serper)", searchTerm, i + 1, activePatterns.Length);
            var regex = new Regex(pattern, RegexOptions.Compiled);

            try
            {
                // Serper erlaubt bis zu 100 Ergebnisse pro Request (num: 100)
                var payload = JsonSerializer.Serialize(new { q = $"\"{searchTerm}\"", num = 100 });
                var request = new HttpRequestMessage(HttpMethod.Post, "https://google.serper.dev/search")
                {
                    Content = new StringContent(payload, Encoding.UTF8, "application/json")
                };

                var resp = await client.SendAsync(request, ct);
                resp.EnsureSuccessStatusCode();

                var json = await resp.Content.ReadAsStringAsync(ct);
                var doc = JsonDocument.Parse(json);

                // Organic results durchsuchen
                if (doc.RootElement.TryGetProperty("organic", out var organic))
                {
                    foreach (var item in organic.EnumerateArray())
                    {
                        var link = item.TryGetProperty("link", out var l) ? l.GetString() ?? "" : "";
                        var snippet = item.TryGetProperty("snippet", out var s) ? s.GetString() ?? "" : "";
                        var title = item.TryGetProperty("title", out var t) ? t.GetString() ?? "" : "";

                        // Keys aus Snippet und Title extrahieren
                        var text = snippet + " " + title;
                        var matches = regex.Matches(text);
                        foreach (Match m in matches)
                            if (!allKeys.ContainsKey(m.Value)) allKeys[m.Value] = link;
                    }
                }

                AddLog("scan", "Serper '" + searchTerm + "': " + allKeys.Count + " Ergebnis(se)");
                await Task.Delay(1000, ct); // Rate-Limit schonen
            }
            catch (Exception ex)
            {
                AddLog("error", "Serper-Suche fehlgeschlagen für '" + searchTerm + "': " + ex.Message);
            }
        }

        return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
    }

    // ── Hugging Face Hub Scanner ──

    private static readonly string[] HuggingFaceSearchTerms =
    {
        "anthropic", "claude", "openai", "api-key", "chatgpt", "gpt4"
    };

    /// <summary>Zu scannende Dateien – priorisiert nach Wahrscheinlichkeit für Keys.</summary>
    private static readonly string[] HuggingFaceScanFiles =
    {
        ".env", ".env.local", ".env.production", ".env.example",
        "app.py", "main.py", "config.py", "settings.py", "utils.py",
        ".streamlit/secrets.toml", "config.yaml", "config.yml", "config.json",
        "docker-compose.yml", "Dockerfile"
    };

    /// <summary>Datei-Extensions die bei der Dateiliste zusätzlich gescannt werden.</summary>
    private static readonly string[] HuggingFaceScanExtensions =
    {
        ".py", ".env", ".toml", ".yaml", ".yml", ".json", ".txt", ".cfg", ".ini", ".conf", ".ipynb"
    };

    private const int HuggingFaceMaxFilesPerRepo = 50;
    private const int HuggingFaceMaxFileSizeBytes = 500_000; // 500 KB
    private const int HuggingFaceRequestDelayMs = 300;

    /// <summary>
    /// Scannt Hugging Face Spaces und Models nach geleakten API-Keys.
    /// Sucht nach relevanten Repos, lädt Dateien herunter und prüft mit allen Key-Patterns.
    /// </summary>
    private async Task<List<(string Key, string Url)>> ScanHuggingFace(CancellationToken ct)
    {
        var allKeys = new Dictionary<string, string>(StringComparer.Ordinal);
        var client = _httpFactory.CreateClient();
        client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));

        var hfToken = _settings.Get("HuggingFaceToken");
        if (!string.IsNullOrWhiteSpace(hfToken))
            client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", hfToken);

        var scannedRepos = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
        var totalQueries = HuggingFaceSearchTerms.Length * 2; // spaces + models
        var queryIdx = 0;

        // Phase 1: Spaces durchsuchen
        foreach (var term in HuggingFaceSearchTerms)
        {
            ct.ThrowIfCancellationRequested();
            queryIdx++;
            SetProgress("HuggingFace Spaces", term, queryIdx, totalQueries);

            var repos = await HuggingFaceSearchRepos(client, "spaces", term, ct);
            foreach (var repoId in repos)
            {
                if (!scannedRepos.Add(repoId)) continue;
                var keys = await HuggingFaceScanRepo(client, "spaces", repoId, ct);
                foreach (var (key, url) in keys)
                    if (!allKeys.ContainsKey(key)) allKeys[key] = url;
            }
        }

        // Phase 2: Models durchsuchen
        foreach (var term in HuggingFaceSearchTerms)
        {
            ct.ThrowIfCancellationRequested();
            queryIdx++;
            SetProgress("HuggingFace Models", term, queryIdx, totalQueries);

            var repos = await HuggingFaceSearchRepos(client, "models", term, ct);
            foreach (var repoId in repos)
            {
                if (!scannedRepos.Add(repoId)) continue;
                var keys = await HuggingFaceScanRepo(client, "models", repoId, ct);
                foreach (var (key, url) in keys)
                    if (!allKeys.ContainsKey(key)) allKeys[key] = url;
            }
        }

        AddLog("scan", $"HuggingFace: {scannedRepos.Count} Repos gescannt, {allKeys.Count} Key(s) gefunden");
        return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
    }

    /// <summary>Sucht Repos (Spaces oder Models) auf HuggingFace.</summary>
    private async Task<List<string>> HuggingFaceSearchRepos(HttpClient client, string repoType, string searchTerm, CancellationToken ct)
    {
        var repos = new List<string>();
        try
        {
            await Task.Delay(HuggingFaceRequestDelayMs, ct);
            var url = $"https://huggingface.co/api/{repoType}?search={Uri.EscapeDataString(searchTerm)}&limit=100";
            var resp = await client.GetAsync(url, ct);
            if (!resp.IsSuccessStatusCode) return repos;

            var json = await resp.Content.ReadAsStringAsync(ct);
            var doc = JsonDocument.Parse(json);
            foreach (var item in doc.RootElement.EnumerateArray())
            {
                if (item.TryGetProperty("id", out var id))
                    repos.Add(id.GetString());
            }
        }
        catch (Exception ex)
        {
            AddLog("error", $"HuggingFace-Suche fehlgeschlagen ({repoType}/{searchTerm}): {ex.Message}");
        }
        return repos;
    }

    /// <summary>Scannt ein einzelnes HuggingFace-Repo nach API-Keys.</summary>
    private async Task<List<(string Key, string Url)>> HuggingFaceScanRepo(HttpClient client, string repoType, string repoId, CancellationToken ct)
    {
        var keys = new List<(string Key, string Url)>();
        try
        {
            // Dateiliste abrufen
            await Task.Delay(HuggingFaceRequestDelayMs, ct);
            var treeUrl = $"https://huggingface.co/api/{repoType}/{repoId}/tree/main?recursive=true";
            var treeResp = await client.GetAsync(treeUrl, ct);
            if (!treeResp.IsSuccessStatusCode) return keys;

            var treeJson = await treeResp.Content.ReadAsStringAsync(ct);
            var treeDoc = JsonDocument.Parse(treeJson);

            // Relevante Dateien filtern
            var filesToScan = new List<(string Path, long Size)>();
            foreach (var file in treeDoc.RootElement.EnumerateArray())
            {
                if (!file.TryGetProperty("type", out var type) || type.GetString() != "file") continue;
                if (!file.TryGetProperty("path", out var path)) continue;
                var filePath = path.GetString();
                var fileSize = file.TryGetProperty("size", out var sz) ? sz.GetInt64() : 0;

                if (fileSize > HuggingFaceMaxFileSizeBytes) continue;

                // Prüfe ob die Datei in der Prioritätsliste ist oder eine relevante Extension hat
                var isPriority = HuggingFaceScanFiles.Any(f => filePath.EndsWith(f, StringComparison.OrdinalIgnoreCase)
                    || filePath.Equals(f, StringComparison.OrdinalIgnoreCase));
                var hasRelevantExt = HuggingFaceScanExtensions.Any(ext =>
                    filePath.EndsWith(ext, StringComparison.OrdinalIgnoreCase));

                if (isPriority || hasRelevantExt)
                    filesToScan.Add((filePath, fileSize));
            }

            // Max Dateien pro Repo begrenzen
            if (filesToScan.Count > HuggingFaceMaxFilesPerRepo)
                filesToScan = filesToScan.Take(HuggingFaceMaxFilesPerRepo).ToList();

            // Dateien herunterladen und scannen
            // repoType "spaces" → URL-Prefix "spaces", "models" → kein Prefix (nur repoId)
            var urlPrefix = repoType == "spaces" ? "spaces" : repoType == "models" ? "" : repoType;

            foreach (var (filePath, _) in filesToScan)
            {
                ct.ThrowIfCancellationRequested();
                try
                {
                    await Task.Delay(HuggingFaceRequestDelayMs, ct);
                    var rawUrl = urlPrefix == ""
                        ? $"https://huggingface.co/{repoId}/resolve/main/{filePath}"
                        : $"https://huggingface.co/{urlPrefix}/{repoId}/resolve/main/{filePath}";

                    var request = new HttpRequestMessage(HttpMethod.Get, rawUrl);
                    request.Headers.Add("Accept", "text/plain");
                    var resp = await client.SendAsync(request, ct);
                    if (!resp.IsSuccessStatusCode) continue;

                    var content = await resp.Content.ReadAsStringAsync(ct);
                    if (string.IsNullOrWhiteSpace(content)) continue;

                    // Alle Key-Patterns prüfen
                    foreach (var regex in KeyPatterns.Compiled)
                    {
                        var matches = regex.Matches(content);
                        foreach (Match m in matches)
                        {
                            var sourceUrl = urlPrefix == ""
                                ? $"https://huggingface.co/{repoId}/blob/main/{filePath}"
                                : $"https://huggingface.co/{urlPrefix}/{repoId}/blob/main/{filePath}";
                            keys.Add((m.Value, sourceUrl));
                        }
                    }
                }
                catch { /* Einzelne Datei fehlgeschlagen – ignorieren */ }
            }
        }
        catch (Exception ex)
        {
            AddLog("error", $"HuggingFace-Repo-Scan fehlgeschlagen ({repoId}): {ex.Message}");
        }
        return keys;
    }

    // ── Key Check ──

    public async Task CheckAndUpdateKey(ScannedKeyRecord sk)
    {
        var type = KeyPatterns.DetectType(sk.ApiKey);
        var keyPrefix = sk.ApiKey[..Math.Min(20, sk.ApiKey.Length)] + "...";
        var typeName = KeyPatterns.DisplayName(type);

        try
        {
            var check = await KeyCheckDispatcher.CheckByKeyType(_httpFactory, type, sk.ApiKey);
            if (check != null)
            {
                await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, check.DbStatus, check.Plan, check.Usage7d, check.Usage5h);
                var checkMsg = check.Usage7d.HasValue
                    ? $"🔍 {typeName} geprüft: {check.DbStatus} · Plan: {check.Plan} · Usage 7d: {check.Usage7d?.ToString("F2") ?? "n/a"} · Usage 5h: {check.Usage5h?.ToString("F2") ?? "n/a"} · {keyPrefix}"
                    : $"🔍 {typeName} geprüft: {check.DbStatus} · {check.Message} · {keyPrefix}";
                AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status = check.DbStatus, plan = check.Plan, usage_7d = check.Usage7d, usage_5h = check.Usage5h, message = check.Message, source_url = sk.SourceUrl }));
            }
            else
            {
                // Andere Key-Typen: Typ korrekt setzen, kein Check möglich
                await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "unchecked", type, null, null);
                AddLog("key_checked", $"⏭️ {typeName} übersprungen (kein Checker) · {keyPrefix}");
            }
        }
        catch (Exception ex)
        {
            AddLog("error", $"Key-Check fehlgeschlagen für {typeName} ({keyPrefix}): {ex.Message}");
            await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null);
        }
    }

    private static string DetectKeyType(string key) => KeyPatterns.DetectType(key);
}