feat: HuggingFace Hub Scanner - Spaces und Models nach API-Keys durchsuchen

- Neue ScanHuggingFace()-Methode: durchsucht Spaces und Models - 6 Suchbegriffe (anthropic, claude, openai, api-key, chatgpt, gpt4) - Rekursive Dateiliste, filtert nach relevanten Extensions - Limits: 50 Dateien/Repo, 500KB/Datei, 300ms Delay - Läuft in Web- und Full-Mode (nicht Quick) - Optionales HuggingFaceToken-Setting für höhere Rate-Limits - CSS: font-size Bereinigung in chat.css
2026-03-14 18:02:47 +01:00
1 parent e1a9117b4a
commit 79d7a88c5c
@@ -228,6 +228,14 @@ public class KeyScannerService : BackgroundService
                AddLog("scan", "Serper-Scan: " + spKeys.Count + " Key(s) gefunden");
            }

+            if (mode != "quick")
+            {
+                SetProgress("HuggingFace Hub", "Starte…");
+                var hfKeys = await ScanHuggingFace(ct);
+                allFoundKeys.AddRange(hfKeys.Select(k => (k.Key, k.Url, "huggingface")));
+                AddLog("scan", "HuggingFace-Scan: " + hfKeys.Count + " Key(s) gefunden");
+            }
+
            totalFound = allFoundKeys.Count;

            // Phase 2: Neue Keys in DB speichern (dedupliziert)
@@ -591,6 +599,199 @@ public class KeyScannerService : BackgroundService
        return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
    }

+    // ── Hugging Face Hub Scanner ──
+
+    private static readonly string[] HuggingFaceSearchTerms =
+    {
+        "anthropic", "claude", "openai", "api-key", "chatgpt", "gpt4"
+    };
+
+    /// <summary>Zu scannende Dateien – priorisiert nach Wahrscheinlichkeit für Keys.</summary>
+    private static readonly string[] HuggingFaceScanFiles =
+    {
+        ".env", ".env.local", ".env.production", ".env.example",
+        "app.py", "main.py", "config.py", "settings.py", "utils.py",
+        ".streamlit/secrets.toml", "config.yaml", "config.yml", "config.json",
+        "docker-compose.yml", "Dockerfile"
+    };
+
+    /// <summary>Datei-Extensions die bei der Dateiliste zusätzlich gescannt werden.</summary>
+    private static readonly string[] HuggingFaceScanExtensions =
+    {
+        ".py", ".env", ".toml", ".yaml", ".yml", ".json", ".txt", ".cfg", ".ini", ".conf", ".ipynb"
+    };
+
+    private const int HuggingFaceMaxFilesPerRepo = 50;
+    private const int HuggingFaceMaxFileSizeBytes = 500_000; // 500 KB
+    private const int HuggingFaceRequestDelayMs = 300;
+
+    /// <summary>
+    /// Scannt Hugging Face Spaces und Models nach geleakten API-Keys.
+    /// Sucht nach relevanten Repos, lädt Dateien herunter und prüft mit allen Key-Patterns.
+    /// </summary>
+    private async Task<List<(string Key, string Url)>> ScanHuggingFace(CancellationToken ct)
+    {
+        var allKeys = new Dictionary<string, string>(StringComparer.Ordinal);
+        var client = _httpFactory.CreateClient();
+        client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
+
+        var hfToken = _settings.Get("HuggingFaceToken");
+        if (!string.IsNullOrWhiteSpace(hfToken))
+            client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", hfToken);
+
+        var scannedRepos = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        var totalQueries = HuggingFaceSearchTerms.Length * 2; // spaces + models
+        var queryIdx = 0;
+
+        // Phase 1: Spaces durchsuchen
+        foreach (var term in HuggingFaceSearchTerms)
+        {
+            ct.ThrowIfCancellationRequested();
+            queryIdx++;
+            SetProgress("HuggingFace Spaces", term, queryIdx, totalQueries);
+
+            var repos = await HuggingFaceSearchRepos(client, "spaces", term, ct);
+            foreach (var repoId in repos)
+            {
+                if (!scannedRepos.Add(repoId)) continue;
+                var keys = await HuggingFaceScanRepo(client, "spaces", repoId, ct);
+                foreach (var (key, url) in keys)
+                    if (!allKeys.ContainsKey(key)) allKeys[key] = url;
+            }
+        }
+
+        // Phase 2: Models durchsuchen
+        foreach (var term in HuggingFaceSearchTerms)
+        {
+            ct.ThrowIfCancellationRequested();
+            queryIdx++;
+            SetProgress("HuggingFace Models", term, queryIdx, totalQueries);
+
+            var repos = await HuggingFaceSearchRepos(client, "models", term, ct);
+            foreach (var repoId in repos)
+            {
+                if (!scannedRepos.Add(repoId)) continue;
+                var keys = await HuggingFaceScanRepo(client, "models", repoId, ct);
+                foreach (var (key, url) in keys)
+                    if (!allKeys.ContainsKey(key)) allKeys[key] = url;
+            }
+        }
+
+        AddLog("scan", $"HuggingFace: {scannedRepos.Count} Repos gescannt, {allKeys.Count} Key(s) gefunden");
+        return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
+    }
+
+    /// <summary>Sucht Repos (Spaces oder Models) auf HuggingFace.</summary>
+    private async Task<List<string>> HuggingFaceSearchRepos(HttpClient client, string repoType, string searchTerm, CancellationToken ct)
+    {
+        var repos = new List<string>();
+        try
+        {
+            await Task.Delay(HuggingFaceRequestDelayMs, ct);
+            var url = $"https://huggingface.co/api/{repoType}?search={Uri.EscapeDataString(searchTerm)}&limit=100";
+            var resp = await client.GetAsync(url, ct);
+            if (!resp.IsSuccessStatusCode) return repos;
+
+            var json = await resp.Content.ReadAsStringAsync(ct);
+            var doc = JsonDocument.Parse(json);
+            foreach (var item in doc.RootElement.EnumerateArray())
+            {
+                if (item.TryGetProperty("id", out var id))
+                    repos.Add(id.GetString());
+            }
+        }
+        catch (Exception ex)
+        {
+            AddLog("error", $"HuggingFace-Suche fehlgeschlagen ({repoType}/{searchTerm}): {ex.Message}");
+        }
+        return repos;
+    }
+
+    /// <summary>Scannt ein einzelnes HuggingFace-Repo nach API-Keys.</summary>
+    private async Task<List<(string Key, string Url)>> HuggingFaceScanRepo(HttpClient client, string repoType, string repoId, CancellationToken ct)
+    {
+        var keys = new List<(string Key, string Url)>();
+        try
+        {
+            // Dateiliste abrufen
+            await Task.Delay(HuggingFaceRequestDelayMs, ct);
+            var treeUrl = $"https://huggingface.co/api/{repoType}/{repoId}/tree/main?recursive=true";
+            var treeResp = await client.GetAsync(treeUrl, ct);
+            if (!treeResp.IsSuccessStatusCode) return keys;
+
+            var treeJson = await treeResp.Content.ReadAsStringAsync(ct);
+            var treeDoc = JsonDocument.Parse(treeJson);
+
+            // Relevante Dateien filtern
+            var filesToScan = new List<(string Path, long Size)>();
+            foreach (var file in treeDoc.RootElement.EnumerateArray())
+            {
+                if (!file.TryGetProperty("type", out var type) || type.GetString() != "file") continue;
+                if (!file.TryGetProperty("path", out var path)) continue;
+                var filePath = path.GetString();
+                var fileSize = file.TryGetProperty("size", out var sz) ? sz.GetInt64() : 0;
+
+                if (fileSize > HuggingFaceMaxFileSizeBytes) continue;
+
+                // Prüfe ob die Datei in der Prioritätsliste ist oder eine relevante Extension hat
+                var isPriority = HuggingFaceScanFiles.Any(f => filePath.EndsWith(f, StringComparison.OrdinalIgnoreCase)
+                    || filePath.Equals(f, StringComparison.OrdinalIgnoreCase));
+                var hasRelevantExt = HuggingFaceScanExtensions.Any(ext =>
+                    filePath.EndsWith(ext, StringComparison.OrdinalIgnoreCase));
+
+                if (isPriority || hasRelevantExt)
+                    filesToScan.Add((filePath, fileSize));
+            }
+
+            // Max Dateien pro Repo begrenzen
+            if (filesToScan.Count > HuggingFaceMaxFilesPerRepo)
+                filesToScan = filesToScan.Take(HuggingFaceMaxFilesPerRepo).ToList();
+
+            // Dateien herunterladen und scannen
+            // repoType "spaces" → URL-Prefix "spaces", "models" → kein Prefix (nur repoId)
+            var urlPrefix = repoType == "spaces" ? "spaces" : repoType == "models" ? "" : repoType;
+
+            foreach (var (filePath, _) in filesToScan)
+            {
+                ct.ThrowIfCancellationRequested();
+                try
+                {
+                    await Task.Delay(HuggingFaceRequestDelayMs, ct);
+                    var rawUrl = urlPrefix == ""
+                        ? $"https://huggingface.co/{repoId}/resolve/main/{filePath}"
+                        : $"https://huggingface.co/{urlPrefix}/{repoId}/resolve/main/{filePath}";
+
+                    var request = new HttpRequestMessage(HttpMethod.Get, rawUrl);
+                    request.Headers.Add("Accept", "text/plain");
+                    var resp = await client.SendAsync(request, ct);
+                    if (!resp.IsSuccessStatusCode) continue;
+
+                    var content = await resp.Content.ReadAsStringAsync(ct);
+                    if (string.IsNullOrWhiteSpace(content)) continue;
+
+                    // Alle Key-Patterns prüfen
+                    foreach (var regex in KeyPatterns.Compiled)
+                    {
+                        var matches = regex.Matches(content);
+                        foreach (Match m in matches)
+                        {
+                            var sourceUrl = urlPrefix == ""
+                                ? $"https://huggingface.co/{repoId}/blob/main/{filePath}"
+                                : $"https://huggingface.co/{urlPrefix}/{repoId}/blob/main/{filePath}";
+                            keys.Add((m.Value, sourceUrl));
+                        }
+                    }
+                }
+                catch { /* Einzelne Datei fehlgeschlagen – ignorieren */ }
+            }
+        }
+        catch (Exception ex)
+        {
+            AddLog("error", $"HuggingFace-Repo-Scan fehlgeschlagen ({repoId}): {ex.Message}");
+        }
+        return keys;
+    }
+
    // ── Key Check ──

    public async Task CheckAndUpdateKey(ScannedKeyRecord sk)
@@ -35,6 +35,7 @@ public class ServerSettingsService
        new SettingDefinition( "SerperApiKey",       "string", "",      "Serper.dev API-Key für Google-Suche (Key-Scanner + Tool)",       "password" ),
        new SettingDefinition( "ScanIntervalHours",  "int",    "0",     "Key-Scanner Intervall in Stunden (0 = nur manuell)",             "number"   ),
        new SettingDefinition( "GitHubEventScanner", "bool",   "false", "GitHub Event Scanner (Echtzeit-Monitoring öffentlicher Pushes)", "toggle"   ),
+        new SettingDefinition( "HuggingFaceToken",   "string", "",      "HuggingFace Token (optional, erhöht Rate-Limits)",            "password" ),
        new SettingDefinition( "ScannerKeyTypes",    "string_array", "anthropic_oauth,krea_ai", "Key-Scanner: Aktive Key-Typen",                               "checkboxes" ),
    };

@@ -324,7 +324,6 @@
  padding-top: 0;
  display: flex;
  flex-direction: column;
-  font-size: var(--font-size);
  gap: var(--badge-spacing);
 }