From 79d7a88c5cf3364dd49a09bc6bde03d9c3f08787 Mon Sep 17 00:00:00 2001 From: Meleki Date: Sat, 14 Mar 2026 18:02:47 +0100 Subject: [PATCH] feat: HuggingFace Hub Scanner - Spaces und Models nach API-Keys durchsuchen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Neue ScanHuggingFace()-Methode: durchsucht Spaces und Models - 6 Suchbegriffe (anthropic, claude, openai, api-key, chatgpt, gpt4) - Rekursive Dateiliste, filtert nach relevanten Extensions - Limits: 50 Dateien/Repo, 500KB/Datei, 300ms Delay - Läuft in Web- und Full-Mode (nicht Quick) - Optionales HuggingFaceToken-Setting für höhere Rate-Limits - CSS: font-size Bereinigung in chat.css --- Services/KeyScannerService.cs | 201 ++++++++++++++++++++++++++++++ Services/ServerSettingsService.cs | 1 + wwwroot/css/chat.css | 1 - 3 files changed, 202 insertions(+), 1 deletion(-) diff --git a/Services/KeyScannerService.cs b/Services/KeyScannerService.cs index f3de668..9559780 100644 --- a/Services/KeyScannerService.cs +++ b/Services/KeyScannerService.cs @@ -228,6 +228,14 @@ public class KeyScannerService : BackgroundService AddLog("scan", "Serper-Scan: " + spKeys.Count + " Key(s) gefunden"); } + if (mode != "quick") + { + SetProgress("HuggingFace Hub", "Starte…"); + var hfKeys = await ScanHuggingFace(ct); + allFoundKeys.AddRange(hfKeys.Select(k => (k.Key, k.Url, "huggingface"))); + AddLog("scan", "HuggingFace-Scan: " + hfKeys.Count + " Key(s) gefunden"); + } + totalFound = allFoundKeys.Count; // Phase 2: Neue Keys in DB speichern (dedupliziert) @@ -591,6 +599,199 @@ public class KeyScannerService : BackgroundService return allKeys.Select(kv => (kv.Key, kv.Value)).ToList(); } + // ── Hugging Face Hub Scanner ── + + private static readonly string[] HuggingFaceSearchTerms = + { + "anthropic", "claude", "openai", "api-key", "chatgpt", "gpt4" + }; + + /// Zu scannende Dateien – priorisiert nach Wahrscheinlichkeit für Keys. + private static readonly string[] HuggingFaceScanFiles = + { + ".env", ".env.local", ".env.production", ".env.example", + "app.py", "main.py", "config.py", "settings.py", "utils.py", + ".streamlit/secrets.toml", "config.yaml", "config.yml", "config.json", + "docker-compose.yml", "Dockerfile" + }; + + /// Datei-Extensions die bei der Dateiliste zusätzlich gescannt werden. + private static readonly string[] HuggingFaceScanExtensions = + { + ".py", ".env", ".toml", ".yaml", ".yml", ".json", ".txt", ".cfg", ".ini", ".conf", ".ipynb" + }; + + private const int HuggingFaceMaxFilesPerRepo = 50; + private const int HuggingFaceMaxFileSizeBytes = 500_000; // 500 KB + private const int HuggingFaceRequestDelayMs = 300; + + /// + /// Scannt Hugging Face Spaces und Models nach geleakten API-Keys. + /// Sucht nach relevanten Repos, lädt Dateien herunter und prüft mit allen Key-Patterns. + /// + private async Task> ScanHuggingFace(CancellationToken ct) + { + var allKeys = new Dictionary(StringComparer.Ordinal); + var client = _httpFactory.CreateClient(); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0")); + + var hfToken = _settings.Get("HuggingFaceToken"); + if (!string.IsNullOrWhiteSpace(hfToken)) + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", hfToken); + + var scannedRepos = new HashSet(StringComparer.OrdinalIgnoreCase); + var totalQueries = HuggingFaceSearchTerms.Length * 2; // spaces + models + var queryIdx = 0; + + // Phase 1: Spaces durchsuchen + foreach (var term in HuggingFaceSearchTerms) + { + ct.ThrowIfCancellationRequested(); + queryIdx++; + SetProgress("HuggingFace Spaces", term, queryIdx, totalQueries); + + var repos = await HuggingFaceSearchRepos(client, "spaces", term, ct); + foreach (var repoId in repos) + { + if (!scannedRepos.Add(repoId)) continue; + var keys = await HuggingFaceScanRepo(client, "spaces", repoId, ct); + foreach (var (key, url) in keys) + if (!allKeys.ContainsKey(key)) allKeys[key] = url; + } + } + + // Phase 2: Models durchsuchen + foreach (var term in HuggingFaceSearchTerms) + { + ct.ThrowIfCancellationRequested(); + queryIdx++; + SetProgress("HuggingFace Models", term, queryIdx, totalQueries); + + var repos = await HuggingFaceSearchRepos(client, "models", term, ct); + foreach (var repoId in repos) + { + if (!scannedRepos.Add(repoId)) continue; + var keys = await HuggingFaceScanRepo(client, "models", repoId, ct); + foreach (var (key, url) in keys) + if (!allKeys.ContainsKey(key)) allKeys[key] = url; + } + } + + AddLog("scan", $"HuggingFace: {scannedRepos.Count} Repos gescannt, {allKeys.Count} Key(s) gefunden"); + return allKeys.Select(kv => (kv.Key, kv.Value)).ToList(); + } + + /// Sucht Repos (Spaces oder Models) auf HuggingFace. + private async Task> HuggingFaceSearchRepos(HttpClient client, string repoType, string searchTerm, CancellationToken ct) + { + var repos = new List(); + try + { + await Task.Delay(HuggingFaceRequestDelayMs, ct); + var url = $"https://huggingface.co/api/{repoType}?search={Uri.EscapeDataString(searchTerm)}&limit=100"; + var resp = await client.GetAsync(url, ct); + if (!resp.IsSuccessStatusCode) return repos; + + var json = await resp.Content.ReadAsStringAsync(ct); + var doc = JsonDocument.Parse(json); + foreach (var item in doc.RootElement.EnumerateArray()) + { + if (item.TryGetProperty("id", out var id)) + repos.Add(id.GetString()); + } + } + catch (Exception ex) + { + AddLog("error", $"HuggingFace-Suche fehlgeschlagen ({repoType}/{searchTerm}): {ex.Message}"); + } + return repos; + } + + /// Scannt ein einzelnes HuggingFace-Repo nach API-Keys. + private async Task> HuggingFaceScanRepo(HttpClient client, string repoType, string repoId, CancellationToken ct) + { + var keys = new List<(string Key, string Url)>(); + try + { + // Dateiliste abrufen + await Task.Delay(HuggingFaceRequestDelayMs, ct); + var treeUrl = $"https://huggingface.co/api/{repoType}/{repoId}/tree/main?recursive=true"; + var treeResp = await client.GetAsync(treeUrl, ct); + if (!treeResp.IsSuccessStatusCode) return keys; + + var treeJson = await treeResp.Content.ReadAsStringAsync(ct); + var treeDoc = JsonDocument.Parse(treeJson); + + // Relevante Dateien filtern + var filesToScan = new List<(string Path, long Size)>(); + foreach (var file in treeDoc.RootElement.EnumerateArray()) + { + if (!file.TryGetProperty("type", out var type) || type.GetString() != "file") continue; + if (!file.TryGetProperty("path", out var path)) continue; + var filePath = path.GetString(); + var fileSize = file.TryGetProperty("size", out var sz) ? sz.GetInt64() : 0; + + if (fileSize > HuggingFaceMaxFileSizeBytes) continue; + + // Prüfe ob die Datei in der Prioritätsliste ist oder eine relevante Extension hat + var isPriority = HuggingFaceScanFiles.Any(f => filePath.EndsWith(f, StringComparison.OrdinalIgnoreCase) + || filePath.Equals(f, StringComparison.OrdinalIgnoreCase)); + var hasRelevantExt = HuggingFaceScanExtensions.Any(ext => + filePath.EndsWith(ext, StringComparison.OrdinalIgnoreCase)); + + if (isPriority || hasRelevantExt) + filesToScan.Add((filePath, fileSize)); + } + + // Max Dateien pro Repo begrenzen + if (filesToScan.Count > HuggingFaceMaxFilesPerRepo) + filesToScan = filesToScan.Take(HuggingFaceMaxFilesPerRepo).ToList(); + + // Dateien herunterladen und scannen + // repoType "spaces" → URL-Prefix "spaces", "models" → kein Prefix (nur repoId) + var urlPrefix = repoType == "spaces" ? "spaces" : repoType == "models" ? "" : repoType; + + foreach (var (filePath, _) in filesToScan) + { + ct.ThrowIfCancellationRequested(); + try + { + await Task.Delay(HuggingFaceRequestDelayMs, ct); + var rawUrl = urlPrefix == "" + ? $"https://huggingface.co/{repoId}/resolve/main/{filePath}" + : $"https://huggingface.co/{urlPrefix}/{repoId}/resolve/main/{filePath}"; + + var request = new HttpRequestMessage(HttpMethod.Get, rawUrl); + request.Headers.Add("Accept", "text/plain"); + var resp = await client.SendAsync(request, ct); + if (!resp.IsSuccessStatusCode) continue; + + var content = await resp.Content.ReadAsStringAsync(ct); + if (string.IsNullOrWhiteSpace(content)) continue; + + // Alle Key-Patterns prüfen + foreach (var regex in KeyPatterns.Compiled) + { + var matches = regex.Matches(content); + foreach (Match m in matches) + { + var sourceUrl = urlPrefix == "" + ? $"https://huggingface.co/{repoId}/blob/main/{filePath}" + : $"https://huggingface.co/{urlPrefix}/{repoId}/blob/main/{filePath}"; + keys.Add((m.Value, sourceUrl)); + } + } + } + catch { /* Einzelne Datei fehlgeschlagen – ignorieren */ } + } + } + catch (Exception ex) + { + AddLog("error", $"HuggingFace-Repo-Scan fehlgeschlagen ({repoId}): {ex.Message}"); + } + return keys; + } + // ── Key Check ── public async Task CheckAndUpdateKey(ScannedKeyRecord sk) diff --git a/Services/ServerSettingsService.cs b/Services/ServerSettingsService.cs index d14d5c5..7eec3f3 100644 --- a/Services/ServerSettingsService.cs +++ b/Services/ServerSettingsService.cs @@ -35,6 +35,7 @@ public class ServerSettingsService new SettingDefinition( "SerperApiKey", "string", "", "Serper.dev API-Key für Google-Suche (Key-Scanner + Tool)", "password" ), new SettingDefinition( "ScanIntervalHours", "int", "0", "Key-Scanner Intervall in Stunden (0 = nur manuell)", "number" ), new SettingDefinition( "GitHubEventScanner", "bool", "false", "GitHub Event Scanner (Echtzeit-Monitoring öffentlicher Pushes)", "toggle" ), + new SettingDefinition( "HuggingFaceToken", "string", "", "HuggingFace Token (optional, erhöht Rate-Limits)", "password" ), new SettingDefinition( "ScannerKeyTypes", "string_array", "anthropic_oauth,krea_ai", "Key-Scanner: Aktive Key-Typen", "checkboxes" ), }; diff --git a/wwwroot/css/chat.css b/wwwroot/css/chat.css index ba8c571..06dcfe9 100644 --- a/wwwroot/css/chat.css +++ b/wwwroot/css/chat.css @@ -324,7 +324,6 @@ padding-top: 0; display: flex; flex-direction: column; - font-size: var(--font-size); gap: var(--badge-spacing); }