meleki/gott
1
0 Fork 0
Code Activity

feat: HuggingFace Hub Scanner - Spaces und Models nach API-Keys durchsuchen

Browse Source 
- Neue ScanHuggingFace()-Methode: durchsucht Spaces und Models
- 6 Suchbegriffe (anthropic, claude, openai, api-key, chatgpt, gpt4)
- Rekursive Dateiliste, filtert nach relevanten Extensions
- Limits: 50 Dateien/Repo, 500KB/Datei, 300ms Delay
- Läuft in Web- und Full-Mode (nicht Quick)
- Optionales HuggingFaceToken-Setting für höhere Rate-Limits
- CSS: font-size Bereinigung in chat.css
This commit is contained in:
Meleki committed 2026-03-14 18:02:47 +01:00
1 parent e1a9117b4a
commit 79d7a88c5c
3 files changed
+202 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Services/KeyScannerService.cs
+201
View File
@@ -228,6 +228,14 @@ public class KeyScannerService : BackgroundService
AddLog("scan", "Serper-Scan: " + spKeys.Count + " Key(s) gefunden");
}
if (mode != "quick")
{
SetProgress("HuggingFace Hub", "Starte…");
var hfKeys = await ScanHuggingFace(ct);
allFoundKeys.AddRange(hfKeys.Select(k => (k.Key, k.Url, "huggingface")));
AddLog("scan", "HuggingFace-Scan: " + hfKeys.Count + " Key(s) gefunden");
}
totalFound = allFoundKeys.Count;
// Phase 2: Neue Keys in DB speichern (dedupliziert)
@@ -591,6 +599,199 @@ public class KeyScannerService : BackgroundService
return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
}
// ── Hugging Face Hub Scanner ──
private static readonly string[] HuggingFaceSearchTerms =
{
"anthropic", "claude", "openai", "api-key", "chatgpt", "gpt4"
};
/// <summary>Zu scannende Dateien – priorisiert nach Wahrscheinlichkeit für Keys.</summary>
private static readonly string[] HuggingFaceScanFiles =
{
".env", ".env.local", ".env.production", ".env.example",
"app.py", "main.py", "config.py", "settings.py", "utils.py",
".streamlit/secrets.toml", "config.yaml", "config.yml", "config.json",
"docker-compose.yml", "Dockerfile"
};
/// <summary>Datei-Extensions die bei der Dateiliste zusätzlich gescannt werden.</summary>
private static readonly string[] HuggingFaceScanExtensions =
{
".py", ".env", ".toml", ".yaml", ".yml", ".json", ".txt", ".cfg", ".ini", ".conf", ".ipynb"
};
private const int HuggingFaceMaxFilesPerRepo = 50;
private const int HuggingFaceMaxFileSizeBytes = 500_000; // 500 KB
private const int HuggingFaceRequestDelayMs = 300;
/// <summary>
/// Scannt Hugging Face Spaces und Models nach geleakten API-Keys.
/// Sucht nach relevanten Repos, lädt Dateien herunter und prüft mit allen Key-Patterns.
/// </summary>
private async Task<List<(string Key, string Url)>> ScanHuggingFace(CancellationToken ct)
{
var allKeys = new Dictionary<string, string>(StringComparer.Ordinal);
var client = _httpFactory.CreateClient();
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
var hfToken = _settings.Get("HuggingFaceToken");
if (!string.IsNullOrWhiteSpace(hfToken))
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", hfToken);
var scannedRepos = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var totalQueries = HuggingFaceSearchTerms.Length * 2; // spaces + models
var queryIdx = 0;
// Phase 1: Spaces durchsuchen
foreach (var term in HuggingFaceSearchTerms)
{
ct.ThrowIfCancellationRequested();
queryIdx++;
SetProgress("HuggingFace Spaces", term, queryIdx, totalQueries);
var repos = await HuggingFaceSearchRepos(client, "spaces", term, ct);
foreach (var repoId in repos)
{
if (!scannedRepos.Add(repoId)) continue;
var keys = await HuggingFaceScanRepo(client, "spaces", repoId, ct);
foreach (var (key, url) in keys)
if (!allKeys.ContainsKey(key)) allKeys[key] = url;
}
}
// Phase 2: Models durchsuchen
foreach (var term in HuggingFaceSearchTerms)
{
ct.ThrowIfCancellationRequested();
queryIdx++;
SetProgress("HuggingFace Models", term, queryIdx, totalQueries);
var repos = await HuggingFaceSearchRepos(client, "models", term, ct);
foreach (var repoId in repos)
{
if (!scannedRepos.Add(repoId)) continue;
var keys = await HuggingFaceScanRepo(client, "models", repoId, ct);
foreach (var (key, url) in keys)
if (!allKeys.ContainsKey(key)) allKeys[key] = url;
}
}
AddLog("scan", $"HuggingFace: {scannedRepos.Count} Repos gescannt, {allKeys.Count} Key(s) gefunden");
return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
}
/// <summary>Sucht Repos (Spaces oder Models) auf HuggingFace.</summary>
private async Task<List<string>> HuggingFaceSearchRepos(HttpClient client, string repoType, string searchTerm, CancellationToken ct)
{
var repos = new List<string>();
try
{
await Task.Delay(HuggingFaceRequestDelayMs, ct);
var url = $"https://huggingface.co/api/{repoType}?search={Uri.EscapeDataString(searchTerm)}&limit=100";
var resp = await client.GetAsync(url, ct);
if (!resp.IsSuccessStatusCode) return repos;
var json = await resp.Content.ReadAsStringAsync(ct);
var doc = JsonDocument.Parse(json);
foreach (var item in doc.RootElement.EnumerateArray())
{
if (item.TryGetProperty("id", out var id))
repos.Add(id.GetString());
}
}
catch (Exception ex)
{
AddLog("error", $"HuggingFace-Suche fehlgeschlagen ({repoType}/{searchTerm}): {ex.Message}");
}
return repos;
}
/// <summary>Scannt ein einzelnes HuggingFace-Repo nach API-Keys.</summary>
private async Task<List<(string Key, string Url)>> HuggingFaceScanRepo(HttpClient client, string repoType, string repoId, CancellationToken ct)
{
var keys = new List<(string Key, string Url)>();
try
{
// Dateiliste abrufen
await Task.Delay(HuggingFaceRequestDelayMs, ct);
var treeUrl = $"https://huggingface.co/api/{repoType}/{repoId}/tree/main?recursive=true";
var treeResp = await client.GetAsync(treeUrl, ct);
if (!treeResp.IsSuccessStatusCode) return keys;
var treeJson = await treeResp.Content.ReadAsStringAsync(ct);
var treeDoc = JsonDocument.Parse(treeJson);
// Relevante Dateien filtern
var filesToScan = new List<(string Path, long Size)>();
foreach (var file in treeDoc.RootElement.EnumerateArray())
{
if (!file.TryGetProperty("type", out var type) || type.GetString() != "file") continue;
if (!file.TryGetProperty("path", out var path)) continue;
var filePath = path.GetString();
var fileSize = file.TryGetProperty("size", out var sz) ? sz.GetInt64() : 0;
if (fileSize > HuggingFaceMaxFileSizeBytes) continue;
// Prüfe ob die Datei in der Prioritätsliste ist oder eine relevante Extension hat
var isPriority = HuggingFaceScanFiles.Any(f => filePath.EndsWith(f, StringComparison.OrdinalIgnoreCase)
|| filePath.Equals(f, StringComparison.OrdinalIgnoreCase));
var hasRelevantExt = HuggingFaceScanExtensions.Any(ext =>
filePath.EndsWith(ext, StringComparison.OrdinalIgnoreCase));
if (isPriority || hasRelevantExt)
filesToScan.Add((filePath, fileSize));
}
// Max Dateien pro Repo begrenzen
if (filesToScan.Count > HuggingFaceMaxFilesPerRepo)
filesToScan = filesToScan.Take(HuggingFaceMaxFilesPerRepo).ToList();
// Dateien herunterladen und scannen
// repoType "spaces" → URL-Prefix "spaces", "models" → kein Prefix (nur repoId)
var urlPrefix = repoType == "spaces" ? "spaces" : repoType == "models" ? "" : repoType;
foreach (var (filePath, _) in filesToScan)
{
ct.ThrowIfCancellationRequested();
try
{
await Task.Delay(HuggingFaceRequestDelayMs, ct);
var rawUrl = urlPrefix == ""
? $"https://huggingface.co/{repoId}/resolve/main/{filePath}"
: $"https://huggingface.co/{urlPrefix}/{repoId}/resolve/main/{filePath}";
var request = new HttpRequestMessage(HttpMethod.Get, rawUrl);
request.Headers.Add("Accept", "text/plain");
var resp = await client.SendAsync(request, ct);
if (!resp.IsSuccessStatusCode) continue;
var content = await resp.Content.ReadAsStringAsync(ct);
if (string.IsNullOrWhiteSpace(content)) continue;
// Alle Key-Patterns prüfen
foreach (var regex in KeyPatterns.Compiled)
{
var matches = regex.Matches(content);
foreach (Match m in matches)
{
var sourceUrl = urlPrefix == ""
? $"https://huggingface.co/{repoId}/blob/main/{filePath}"
: $"https://huggingface.co/{urlPrefix}/{repoId}/blob/main/{filePath}";
keys.Add((m.Value, sourceUrl));
}
}
}
catch { /* Einzelne Datei fehlgeschlagen – ignorieren */ }
}
}
catch (Exception ex)
{
AddLog("error", $"HuggingFace-Repo-Scan fehlgeschlagen ({repoId}): {ex.Message}");
}
return keys;
}
// ── Key Check ──
public async Task CheckAndUpdateKey(ScannedKeyRecord sk)
Services/ServerSettingsService.cs
+1
View File
@@ -35,6 +35,7 @@ public class ServerSettingsService
new SettingDefinition( "SerperApiKey", "string", "", "Serper.dev API-Key für Google-Suche (Key-Scanner + Tool)", "password" ),
new SettingDefinition( "ScanIntervalHours", "int", "0", "Key-Scanner Intervall in Stunden (0 = nur manuell)", "number" ),
new SettingDefinition( "GitHubEventScanner", "bool", "false", "GitHub Event Scanner (Echtzeit-Monitoring öffentlicher Pushes)", "toggle" ),
new SettingDefinition( "HuggingFaceToken", "string", "", "HuggingFace Token (optional, erhöht Rate-Limits)", "password" ),
new SettingDefinition( "ScannerKeyTypes", "string_array", "anthropic_oauth,krea_ai", "Key-Scanner: Aktive Key-Typen", "checkboxes" ),
};