Meleki
22d966b830
- ScanSerper() durchsucht Google via Serper.dev nach geleakten API-Keys - Exakte Phrase-Suche pro KeyPattern, bis zu 100 Ergebnisse pro Query - Keys aus Snippets und Titles via Regex extrahiert - Source-Feld (github/serper/gist/pastebin) in ScannedKeyRecord + DB-Migration - RunScan() kombiniert GitHub + Serper mit Deduplizierung - Service startet auch wenn nur SerperApiKey konfiguriert ist - Frontend: Source-Icons und verkürzte URLs für Nicht-GitHub-Quellen - Config: SerperApiKey in appsettings.json
475 lines
18 KiB
C#
475 lines
18 KiB
C#
using System;
|
||
using System.Collections.Generic;
|
||
using System.Linq;
|
||
using System.Net.Http;
|
||
using System.Net.Http.Headers;
|
||
using System.Text;
|
||
using System.Text.Json;
|
||
using System.Text.RegularExpressions;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
using Gott.Models;
|
||
using Microsoft.Extensions.Configuration;
|
||
using Microsoft.Extensions.Hosting;
|
||
using Microsoft.Extensions.Logging;
|
||
|
||
namespace Gott.Services;
|
||
|
||
public class KeyScannerService : BackgroundService
|
||
{
|
||
private readonly DatabaseService _db;
|
||
private readonly IHttpClientFactory _httpFactory;
|
||
private readonly ILogger<KeyScannerService> _logger;
|
||
private readonly string _githubToken;
|
||
private readonly string _serperApiKey;
|
||
private readonly int _intervalHours;
|
||
|
||
// Status-Tracking
|
||
private readonly object _lock = new();
|
||
private bool _isRunning;
|
||
private string _lastRun;
|
||
private string _lastResult;
|
||
private readonly ManualResetEventSlim _triggerNow = new(false);
|
||
|
||
public bool IsRunning { get { lock (_lock) return _isRunning; } }
|
||
public string LastRun { get { lock (_lock) return _lastRun; } }
|
||
public string LastResult { get { lock (_lock) return _lastResult; } }
|
||
|
||
// ── GitHub Search Config ──
|
||
|
||
private static readonly (string Pattern, string SearchTerm)[] KeyPatterns =
|
||
{
|
||
(@"sk-ant-oat01-[A-Za-z0-9_\-]{95}", "sk-ant-oat01"),
|
||
};
|
||
|
||
private static readonly string[] SearchLanguages =
|
||
{
|
||
"Dotenv", "Text", "JavaScript", "Python", "TypeScript",
|
||
"Dockerfile", "Markdown", "Jupyter Notebook", "Shell",
|
||
"Java", "Go", "C++", "PHP"
|
||
};
|
||
|
||
private static readonly string[] PathExtensions =
|
||
{
|
||
".env", ".envrc", ".prod", ".secret", ".private", ".key",
|
||
".xml", ".json", ".properties", ".sql", ".txt", ".log",
|
||
".yml", ".yaml", ".toml", ".ini", ".config", ".conf", ".cfg",
|
||
".tmp", ".backup", ".bak", ".enc",
|
||
};
|
||
|
||
private const int RateLimitDelayMs = 6_500;
|
||
private const int MaxResultsPerSearch = 1000;
|
||
private const int ResultsPerPage = 100;
|
||
|
||
public KeyScannerService(
|
||
DatabaseService db,
|
||
IHttpClientFactory httpFactory,
|
||
IConfiguration config,
|
||
ILogger<KeyScannerService> logger)
|
||
{
|
||
_db = db;
|
||
_httpFactory = httpFactory;
|
||
_logger = logger;
|
||
_githubToken = config["GitHubToken"] ?? "";
|
||
_serperApiKey = config["SerperApiKey"] ?? "";
|
||
_intervalHours = config.GetValue<int>("ScanIntervalHours", 0);
|
||
}
|
||
|
||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||
{
|
||
var hasGitHub = !string.IsNullOrWhiteSpace(_githubToken);
|
||
var hasSerper = !string.IsNullOrWhiteSpace(_serperApiKey);
|
||
|
||
if (!hasGitHub && !hasSerper)
|
||
{
|
||
_logger.LogInformation("KeyScannerService deaktiviert (kein GitHubToken/SerperApiKey konfiguriert)");
|
||
// Warte trotzdem auf manuelles Trigger-Signal
|
||
await WaitForTriggerOnly(stoppingToken);
|
||
return;
|
||
}
|
||
|
||
if (_intervalHours <= 0)
|
||
{
|
||
_logger.LogInformation("KeyScannerService: Kein Intervall konfiguriert, nur manueller Trigger aktiv");
|
||
await WaitForTriggerOnly(stoppingToken);
|
||
return;
|
||
}
|
||
|
||
_logger.LogInformation("KeyScannerService gestartet (Intervall: {Hours}h)", _intervalHours);
|
||
|
||
while (!stoppingToken.IsCancellationRequested)
|
||
{
|
||
await RunScan(stoppingToken);
|
||
|
||
// Warte auf Intervall oder manuelles Signal
|
||
var waitResult = await WaitWithTrigger(TimeSpan.FromHours(_intervalHours), stoppingToken);
|
||
if (waitResult) _logger.LogInformation("KeyScannerService: Manueller Scan angefordert");
|
||
}
|
||
}
|
||
|
||
/// <summary>Wartet nur auf manuelles Trigger-Signal (kein automatisches Intervall).</summary>
|
||
private async Task WaitForTriggerOnly(CancellationToken stoppingToken)
|
||
{
|
||
while (!stoppingToken.IsCancellationRequested)
|
||
{
|
||
await WaitWithTrigger(Timeout.InfiniteTimeSpan, stoppingToken);
|
||
if (!stoppingToken.IsCancellationRequested)
|
||
await RunScan(stoppingToken);
|
||
}
|
||
}
|
||
|
||
/// <summary>Wartet auf Timeout oder Trigger-Signal. Gibt true zurück wenn getriggert.</summary>
|
||
private Task<bool> WaitWithTrigger(TimeSpan timeout, CancellationToken ct)
|
||
{
|
||
return Task.Run(() =>
|
||
{
|
||
_triggerNow.Reset();
|
||
var waitHandles = new[] { _triggerNow.WaitHandle, ct.WaitHandle };
|
||
var idx = WaitHandle.WaitAny(waitHandles, timeout);
|
||
return idx == 0; // true = triggered
|
||
}, ct);
|
||
}
|
||
|
||
/// <summary>Wird vom Endpoint aufgerufen um einen sofortigen Scan auszulösen.</summary>
|
||
public void TriggerScan()
|
||
{
|
||
_triggerNow.Set();
|
||
}
|
||
|
||
private async Task RunScan(CancellationToken ct)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_isRunning) return;
|
||
_isRunning = true;
|
||
}
|
||
|
||
_logger.LogInformation("KeyScannerService: Scan gestartet");
|
||
var newKeysFound = 0;
|
||
var checkedCount = 0;
|
||
var totalFound = 0;
|
||
|
||
try
|
||
{
|
||
// Phase 1: Alle konfigurierten Quellen scannen
|
||
var allFoundKeys = new List<(string Key, string Url, string Source)>();
|
||
|
||
if (!string.IsNullOrWhiteSpace(_githubToken))
|
||
{
|
||
var ghKeys = await ScanGitHub(ct);
|
||
allFoundKeys.AddRange(ghKeys.Select(k => (k.Key, k.Url, "github")));
|
||
_logger.LogInformation("GitHub-Scan: {Count} Key(s) gefunden", ghKeys.Count);
|
||
}
|
||
|
||
if (!string.IsNullOrWhiteSpace(_serperApiKey))
|
||
{
|
||
var spKeys = await ScanSerper(ct);
|
||
allFoundKeys.AddRange(spKeys.Select(k => (k.Key, k.Url, "serper")));
|
||
_logger.LogInformation("Serper-Scan: {Count} Key(s) gefunden", spKeys.Count);
|
||
}
|
||
|
||
totalFound = allFoundKeys.Count;
|
||
|
||
// Phase 2: Neue Keys in DB speichern (dedupliziert)
|
||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||
foreach (var (key, sourceUrl, source) in allFoundKeys)
|
||
{
|
||
if (!seen.Add(key)) continue;
|
||
if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key))
|
||
continue;
|
||
|
||
newKeysFound++;
|
||
await _db.CreateScannedKey(new ScannedKeyRecord
|
||
{
|
||
ScannedKeyId = Guid.NewGuid().ToString("N"),
|
||
ApiKey = key,
|
||
Source = source,
|
||
SourceUrl = sourceUrl,
|
||
Status = "unchecked",
|
||
Plan = "Unknown",
|
||
ScannedAt = DateTime.UtcNow.ToString("o")
|
||
});
|
||
}
|
||
_logger.LogInformation("{Count} neue Key(s) gespeichert", newKeysFound);
|
||
|
||
// Phase 3: Alle unchecked Keys prüfen
|
||
var allScanned = (await _db.GetScannedKeys()).ToList();
|
||
var toCheck = allScanned.Where(k => k.Status == "unchecked").ToList();
|
||
foreach (var sk in toCheck)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
await CheckAndUpdateKey(sk);
|
||
checkedCount++;
|
||
await Task.Delay(500, ct); // Rate-Limit schonen
|
||
}
|
||
|
||
lock (_lock)
|
||
{
|
||
_lastRun = DateTime.UtcNow.ToString("o");
|
||
_lastResult = $"{totalFound} gefunden, {newKeysFound} neu, {checkedCount} geprüft";
|
||
}
|
||
|
||
_logger.LogInformation("KeyScannerService: Scan abgeschlossen – {Result}", _lastResult);
|
||
}
|
||
catch (OperationCanceledException) { /* shutdown */ }
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogError(ex, "KeyScannerService: Fehler beim Scan");
|
||
lock (_lock)
|
||
{
|
||
_lastRun = DateTime.UtcNow.ToString("o");
|
||
_lastResult = $"Fehler: {ex.Message}";
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
lock (_lock) _isRunning = false;
|
||
}
|
||
}
|
||
|
||
// ── GitHub Code Search (portiert aus ExampleScannerProject) ──
|
||
|
||
private async Task<List<(string Key, string Url)>> ScanGitHub(CancellationToken ct)
|
||
{
|
||
var allKeys = new Dictionary<string, string>(StringComparer.Ordinal); // Key → erste URL
|
||
var queries = BuildSearchQueries();
|
||
|
||
var client = _httpFactory.CreateClient();
|
||
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
|
||
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
|
||
client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
|
||
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.text-match+json"));
|
||
|
||
for (int i = 0; i < queries.Count; i++)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
var (query, regex) = queries[i];
|
||
|
||
try
|
||
{
|
||
var results = await SearchCodeAsync(client, query, regex, ct);
|
||
foreach (var (key, url) in results)
|
||
if (!allKeys.ContainsKey(key)) allKeys[key] = url;
|
||
}
|
||
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
||
{
|
||
_logger.LogWarning("GitHub Rate-Limit erreicht, warte 60s ...");
|
||
await Task.Delay(60_000, ct);
|
||
try
|
||
{
|
||
var results = await SearchCodeAsync(client, query, regex, ct);
|
||
foreach (var (key, url) in results)
|
||
if (!allKeys.ContainsKey(key)) allKeys[key] = url;
|
||
}
|
||
catch (Exception retryEx)
|
||
{
|
||
_logger.LogWarning("Retry fehlgeschlagen: {Msg}", retryEx.Message);
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogWarning("GitHub-Suche fehlgeschlagen: {Msg}", ex.Message);
|
||
}
|
||
}
|
||
|
||
return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
|
||
}
|
||
|
||
private static async Task<List<(string Key, string Url)>> SearchCodeAsync(HttpClient client, string query, Regex regex, CancellationToken ct)
|
||
{
|
||
var keys = new List<(string Key, string Url)>();
|
||
int page = 1;
|
||
int maxPages = MaxResultsPerSearch / ResultsPerPage;
|
||
|
||
while (page <= maxPages)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
var url = $"https://api.github.com/search/code?q={Uri.EscapeDataString(query)}&per_page={ResultsPerPage}&page={page}";
|
||
await Task.Delay(RateLimitDelayMs, ct);
|
||
|
||
var resp = await client.GetAsync(url, ct);
|
||
if (resp.StatusCode == System.Net.HttpStatusCode.UnprocessableEntity) break;
|
||
resp.EnsureSuccessStatusCode();
|
||
|
||
var json = await resp.Content.ReadAsStringAsync(ct);
|
||
var doc = JsonDocument.Parse(json);
|
||
var root = doc.RootElement;
|
||
var totalCount = root.GetProperty("total_count").GetInt32();
|
||
|
||
if (!root.TryGetProperty("items", out var items)) break;
|
||
|
||
foreach (var item in items.EnumerateArray())
|
||
{
|
||
var htmlUrl = BuildHtmlUrl(item);
|
||
|
||
if (item.TryGetProperty("text_matches", out var textMatches))
|
||
{
|
||
foreach (var tm in textMatches.EnumerateArray())
|
||
{
|
||
if (tm.TryGetProperty("fragment", out var frag))
|
||
{
|
||
var matches = regex.Matches(frag.GetString() ?? "");
|
||
foreach (Match m in matches) keys.Add((m.Value, htmlUrl));
|
||
}
|
||
}
|
||
}
|
||
|
||
// Kein text_match → Raw-File fetchen
|
||
var hasMatches = item.TryGetProperty("text_matches", out var tm2) && tm2.EnumerateArray().Any();
|
||
if (!hasMatches)
|
||
{
|
||
var rawKeys = await TryFetchRawFileKeysAsync(client, item, regex, ct);
|
||
foreach (var k in rawKeys) keys.Add((k, htmlUrl));
|
||
}
|
||
}
|
||
|
||
if (items.GetArrayLength() < ResultsPerPage || page * ResultsPerPage >= totalCount) break;
|
||
page++;
|
||
}
|
||
|
||
return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList();
|
||
}
|
||
|
||
/// <summary>Baut die GitHub HTML-URL aus einem Search-API-Item.</summary>
|
||
private static string BuildHtmlUrl(JsonElement item)
|
||
{
|
||
if (item.TryGetProperty("html_url", out var htmlUrl))
|
||
return htmlUrl.GetString() ?? "";
|
||
|
||
// Fallback: aus repository.full_name + path zusammenbauen
|
||
if (item.TryGetProperty("repository", out var repo) &&
|
||
repo.TryGetProperty("full_name", out var fullName) &&
|
||
item.TryGetProperty("path", out var path))
|
||
return $"https://github.com/{fullName.GetString()}/blob/HEAD/{path.GetString()}";
|
||
|
||
return "";
|
||
}
|
||
|
||
private static async Task<List<string>> TryFetchRawFileKeysAsync(HttpClient client, JsonElement item, Regex regex, CancellationToken ct)
|
||
{
|
||
var keys = new List<string>();
|
||
try
|
||
{
|
||
if (!item.TryGetProperty("repository", out var repo) ||
|
||
!repo.TryGetProperty("full_name", out var fullName) ||
|
||
!item.TryGetProperty("path", out var path))
|
||
return keys;
|
||
|
||
var contentsUrl = $"https://api.github.com/repos/{fullName.GetString()}/contents/{path.GetString()}";
|
||
var request = new HttpRequestMessage(HttpMethod.Get, contentsUrl);
|
||
request.Headers.Accept.Clear();
|
||
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.raw+json"));
|
||
|
||
await Task.Delay(500, ct);
|
||
var resp = await client.SendAsync(request, ct);
|
||
if (!resp.IsSuccessStatusCode) return keys;
|
||
|
||
var content = await resp.Content.ReadAsStringAsync(ct);
|
||
var matches = regex.Matches(content);
|
||
foreach (Match m in matches) keys.Add(m.Value);
|
||
}
|
||
catch { /* Datei nicht lesbar – ignorieren */ }
|
||
return keys;
|
||
}
|
||
|
||
private static List<(string Query, Regex Regex)> BuildSearchQueries()
|
||
{
|
||
var queries = new List<(string, Regex)>();
|
||
foreach (var (pattern, searchTerm) in KeyPatterns)
|
||
{
|
||
var regex = new Regex(pattern, RegexOptions.Compiled);
|
||
foreach (var lang in SearchLanguages)
|
||
{
|
||
var langFilter = lang.Contains(' ') ? $"language:\"{lang}\"" : $"language:{lang}";
|
||
queries.Add(($"{searchTerm} {langFilter}", regex));
|
||
}
|
||
|
||
foreach (var pathExt in PathExtensions)
|
||
{
|
||
queries.Add(($"{searchTerm} path:{pathExt}", regex));
|
||
}
|
||
}
|
||
return queries;
|
||
}
|
||
|
||
// ── Serper.dev Google Search ──
|
||
|
||
/// <summary>
|
||
/// Durchsucht Google via Serper.dev nach geleakten API-Keys.
|
||
/// Jeder KeyPattern-SearchTerm wird als exakte Phrase gesucht.
|
||
/// Snippets werden nach dem vollständigen Key-Pattern geparst.
|
||
/// </summary>
|
||
private async Task<List<(string Key, string Url)>> ScanSerper(CancellationToken ct)
|
||
{
|
||
var allKeys = new Dictionary<string, string>(StringComparer.Ordinal);
|
||
var client = _httpFactory.CreateClient();
|
||
client.DefaultRequestHeaders.Add("X-API-KEY", _serperApiKey);
|
||
|
||
foreach (var (pattern, searchTerm) in KeyPatterns)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
var regex = new Regex(pattern, RegexOptions.Compiled);
|
||
|
||
try
|
||
{
|
||
// Serper erlaubt bis zu 100 Ergebnisse pro Request (num: 100)
|
||
var payload = JsonSerializer.Serialize(new { q = $"\"{searchTerm}\"", num = 100 });
|
||
var request = new HttpRequestMessage(HttpMethod.Post, "https://google.serper.dev/search")
|
||
{
|
||
Content = new StringContent(payload, Encoding.UTF8, "application/json")
|
||
};
|
||
|
||
var resp = await client.SendAsync(request, ct);
|
||
resp.EnsureSuccessStatusCode();
|
||
|
||
var json = await resp.Content.ReadAsStringAsync(ct);
|
||
var doc = JsonDocument.Parse(json);
|
||
|
||
// Organic results durchsuchen
|
||
if (doc.RootElement.TryGetProperty("organic", out var organic))
|
||
{
|
||
foreach (var item in organic.EnumerateArray())
|
||
{
|
||
var link = item.TryGetProperty("link", out var l) ? l.GetString() ?? "" : "";
|
||
var snippet = item.TryGetProperty("snippet", out var s) ? s.GetString() ?? "" : "";
|
||
var title = item.TryGetProperty("title", out var t) ? t.GetString() ?? "" : "";
|
||
|
||
// Keys aus Snippet und Title extrahieren
|
||
var text = snippet + " " + title;
|
||
var matches = regex.Matches(text);
|
||
foreach (Match m in matches)
|
||
if (!allKeys.ContainsKey(m.Value)) allKeys[m.Value] = link;
|
||
}
|
||
}
|
||
|
||
_logger.LogInformation("Serper-Suche für '{Term}': {Count} Ergebnis(se)", searchTerm, allKeys.Count);
|
||
await Task.Delay(1000, ct); // Rate-Limit schonen
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogWarning("Serper-Suche fehlgeschlagen für '{Term}': {Msg}", searchTerm, ex.Message);
|
||
}
|
||
}
|
||
|
||
return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
|
||
}
|
||
|
||
// ── Anthropic Key Check (via shared AnthropicKeyChecker) ──
|
||
|
||
public async Task CheckAndUpdateKey(ScannedKeyRecord sk)
|
||
{
|
||
try
|
||
{
|
||
var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
|
||
var status = (result.Status == "ok" || result.Status == "usage_limited") ? "valid" : result.Status;
|
||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Plan, result.Usage7d, result.Usage5h);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogWarning("Key-Check fehlgeschlagen für {Id}: {Msg}", sk.ScannedKeyId, ex.Message);
|
||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null);
|
||
}
|
||
}
|
||
}
|