feat: GitHub Issues & PR Comments Scanner
- Neue ScanGitHubIssues()-Methode durchsucht Issues, PRs und Kommentare - Quick-Mode: Search API (nur Body) - Full-Mode: Search API (Body + Comments) + Kommentare der Treffer abrufen - Rate-Limiting: 2.5s (Search), 1s (Comments), 60s Pause bei 403 - Max 100 Issues mit Kommentaren im Full-Mode - Integration in RunScan() zwischen GitHub Code Search und Serper
This commit is contained in:
Meleki committed
1 parent
79d7a88c5c
commit
808c539e86
1 file changed
+165
@@ -220,6 +220,15 @@ public class KeyScannerService : BackgroundService
|
|||||||
AddLog("scan", "GitHub-Scan (" + modeLabel + "): " + ghKeys.Count + " Key(s) gefunden");
|
AddLog("scan", "GitHub-Scan (" + modeLabel + "): " + ghKeys.Count + " Key(s) gefunden");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GitHub Issues & PR Comments (benötigt GitHub Token)
|
||||||
|
if (mode != "web" && !string.IsNullOrWhiteSpace(_githubToken))
|
||||||
|
{
|
||||||
|
SetProgress("GitHub Issues & Comments", "Starte…");
|
||||||
|
var issueKeys = await ScanGitHubIssues(mode == "quick", ct);
|
||||||
|
allFoundKeys.AddRange(issueKeys.Select(k => (k.Key, k.Url, "github_issues")));
|
||||||
|
AddLog("scan", "GitHub Issues (" + modeLabel + "): " + issueKeys.Count + " Key(s) gefunden");
|
||||||
|
}
|
||||||
|
|
||||||
if (mode != "quick" && !string.IsNullOrWhiteSpace(_serperApiKey))
|
if (mode != "quick" && !string.IsNullOrWhiteSpace(_serperApiKey))
|
||||||
{
|
{
|
||||||
SetProgress("Web-Suche (Serper)", "Starte…");
|
SetProgress("Web-Suche (Serper)", "Starte…");
|
||||||
@@ -536,6 +545,162 @@ public class KeyScannerService : BackgroundService
|
|||||||
|
|
||||||
// ── Serper.dev Google Search ──
|
// ── Serper.dev Google Search ──
|
||||||
|
|
||||||
|
// ── GitHub Issues & PR Comments Scanner ──
|
||||||
|
|
||||||
|
private const int IssueSearchDelayMs = 2500; // 30 req/min = 1 req/2s + buffer
|
||||||
|
private const int CommentFetchDelayMs = 1000; // 5000 req/h = locker
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Durchsucht GitHub Issues, PR Bodies und Kommentare nach geleakten API-Keys.
|
||||||
|
/// Quick-Mode: Nur Search API (Issue/PR body + comment search).
|
||||||
|
/// Full-Mode: Search API + Kommentare der Treffer abrufen und scannen.
|
||||||
|
/// </summary>
|
||||||
|
private async Task<List<(string Key, string Url)>> ScanGitHubIssues(bool quickMode, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var allKeys = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||||
|
var client = _httpFactory.CreateClient();
|
||||||
|
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GottKeyScanner", "1.0"));
|
||||||
|
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
|
||||||
|
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _githubToken);
|
||||||
|
client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
|
||||||
|
|
||||||
|
var activePatterns = GetActiveSearchPatterns();
|
||||||
|
// Queries: jeder SearchTerm × (in:body, in:comments)
|
||||||
|
var searchScopes = quickMode
|
||||||
|
? new[] { "in:body" } // Quick: nur Body
|
||||||
|
: new[] { "in:body", "in:comments" }; // Full: Body + Comments
|
||||||
|
|
||||||
|
var queries = new List<(string Query, Regex Regex)>();
|
||||||
|
foreach (var (pattern, searchTerm, _) in activePatterns)
|
||||||
|
{
|
||||||
|
var regex = new Regex(pattern, RegexOptions.Compiled);
|
||||||
|
foreach (var scope in searchScopes)
|
||||||
|
queries.Add(($"\"{searchTerm}\" {scope}", regex));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 1: Issue Search API
|
||||||
|
var issueUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase); // Deduplizierung
|
||||||
|
var allRegexes = activePatterns.Select(p => new Regex(p.Pattern, RegexOptions.Compiled)).ToArray();
|
||||||
|
var issuesWithComments = new List<(string Owner, string Repo, int Number)>();
|
||||||
|
|
||||||
|
for (int i = 0; i < queries.Count; i++)
|
||||||
|
{
|
||||||
|
ct.ThrowIfCancellationRequested();
|
||||||
|
var (query, regex) = queries[i];
|
||||||
|
SetProgress("GitHub Issues Search", query, i + 1, queries.Count);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(IssueSearchDelayMs, ct);
|
||||||
|
var url = $"https://api.github.com/search/issues?q={Uri.EscapeDataString(query)}&sort=created&order=desc&per_page=100";
|
||||||
|
var resp = await client.GetAsync(url, ct);
|
||||||
|
|
||||||
|
if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
||||||
|
{
|
||||||
|
SetProgress("GitHub Issues Search", "⏳ Rate-Limit, warte 60s…", i + 1, queries.Count);
|
||||||
|
AddLog("rate_limit", "GitHub Issues Rate-Limit, warte 60s");
|
||||||
|
await Task.Delay(60_000, ct);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (resp.StatusCode == System.Net.HttpStatusCode.UnprocessableEntity) continue;
|
||||||
|
resp.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
|
var json = await resp.Content.ReadAsStringAsync(ct);
|
||||||
|
var doc = JsonDocument.Parse(json);
|
||||||
|
|
||||||
|
if (!doc.RootElement.TryGetProperty("items", out var items)) continue;
|
||||||
|
|
||||||
|
foreach (var item in items.EnumerateArray())
|
||||||
|
{
|
||||||
|
var htmlUrl = item.TryGetProperty("html_url", out var hu) ? hu.GetString() ?? "" : "";
|
||||||
|
var body = item.TryGetProperty("body", out var b) ? b.GetString() ?? "" : "";
|
||||||
|
|
||||||
|
// Keys aus Issue/PR Body extrahieren
|
||||||
|
var matches = regex.Matches(body);
|
||||||
|
foreach (Match m in matches)
|
||||||
|
if (!allKeys.ContainsKey(m.Value)) allKeys[m.Value] = htmlUrl;
|
||||||
|
|
||||||
|
// Für Full-Mode: Issue merken um Kommentare abzurufen
|
||||||
|
if (!quickMode && !string.IsNullOrEmpty(htmlUrl) && issueUrls.Add(htmlUrl))
|
||||||
|
{
|
||||||
|
// Owner/Repo/Number aus URL parsen: https://github.com/{owner}/{repo}/issues/{number}
|
||||||
|
var parts = htmlUrl.Split('/');
|
||||||
|
if (parts.Length >= 7 && int.TryParse(parts[^1], out var number))
|
||||||
|
{
|
||||||
|
var owner = parts[^4];
|
||||||
|
var repo = parts[^3];
|
||||||
|
issuesWithComments.Add((owner, repo, number));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
||||||
|
{
|
||||||
|
AddLog("rate_limit", "GitHub Issues Rate-Limit: " + ex.Message);
|
||||||
|
await Task.Delay(60_000, ct);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
AddLog("error", "GitHub Issues-Suche fehlgeschlagen: " + ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 2 (nur Full-Mode): Kommentare der Treffer abrufen
|
||||||
|
if (!quickMode && issuesWithComments.Count > 0)
|
||||||
|
{
|
||||||
|
// Max 100 Issues abrufen (Rate-Limit schonen)
|
||||||
|
var issuesToFetch = issuesWithComments.Take(100).ToList();
|
||||||
|
for (int i = 0; i < issuesToFetch.Count; i++)
|
||||||
|
{
|
||||||
|
ct.ThrowIfCancellationRequested();
|
||||||
|
var (owner, repo, number) = issuesToFetch[i];
|
||||||
|
SetProgress("GitHub Issue Comments", $"{owner}/{repo}#{number}", i + 1, issuesToFetch.Count);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(CommentFetchDelayMs, ct);
|
||||||
|
var commentsUrl = $"https://api.github.com/repos/{owner}/{repo}/issues/{number}/comments?per_page=100";
|
||||||
|
var resp = await client.GetAsync(commentsUrl, ct);
|
||||||
|
|
||||||
|
if (resp.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
||||||
|
{
|
||||||
|
AddLog("rate_limit", "GitHub Comments Rate-Limit, warte 60s");
|
||||||
|
await Task.Delay(60_000, ct);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!resp.IsSuccessStatusCode) continue;
|
||||||
|
|
||||||
|
var json = await resp.Content.ReadAsStringAsync(ct);
|
||||||
|
var doc = JsonDocument.Parse(json);
|
||||||
|
|
||||||
|
foreach (var comment in doc.RootElement.EnumerateArray())
|
||||||
|
{
|
||||||
|
var commentBody = comment.TryGetProperty("body", out var cb) ? cb.GetString() ?? "" : "";
|
||||||
|
var commentUrl = comment.TryGetProperty("html_url", out var cu) ? cu.GetString() ?? "" : "";
|
||||||
|
if (string.IsNullOrEmpty(commentUrl))
|
||||||
|
commentUrl = $"https://github.com/{owner}/{repo}/issues/{number}";
|
||||||
|
|
||||||
|
foreach (var regex in allRegexes)
|
||||||
|
{
|
||||||
|
var matches = regex.Matches(commentBody);
|
||||||
|
foreach (Match m in matches)
|
||||||
|
if (!allKeys.ContainsKey(m.Value)) allKeys[m.Value] = commentUrl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
AddLog("error", $"GitHub Comments fehlgeschlagen ({owner}/{repo}#{number}): {ex.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
AddLog("scan", $"GitHub Issues: {issueUrls.Count} Issues durchsucht, {allKeys.Count} Key(s) gefunden");
|
||||||
|
return allKeys.Select(kv => (kv.Key, kv.Value)).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Serper.dev Google Search ──
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Durchsucht Google via Serper.dev nach geleakten API-Keys.
|
/// Durchsucht Google via Serper.dev nach geleakten API-Keys.
|
||||||
/// Jeder KeyPattern-SearchTerm wird als exakte Phrase gesucht.
|
/// Jeder KeyPattern-SearchTerm wird als exakte Phrase gesucht.
|
||||||
|
|||||||
Reference in new issue
Block a user