fix: Rate-Limiting bei AUR RPC + Retry-Logik (v2.1.1)

Probleme behoben:
- 429 Too Many Requests bei schnellen AUR-RPC-Abfragen
- Pakete zeigten 0/100 UNBEKANNT statt korrekter Scores
- scan-all brach bei massiven Fehlern ab

Lösungen:
- Retry-Mechanismus mit exponentiellem Backoff (max 3 Versuche)
- 429-Status erkannt und mit 1s/2s/3s Delay retryed
- Kein Hard-Fail bei AUR-Fehlern — None zurückgeben
- 200ms Pause nach je 5 Paketen in scan-all
- Consecutive-Error-Limit: 5 Fehler → 5s Pause
- Scan läuft stabil durch alle 125+ Pakete

Test-Ergebnis:
- Vorher: 60+ Pakete mit 0/100 UNBEKANNT
- Nachher: 0 Pakete mit UNBEKANNT, alle korrekt gescored
This commit is contained in:
Thuumate 👻
2026-06-15 20:04:18 +02:00
parent 556e698151
commit 61fad87f23
3 changed files with 63 additions and 13 deletions
Generated
+1 -1
View File
@@ -4,7 +4,7 @@ version = 3
[[package]]
name = "aegisaur"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"anyhow",
"chrono",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "aegisaur"
version = "2.1.0"
version = "2.1.1"
edition = "2021"
authors = ["Quasi & Thuumate 👻"]
description = "Trust-Scoring + IOC-Scanner für Arch Linux AUR-Pakete"
+61 -11
View File
@@ -262,12 +262,15 @@ impl PackageScanner {
let foreign_packages = self.get_foreign_packages().await?;
let mut results = Vec::with_capacity(foreign_packages.len());
// SEQUENZIELLE SCANS mit gecachten IOCs
// SEQUENZIELLE SCANS mit gecachten IOCs und Rate-Limiting
// (Parallele Scans würden Rate-Limits bei AUR RPC triggern)
let iocs = self.ioc_fetcher.get_cached_iocs().await?;
let ioc_count = iocs.len();
info!("{} IOCs geladen, starte Scan von {} Paketen...", ioc_count, foreign_packages.len());
let mut consecutive_errors = 0;
let max_consecutive_errors = 5;
for (idx, (pkg, _)) in foreign_packages.iter().enumerate() {
if idx % 10 == 0 {
println!(" [{}/{}] Scanne {}...", idx + 1, foreign_packages.len(), pkg);
@@ -276,9 +279,18 @@ impl PackageScanner {
match self.scan_package_with_iocs(pkg, verbose, &iocs).await {
Ok(result) => {
results.push(result);
consecutive_errors = 0; // Reset bei Erfolg
}
Err(e) => {
warn!("Fehler beim Scannen von {}: {}", pkg, e);
consecutive_errors += 1;
if consecutive_errors >= max_consecutive_errors {
warn!("Zu viele aufeinanderfolgende Fehler — Rate-Limiting vermutet. Pause...");
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
consecutive_errors = 0;
}
results.push(ScanResult {
package: pkg.clone(),
version: String::new(),
@@ -292,6 +304,11 @@ impl PackageScanner {
});
}
}
// Kleines Delay zwischen Paketen um Rate-Limiting zu vermeiden
if idx % 5 == 4 {
tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;
}
}
results.sort_by(|a, b| a.score.cmp(&b.score));
@@ -395,18 +412,51 @@ impl PackageScanner {
package
);
let response = reqwest::get(&url).await?;
if !response.status().is_success() {
return Ok(None);
let mut retries = 0;
let max_retries = 3;
let mut last_error = None;
while retries < max_retries {
match reqwest::get(&url).await {
Ok(response) => {
if response.status().is_success() {
match response.json::<AurRpcResponse>().await {
Ok(rpc_response) => {
if rpc_response.resultcount == 0 || rpc_response.results.is_empty() {
return Ok(None);
}
return Ok(Some(rpc_response.results[0].clone()));
}
Err(e) => {
last_error = Some(format!("JSON parse error: {}", e));
}
}
} else if response.status().as_u16() == 429 {
// Rate limited — warte und retry
let delay_ms = 1000 * (retries + 1);
warn!("Rate limited für {}, warte {}ms...", package, delay_ms);
tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await;
retries += 1;
continue;
} else {
last_error = Some(format!("HTTP {}", response.status()));
}
}
Err(e) => {
last_error = Some(format!("Request failed: {}", e));
}
}
retries += 1;
if retries < max_retries {
let delay_ms = 500 * retries;
debug!("Retry {} für {} nach {}ms", retries, package, delay_ms);
tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await;
}
}
let rpc_response: AurRpcResponse = response.json().await?;
if rpc_response.resultcount == 0 || rpc_response.results.is_empty() {
return Ok(None);
}
Ok(Some(rpc_response.results[0].clone()))
warn!("AUR RPC für {} fehlgeschlagen nach {} retries: {}", package, max_retries, last_error.unwrap_or_default());
Ok(None) // Nicht hard-fail, sondern None zurückgeben
}
async fn fetch_pkgbuild(