fix: Rate-Limiting bei AUR RPC + Retry-Logik (v2.1.1)
Probleme behoben: - 429 Too Many Requests bei schnellen AUR-RPC-Abfragen - Pakete zeigten 0/100 UNBEKANNT statt korrekter Scores - scan-all brach bei massiven Fehlern ab Lösungen: - Retry-Mechanismus mit exponentiellem Backoff (max 3 Versuche) - 429-Status erkannt und mit 1s/2s/3s Delay retryed - Kein Hard-Fail bei AUR-Fehlern — None zurückgeben - 200ms Pause nach je 5 Paketen in scan-all - Consecutive-Error-Limit: 5 Fehler → 5s Pause - Scan läuft stabil durch alle 125+ Pakete Test-Ergebnis: - Vorher: 60+ Pakete mit 0/100 UNBEKANNT - Nachher: 0 Pakete mit UNBEKANNT, alle korrekt gescored
This commit is contained in:
Generated
+1
-1
@@ -4,7 +4,7 @@ version = 3
|
||||
|
||||
[[package]]
|
||||
name = "aegisaur"
|
||||
version = "2.0.0"
|
||||
version = "2.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "aegisaur"
|
||||
version = "2.1.0"
|
||||
version = "2.1.1"
|
||||
edition = "2021"
|
||||
authors = ["Quasi & Thuumate 👻"]
|
||||
description = "Trust-Scoring + IOC-Scanner für Arch Linux AUR-Pakete"
|
||||
|
||||
+58
-8
@@ -262,12 +262,15 @@ impl PackageScanner {
|
||||
let foreign_packages = self.get_foreign_packages().await?;
|
||||
let mut results = Vec::with_capacity(foreign_packages.len());
|
||||
|
||||
// SEQUENZIELLE SCANS mit gecachten IOCs
|
||||
// SEQUENZIELLE SCANS mit gecachten IOCs und Rate-Limiting
|
||||
// (Parallele Scans würden Rate-Limits bei AUR RPC triggern)
|
||||
let iocs = self.ioc_fetcher.get_cached_iocs().await?;
|
||||
let ioc_count = iocs.len();
|
||||
info!("{} IOCs geladen, starte Scan von {} Paketen...", ioc_count, foreign_packages.len());
|
||||
|
||||
let mut consecutive_errors = 0;
|
||||
let max_consecutive_errors = 5;
|
||||
|
||||
for (idx, (pkg, _)) in foreign_packages.iter().enumerate() {
|
||||
if idx % 10 == 0 {
|
||||
println!(" [{}/{}] Scanne {}...", idx + 1, foreign_packages.len(), pkg);
|
||||
@@ -276,9 +279,18 @@ impl PackageScanner {
|
||||
match self.scan_package_with_iocs(pkg, verbose, &iocs).await {
|
||||
Ok(result) => {
|
||||
results.push(result);
|
||||
consecutive_errors = 0; // Reset bei Erfolg
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Fehler beim Scannen von {}: {}", pkg, e);
|
||||
consecutive_errors += 1;
|
||||
|
||||
if consecutive_errors >= max_consecutive_errors {
|
||||
warn!("Zu viele aufeinanderfolgende Fehler — Rate-Limiting vermutet. Pause...");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
|
||||
consecutive_errors = 0;
|
||||
}
|
||||
|
||||
results.push(ScanResult {
|
||||
package: pkg.clone(),
|
||||
version: String::new(),
|
||||
@@ -292,6 +304,11 @@ impl PackageScanner {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Kleines Delay zwischen Paketen um Rate-Limiting zu vermeiden
|
||||
if idx % 5 == 4 {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
|
||||
results.sort_by(|a, b| a.score.cmp(&b.score));
|
||||
@@ -395,18 +412,51 @@ impl PackageScanner {
|
||||
package
|
||||
);
|
||||
|
||||
let response = reqwest::get(&url).await?;
|
||||
if !response.status().is_success() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let rpc_response: AurRpcResponse = response.json().await?;
|
||||
let mut retries = 0;
|
||||
let max_retries = 3;
|
||||
let mut last_error = None;
|
||||
|
||||
while retries < max_retries {
|
||||
match reqwest::get(&url).await {
|
||||
Ok(response) => {
|
||||
if response.status().is_success() {
|
||||
match response.json::<AurRpcResponse>().await {
|
||||
Ok(rpc_response) => {
|
||||
if rpc_response.resultcount == 0 || rpc_response.results.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(Some(rpc_response.results[0].clone()));
|
||||
}
|
||||
Err(e) => {
|
||||
last_error = Some(format!("JSON parse error: {}", e));
|
||||
}
|
||||
}
|
||||
} else if response.status().as_u16() == 429 {
|
||||
// Rate limited — warte und retry
|
||||
let delay_ms = 1000 * (retries + 1);
|
||||
warn!("Rate limited für {}, warte {}ms...", package, delay_ms);
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await;
|
||||
retries += 1;
|
||||
continue;
|
||||
} else {
|
||||
last_error = Some(format!("HTTP {}", response.status()));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
last_error = Some(format!("Request failed: {}", e));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(rpc_response.results[0].clone()))
|
||||
retries += 1;
|
||||
if retries < max_retries {
|
||||
let delay_ms = 500 * retries;
|
||||
debug!("Retry {} für {} nach {}ms", retries, package, delay_ms);
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
warn!("AUR RPC für {} fehlgeschlagen nach {} retries: {}", package, max_retries, last_error.unwrap_or_default());
|
||||
Ok(None) // Nicht hard-fail, sondern None zurückgeben
|
||||
}
|
||||
|
||||
async fn fetch_pkgbuild(
|
||||
|
||||
Reference in New Issue
Block a user