fix: Rate-Limiting bei AUR RPC + Retry-Logik (v2.1.1)
Probleme behoben: - 429 Too Many Requests bei schnellen AUR-RPC-Abfragen - Pakete zeigten 0/100 UNBEKANNT statt korrekter Scores - scan-all brach bei massiven Fehlern ab Lösungen: - Retry-Mechanismus mit exponentiellem Backoff (max 3 Versuche) - 429-Status erkannt und mit 1s/2s/3s Delay retryed - Kein Hard-Fail bei AUR-Fehlern — None zurückgeben - 200ms Pause nach je 5 Paketen in scan-all - Consecutive-Error-Limit: 5 Fehler → 5s Pause - Scan läuft stabil durch alle 125+ Pakete Test-Ergebnis: - Vorher: 60+ Pakete mit 0/100 UNBEKANNT - Nachher: 0 Pakete mit UNBEKANNT, alle korrekt gescored
This commit is contained in:
Generated
+1
-1
@@ -4,7 +4,7 @@ version = 3
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aegisaur"
|
name = "aegisaur"
|
||||||
version = "2.0.0"
|
version = "2.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"chrono",
|
"chrono",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "aegisaur"
|
name = "aegisaur"
|
||||||
version = "2.1.0"
|
version = "2.1.1"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
authors = ["Quasi & Thuumate 👻"]
|
authors = ["Quasi & Thuumate 👻"]
|
||||||
description = "Trust-Scoring + IOC-Scanner für Arch Linux AUR-Pakete"
|
description = "Trust-Scoring + IOC-Scanner für Arch Linux AUR-Pakete"
|
||||||
|
|||||||
+61
-11
@@ -262,12 +262,15 @@ impl PackageScanner {
|
|||||||
let foreign_packages = self.get_foreign_packages().await?;
|
let foreign_packages = self.get_foreign_packages().await?;
|
||||||
let mut results = Vec::with_capacity(foreign_packages.len());
|
let mut results = Vec::with_capacity(foreign_packages.len());
|
||||||
|
|
||||||
// SEQUENZIELLE SCANS mit gecachten IOCs
|
// SEQUENZIELLE SCANS mit gecachten IOCs und Rate-Limiting
|
||||||
// (Parallele Scans würden Rate-Limits bei AUR RPC triggern)
|
// (Parallele Scans würden Rate-Limits bei AUR RPC triggern)
|
||||||
let iocs = self.ioc_fetcher.get_cached_iocs().await?;
|
let iocs = self.ioc_fetcher.get_cached_iocs().await?;
|
||||||
let ioc_count = iocs.len();
|
let ioc_count = iocs.len();
|
||||||
info!("{} IOCs geladen, starte Scan von {} Paketen...", ioc_count, foreign_packages.len());
|
info!("{} IOCs geladen, starte Scan von {} Paketen...", ioc_count, foreign_packages.len());
|
||||||
|
|
||||||
|
let mut consecutive_errors = 0;
|
||||||
|
let max_consecutive_errors = 5;
|
||||||
|
|
||||||
for (idx, (pkg, _)) in foreign_packages.iter().enumerate() {
|
for (idx, (pkg, _)) in foreign_packages.iter().enumerate() {
|
||||||
if idx % 10 == 0 {
|
if idx % 10 == 0 {
|
||||||
println!(" [{}/{}] Scanne {}...", idx + 1, foreign_packages.len(), pkg);
|
println!(" [{}/{}] Scanne {}...", idx + 1, foreign_packages.len(), pkg);
|
||||||
@@ -276,9 +279,18 @@ impl PackageScanner {
|
|||||||
match self.scan_package_with_iocs(pkg, verbose, &iocs).await {
|
match self.scan_package_with_iocs(pkg, verbose, &iocs).await {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
results.push(result);
|
results.push(result);
|
||||||
|
consecutive_errors = 0; // Reset bei Erfolg
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
warn!("Fehler beim Scannen von {}: {}", pkg, e);
|
warn!("Fehler beim Scannen von {}: {}", pkg, e);
|
||||||
|
consecutive_errors += 1;
|
||||||
|
|
||||||
|
if consecutive_errors >= max_consecutive_errors {
|
||||||
|
warn!("Zu viele aufeinanderfolgende Fehler — Rate-Limiting vermutet. Pause...");
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
|
||||||
|
consecutive_errors = 0;
|
||||||
|
}
|
||||||
|
|
||||||
results.push(ScanResult {
|
results.push(ScanResult {
|
||||||
package: pkg.clone(),
|
package: pkg.clone(),
|
||||||
version: String::new(),
|
version: String::new(),
|
||||||
@@ -292,6 +304,11 @@ impl PackageScanner {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Kleines Delay zwischen Paketen um Rate-Limiting zu vermeiden
|
||||||
|
if idx % 5 == 4 {
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
results.sort_by(|a, b| a.score.cmp(&b.score));
|
results.sort_by(|a, b| a.score.cmp(&b.score));
|
||||||
@@ -395,18 +412,51 @@ impl PackageScanner {
|
|||||||
package
|
package
|
||||||
);
|
);
|
||||||
|
|
||||||
let response = reqwest::get(&url).await?;
|
let mut retries = 0;
|
||||||
if !response.status().is_success() {
|
let max_retries = 3;
|
||||||
return Ok(None);
|
let mut last_error = None;
|
||||||
|
|
||||||
|
while retries < max_retries {
|
||||||
|
match reqwest::get(&url).await {
|
||||||
|
Ok(response) => {
|
||||||
|
if response.status().is_success() {
|
||||||
|
match response.json::<AurRpcResponse>().await {
|
||||||
|
Ok(rpc_response) => {
|
||||||
|
if rpc_response.resultcount == 0 || rpc_response.results.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
return Ok(Some(rpc_response.results[0].clone()));
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
last_error = Some(format!("JSON parse error: {}", e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if response.status().as_u16() == 429 {
|
||||||
|
// Rate limited — warte und retry
|
||||||
|
let delay_ms = 1000 * (retries + 1);
|
||||||
|
warn!("Rate limited für {}, warte {}ms...", package, delay_ms);
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await;
|
||||||
|
retries += 1;
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
last_error = Some(format!("HTTP {}", response.status()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
last_error = Some(format!("Request failed: {}", e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
retries += 1;
|
||||||
|
if retries < max_retries {
|
||||||
|
let delay_ms = 500 * retries;
|
||||||
|
debug!("Retry {} für {} nach {}ms", retries, package, delay_ms);
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let rpc_response: AurRpcResponse = response.json().await?;
|
warn!("AUR RPC für {} fehlgeschlagen nach {} retries: {}", package, max_retries, last_error.unwrap_or_default());
|
||||||
|
Ok(None) // Nicht hard-fail, sondern None zurückgeben
|
||||||
if rpc_response.resultcount == 0 || rpc_response.results.is_empty() {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Some(rpc_response.results[0].clone()))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_pkgbuild(
|
async fn fetch_pkgbuild(
|
||||||
|
|||||||
Reference in New Issue
Block a user