diff --git a/Cargo.lock b/Cargo.lock index 5b4d464..2287c5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,7 +4,7 @@ version = 3 [[package]] name = "aegisaur" -version = "2.0.0" +version = "2.1.0" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 3773807..12a26c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aegisaur" -version = "2.1.0" +version = "2.1.1" edition = "2021" authors = ["Quasi & Thuumate 👻"] description = "Trust-Scoring + IOC-Scanner für Arch Linux AUR-Pakete" diff --git a/src/scanner.rs b/src/scanner.rs index 71040bf..90991c4 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -262,12 +262,15 @@ impl PackageScanner { let foreign_packages = self.get_foreign_packages().await?; let mut results = Vec::with_capacity(foreign_packages.len()); - // SEQUENZIELLE SCANS mit gecachten IOCs + // SEQUENZIELLE SCANS mit gecachten IOCs und Rate-Limiting // (Parallele Scans würden Rate-Limits bei AUR RPC triggern) let iocs = self.ioc_fetcher.get_cached_iocs().await?; let ioc_count = iocs.len(); info!("{} IOCs geladen, starte Scan von {} Paketen...", ioc_count, foreign_packages.len()); + let mut consecutive_errors = 0; + let max_consecutive_errors = 5; + for (idx, (pkg, _)) in foreign_packages.iter().enumerate() { if idx % 10 == 0 { println!(" [{}/{}] Scanne {}...", idx + 1, foreign_packages.len(), pkg); @@ -276,9 +279,18 @@ impl PackageScanner { match self.scan_package_with_iocs(pkg, verbose, &iocs).await { Ok(result) => { results.push(result); + consecutive_errors = 0; // Reset bei Erfolg } Err(e) => { warn!("Fehler beim Scannen von {}: {}", pkg, e); + consecutive_errors += 1; + + if consecutive_errors >= max_consecutive_errors { + warn!("Zu viele aufeinanderfolgende Fehler — Rate-Limiting vermutet. Pause..."); + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + consecutive_errors = 0; + } + results.push(ScanResult { package: pkg.clone(), version: String::new(), @@ -292,6 +304,11 @@ impl PackageScanner { }); } } + + // Kleines Delay zwischen Paketen um Rate-Limiting zu vermeiden + if idx % 5 == 4 { + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; + } } results.sort_by(|a, b| a.score.cmp(&b.score)); @@ -395,18 +412,51 @@ impl PackageScanner { package ); - let response = reqwest::get(&url).await?; - if !response.status().is_success() { - return Ok(None); + let mut retries = 0; + let max_retries = 3; + let mut last_error = None; + + while retries < max_retries { + match reqwest::get(&url).await { + Ok(response) => { + if response.status().is_success() { + match response.json::().await { + Ok(rpc_response) => { + if rpc_response.resultcount == 0 || rpc_response.results.is_empty() { + return Ok(None); + } + return Ok(Some(rpc_response.results[0].clone())); + } + Err(e) => { + last_error = Some(format!("JSON parse error: {}", e)); + } + } + } else if response.status().as_u16() == 429 { + // Rate limited — warte und retry + let delay_ms = 1000 * (retries + 1); + warn!("Rate limited für {}, warte {}ms...", package, delay_ms); + tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; + retries += 1; + continue; + } else { + last_error = Some(format!("HTTP {}", response.status())); + } + } + Err(e) => { + last_error = Some(format!("Request failed: {}", e)); + } + } + + retries += 1; + if retries < max_retries { + let delay_ms = 500 * retries; + debug!("Retry {} für {} nach {}ms", retries, package, delay_ms); + tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; + } } - let rpc_response: AurRpcResponse = response.json().await?; - - if rpc_response.resultcount == 0 || rpc_response.results.is_empty() { - return Ok(None); - } - - Ok(Some(rpc_response.results[0].clone())) + warn!("AUR RPC für {} fehlgeschlagen nach {} retries: {}", package, max_retries, last_error.unwrap_or_default()); + Ok(None) // Nicht hard-fail, sondern None zurückgeben } async fn fetch_pkgbuild(