feat: Multi-Source IOC Fetcher mit Fallback-Chain

- HedgeDoc: Immer aktuell (primär)
- Gist: Versioniert mit History-API (Fallback)
- Arch Security: Offiziell (Backup)
- AUR RPC: Suspicious Maintainer (Heuristik)

Cache-TTL: 5 Minuten für maximale Aktualität.

Resolves: Immer aktuelle IOC-Daten
This commit is contained in:
Thuumate 👻
2026-06-15 19:17:45 +02:00
parent 55a6477fbe
commit 7c32ae0782
+286 -61
View File
@@ -63,6 +63,42 @@ impl std::fmt::Display for ConfidenceLevel {
} }
} }
/// IOC Quellen — sortiert nach Aktualität und Zuverlässigkeit
pub const IOC_SOURCES_LIVE: [(&str, &str, IocSourceType); 4] = [
// 1. HEDGEDOC — Immer aktuell (Live-Paste)
(
"hedgedoc_live",
"https://md.archlinux.org/s/SxbqukK6IA",
IocSourceType::HedgeDoc,
),
// 2. GIST — Versioniert, aber evtl. verzögert
(
"atomic_arch_gist",
"https://gist.githubusercontent.com/Kidev/85756c3dcad3623ca5604a8135bafd14/raw",
IocSourceType::Gist,
),
// 3. ARCH SECURITY TRACKER — Offiziell, aber langsam
(
"arch_security",
"https://security.archlinux.org/advisory/atomic-arch/json",
IocSourceType::JsonApi,
),
// 4. AUR RPC — Dynamisch, API-basiert
(
"aur_rpc",
"https://aur.archlinux.org/rpc/v5/search?by=maintainer&arg=orphan",
IocSourceType::JsonApi,
),
];
#[derive(Debug, Clone)]
pub enum IocSourceType {
HedgeDoc, // Live-Paste, immer aktuell
Gist, // Versioniert, Git-History
JsonApi, // REST API
TextList, // Plaintext
}
pub struct IocFetcher { pub struct IocFetcher {
client: reqwest::Client, client: reqwest::Client,
cache_dir: std::path::PathBuf, cache_dir: std::path::PathBuf,
@@ -82,41 +118,74 @@ impl IocFetcher {
Ok(IocFetcher { Ok(IocFetcher {
client, client,
cache_dir, cache_dir,
cache_ttl: Duration::from_secs(3600), cache_ttl: Duration::from_secs(300), // 5 Minuten Cache für Live-Daten
}) })
} }
/// Holt ALLE IOC-Quellen mit Fallback-Chain
pub async fn fetch_all_iocs(&self) -> Result<Vec<IocEntry>> { pub async fn fetch_all_iocs(&self) -> Result<Vec<IocEntry>> {
let mut all_threats = Vec::new(); let mut all_threats = Vec::new();
let mut sources_used = Vec::new();
match self.fetch_atomic_arch_list().await { // 1. HEDGEDOC (Live, primär)
match self.fetch_hedgedoc().await {
Ok(threats) => { Ok(threats) => {
info!("{} IOCs von Atomic Arch Gist geladen", threats.len()); if !threats.is_empty() {
all_threats.extend(threats); info!("🟢 {} IOCs von HedgeDoc (LIVE)", threats.len());
all_threats.extend(threats);
sources_used.push("hedgedoc_live");
} else {
warn!("⚠️ HedgeDoc leer — Fallback zu Gist");
}
}
Err(e) => {
warn!("🔴 HedgeDoc fehlgeschlagen: {} — Fallback zu Gist", e);
} }
Err(e) => warn!("Konnte Atomic Arch Gist nicht laden: {}", e),
} }
// 2. GIST (Fallback, versioniert)
if sources_used.is_empty() {
match self.fetch_atomic_arch_gist().await {
Ok(threats) => {
info!("🟡 {} IOCs von Gist (Fallback)", threats.len());
all_threats.extend(threats);
sources_used.push("atomic_arch_gist");
}
Err(e) => warn!("🔴 Gist fehlgeschlagen: {}", e),
}
}
// 3. ARCH SECURITY (Offiziell, langsam)
match self.fetch_arch_security().await {
Ok(threats) => {
info!("🟢 {} IOCs von Arch Security", threats.len());
all_threats.extend(threats);
sources_used.push("arch_security");
}
Err(e) => warn!("🔴 Arch Security fehlgeschlagen: {}", e),
}
// 4. AUR RPC (Suspicious Maintainer)
match self.fetch_suspicious_from_aur().await { match self.fetch_suspicious_from_aur().await {
Ok(threats) => { Ok(threats) => {
info!("{} suspicious Pakete von AUR RPC", threats.len()); if !threats.is_empty() {
all_threats.extend(threats); info!("🟡 {} suspicious Pakete von AUR RPC", threats.len());
all_threats.extend(threats);
sources_used.push("aur_rpc");
}
} }
Err(e) => warn!("Konnte AUR RPC nicht abfragen: {}", e), Err(e) => debug!("AUR RPC fehlgeschlagen: {}", e),
} }
match self.fetch_community_blocklist().await { info!("📊 Gesamt: {} IOCs aus Quellen: {:?}", all_threats.len(), sources_used);
Ok(threats) => {
info!("{} Einträge von Community Blocklist", threats.len());
all_threats.extend(threats);
}
Err(e) => warn!("Konnte Community Blocklist nicht laden: {}", e),
}
// Cache speichern
self.save_cache(&all_threats).await?; self.save_cache(&all_threats).await?;
Ok(all_threats) Ok(all_threats)
} }
/// Prüft Cache auf Aktualität
pub async fn get_cached_iocs(&self) -> Result<Vec<IocEntry>> { pub async fn get_cached_iocs(&self) -> Result<Vec<IocEntry>> {
let cache_file = self.cache_dir.join("iocs.json"); let cache_file = self.cache_dir.join("iocs.json");
@@ -132,56 +201,70 @@ impl IocFetcher {
let content = fs::read_to_string(&cache_file).await?; let content = fs::read_to_string(&cache_file).await?;
let iocs: Vec<IocEntry> = serde_json::from_str(&content) let iocs: Vec<IocEntry> = serde_json::from_str(&content)
.context("Konnte Cache nicht parsen")?; .context("Konnte Cache nicht parsen")?;
info!("{} IOCs aus Cache geladen", iocs.len()); debug!("{} IOCs aus Cache geladen (Alter: {}s)", iocs.len(), age.num_seconds());
return Ok(iocs); return Ok(iocs);
} }
} }
// Cache veraltet oder nicht vorhanden — Live holen
self.fetch_all_iocs().await self.fetch_all_iocs().await
} }
async fn fetch_atomic_arch_list(&self) -> Result<Vec<IocEntry>> { // === INDIVIDUELLE FETCHER ===
let url = "https://gist.githubusercontent.com/Kidev/85756c3dcad3623ca5604a8135bafd14/raw";
/// 1. HEDGEDOC — Immer aktuell
async fn fetch_hedgedoc(&self) -> Result<Vec<IocEntry>> {
let url = "https://md.archlinux.org/s/SxbqukK6IA";
let response = self.client let response = self.client
.get(url) .get(url)
.send() .send()
.await .await
.context("HTTP Request fehlgeschlagen")?; .context("HedgeDoc Request fehlgeschlagen")?;
if !response.status().is_success() { if !response.status().is_success() {
anyhow::bail!("HTTP {} von {}", response.status(), url); anyhow::bail!("HedgeDoc HTTP {}", response.status());
} }
let text = response.text().await?; let text = response.text().await?;
let mut threats = Vec::new(); let mut threats = Vec::new();
// HedgeDoc Format: Markdown-Liste mit Paketnamen
// Format: `- paketname` oder `paketname` pro Zeile
for line in text.lines() { for line in text.lines() {
let trimmed = line.trim(); let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
// Überspringe Markdown-Header, Leerzeilen, Kommentare
if trimmed.is_empty()
|| trimmed.starts_with('#')
|| trimmed.starts_with("---")
|| trimmed.starts_with("Arch Linux")
|| trimmed.starts_with("Liste der")
|| trimmed.starts_with("**")
|| trimmed.starts_with("Quelle:")
|| trimmed.starts_with("Aktualisiert:") {
continue; continue;
} }
if trimmed.starts_with('[') || trimmed.starts_with('{') { // Extrahiere Paketnamen (alles nach `- ` oder einfach die Zeile)
if let Ok(json_list) = serde_json::from_str::<Vec<String>>(trimmed) { let pkg = if trimmed.starts_with("- ") {
for pkg in json_list { trimmed[2..].trim()
threats.push(IocEntry { } else if trimmed.starts_with("* ") {
package_name: pkg, trimmed[2..].trim()
threat_type: ThreatType::MaliciousBuildScript,
source: "atomic_arch_gist".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Atomic Arch Supply Chain Attack".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
}
} else { } else {
trimmed
};
// Validiere: Paketnamen sind lowercase, alphanumerisch, -, _, .
if !pkg.is_empty()
&& pkg.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_' || c == '.')
&& pkg.len() < 100 {
threats.push(IocEntry { threats.push(IocEntry {
package_name: trimmed.to_string(), package_name: pkg.to_string(),
threat_type: ThreatType::MaliciousBuildScript, threat_type: ThreatType::MaliciousBuildScript,
source: "atomic_arch_gist".to_string(), source: "hedgedoc_live".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(), discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Atomic Arch Supply Chain Attack".to_string(), description: "Atomic Arch Supply Chain Attack — Live HedgeDoc".to_string(),
confidence: ConfidenceLevel::Critical, confidence: ConfidenceLevel::Critical,
}); });
} }
@@ -190,48 +273,168 @@ impl IocFetcher {
Ok(threats) Ok(threats)
} }
async fn fetch_suspicious_from_aur(&self) -> Result<Vec<IocEntry>> { /// 2. ATOMIC ARCH GIST — Versionierter Fallback
Ok(Vec::new()) async fn fetch_atomic_arch_gist(&self) -> Result<Vec<IocEntry>> {
} // Gist-History für "Latest" holen
let history_url = "https://api.github.com/gists/85756c3dcad3623ca5604a8135bafd14";
async fn fetch_community_blocklist(&self) -> Result<Vec<IocEntry>> { let latest_version = match self.client.get(history_url).send().await {
let url = "https://raw.githubusercontent.com/Kidev/AUR-Blocklist/main/blocklist.txt"; Ok(resp) => {
if resp.status().is_success() {
let response = match self.client.get(url).send().await { let gist: serde_json::Value = resp.json().await?;
Ok(r) => r, gist["history"][0]["version"].as_str().map(|s| s.to_string())
Err(_) => { } else {
debug!("Community Blocklist nicht erreichbar"); None
return Ok(Vec::new()); }
} }
Err(_) => None,
}; };
// Raw URL mit Version für Cache-Busting
let raw_url = match latest_version {
Some(v) => format!(
"https://gist.githubusercontent.com/Kidev/85756c3dcad3623ca5604a8135bafd14/raw?version={}",
v
),
None => "https://gist.githubusercontent.com/Kidev/85756c3dcad3623ca5604a8135bafd14/raw".to_string(),
};
let response = self.client
.get(&raw_url)
.send()
.await
.context("Gist Request fehlgeschlagen")?;
if !response.status().is_success() { if !response.status().is_success() {
return Ok(Vec::new()); anyhow::bail!("Gist HTTP {}", response.status());
} }
let text = response.text().await?; let text = response.text().await?;
let mut threats = Vec::new(); let mut threats = Vec::new();
// Gist ist ein Script — wir parsen die Paketliste
for line in text.lines() { for line in text.lines() {
let trimmed = line.trim(); let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') { if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with("echo") {
continue; continue;
} }
threats.push(IocEntry { // Extrahiere Paketnamen (ähnlich wie HedgeDoc)
package_name: trimmed.to_string(), if trimmed.starts_with("[") || trimmed.starts_with("{") {
threat_type: ThreatType::Unknown("community_reported".to_string()), if let Ok(json_list) = serde_json::from_str::<Vec<String>>(trimmed) {
source: "community_blocklist".to_string(), for pkg in json_list {
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(), threats.push(IocEntry {
description: "Community-reported suspicious package".to_string(), package_name: pkg,
confidence: ConfidenceLevel::Medium, threat_type: ThreatType::MaliciousBuildScript,
}); source: "atomic_arch_gist".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Atomic Arch Supply Chain Attack — Gist Fallback".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
}
} else if !trimmed.contains(" ") && !trimmed.contains("/") && trimmed.len() < 100 {
// Einzelne Paketnamen
threats.push(IocEntry {
package_name: trimmed.to_string(),
threat_type: ThreatType::MaliciousBuildScript,
source: "atomic_arch_gist".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Atomic Arch Supply Chain Attack — Gist Fallback".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
} }
Ok(threats) Ok(threats)
} }
async fn save_cache(&self, /// 3. ARCH SECURITY TRACKER — Offiziell
async fn fetch_arch_security(&self) -> Result<Vec<IocEntry>> {
let url = "https://security.archlinux.org/advisory/atomic-arch/json";
let response = self.client
.get(url)
.send()
.await;
match response {
Ok(resp) => {
if resp.status().is_success() {
let advisory: serde_json::Value = resp.json().await?;
let mut threats = Vec::new();
// Parsen der Advisory-JSON
if let Some(packages) = advisory["packages"].as_array() {
for pkg in packages {
if let Some(name) = pkg.as_str() {
threats.push(IocEntry {
package_name: name.to_string(),
threat_type: ThreatType::MaliciousBuildScript,
source: "arch_security".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Arch Linux Security Advisory — Atomic Arch".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
}
}
Ok(threats)
} else {
Ok(Vec::new())
}
}
Err(_) => Ok(Vec::new()),
}
}
/// 4. AUR RPC — Suspicious Maintainer
async fn fetch_suspicious_from_aur(&self) -> Result<Vec<IocEntry>> {
// Suche nach orphaned Paketen die kürzlich übernommen wurden
// Dies ist eine Heuristik für mögliche Orphan-Takeover
let url = "https://aur.archlinux.org/rpc/v5/search?by=maintainer&arg=orphan";
let response = self.client.get(url).send().await;
match response {
Ok(resp) => {
if resp.status().is_success() {
let rpc: serde_json::Value = resp.json().await?;
let mut threats = Vec::new();
if let Some(results) = rpc["results"].as_array() {
for pkg in results.iter().take(50) {
if let Some(name) = pkg["Name"].as_str() {
// Hohe Heuristik-Scores
let votes = pkg["NumVotes"].as_u64().unwrap_or(0);
if votes < 10 {
threats.push(IocEntry {
package_name: name.to_string(),
threat_type: ThreatType::OrphanTakeover,
source: "aur_rpc".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "AUR Orphaned Package — niedrige Votes".to_string(),
confidence: ConfidenceLevel::Medium,
});
}
}
}
}
Ok(threats)
} else {
Ok(Vec::new())
}
}
Err(_) => Ok(Vec::new()),
}
}
// === CACHE ===
async fn save_cache(
&self,
iocs: &[IocEntry] iocs: &[IocEntry]
) -> Result<()> { ) -> Result<()> {
let cache_file = self.cache_dir.join("iocs.json"); let cache_file = self.cache_dir.join("iocs.json");
@@ -240,7 +443,10 @@ impl IocFetcher {
Ok(()) Ok(())
} }
pub fn check_package(&self, // === PUBLIC HELPERS ===
pub fn check_package(
&self,
package: &str, package: &str,
iocs: &[IocEntry] iocs: &[IocEntry]
) -> Vec<IocEntry> { ) -> Vec<IocEntry> {
@@ -295,4 +501,23 @@ mod tests {
assert_eq!(cached.len(), 1); assert_eq!(cached.len(), 1);
assert_eq!(cached[0].package_name, "test-pkg"); assert_eq!(cached[0].package_name, "test-pkg");
} }
#[tokio::test]
async fn test_hedgedoc_fetch() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = IocFetcher::new(tmp.path().to_path_buf()).await.unwrap();
// Live-Test (kann fehlschlagen wenn HedgeDoc down)
let threats = fetcher.fetch_hedgedoc().await;
// Sollte entweder IOCs liefern oder fehlschlagen
match threats {
Ok(list) => {
println!("HedgeDoc lieferte {} IOCs", list.len());
}
Err(e) => {
println!("HedgeDoc fehlgeschlagen: {} (normal wenn offline)", e);
}
}
}
} }