arch_agent/aegisaur
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Multi-Source IOC Fetcher mit Fallback-Chain

Browse Source 
- HedgeDoc: Immer aktuell (primär)
- Gist: Versioniert mit History-API (Fallback)
- Arch Security: Offiziell (Backup)
- AUR RPC: Suspicious Maintainer (Heuristik)

Cache-TTL: 5 Minuten für maximale Aktualität.

Resolves: Immer aktuelle IOC-Daten
This commit is contained in:
Thuumate 👻
2026-06-15 19:17:45 +02:00
parent 55a6477fbe
commit 7c32ae0782
1 changed files with 286 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ioc_fetcher.rs
+286 -61
View File
@@ -63,6 +63,42 @@ impl std::fmt::Display for ConfidenceLevel {
}
}
/// IOC Quellen — sortiert nach Aktualität und Zuverlässigkeit
pub const IOC_SOURCES_LIVE: [(&str, &str, IocSourceType); 4] = [
// 1. HEDGEDOC — Immer aktuell (Live-Paste)
(
"hedgedoc_live",
"https://md.archlinux.org/s/SxbqukK6IA",
IocSourceType::HedgeDoc,
),
// 2. GIST — Versioniert, aber evtl. verzögert
(
"atomic_arch_gist",
"https://gist.githubusercontent.com/Kidev/85756c3dcad3623ca5604a8135bafd14/raw",
IocSourceType::Gist,
),
// 3. ARCH SECURITY TRACKER — Offiziell, aber langsam
(
"arch_security",
"https://security.archlinux.org/advisory/atomic-arch/json",
IocSourceType::JsonApi,
),
// 4. AUR RPC — Dynamisch, API-basiert
(
"aur_rpc",
"https://aur.archlinux.org/rpc/v5/search?by=maintainer&arg=orphan",
IocSourceType::JsonApi,
),
];
#[derive(Debug, Clone)]
pub enum IocSourceType {
HedgeDoc, // Live-Paste, immer aktuell
Gist, // Versioniert, Git-History
JsonApi, // REST API
TextList, // Plaintext
}
pub struct IocFetcher {
client: reqwest::Client,
cache_dir: std::path::PathBuf,
@@ -82,41 +118,74 @@ impl IocFetcher {
Ok(IocFetcher {
client,
cache_dir,
cache_ttl: Duration::from_secs(3600),
cache_ttl: Duration::from_secs(300), // 5 Minuten Cache für Live-Daten
})
}
/// Holt ALLE IOC-Quellen mit Fallback-Chain
pub async fn fetch_all_iocs(&self) -> Result<Vec<IocEntry>> {
let mut all_threats = Vec::new();
let mut sources_used = Vec::new();
match self.fetch_atomic_arch_list().await {
// 1. HEDGEDOC (Live, primär)
match self.fetch_hedgedoc().await {
Ok(threats) => {
info!("{} IOCs von Atomic Arch Gist geladen", threats.len());
all_threats.extend(threats);
if !threats.is_empty() {
info!("🟢 {} IOCs von HedgeDoc (LIVE)", threats.len());
all_threats.extend(threats);
sources_used.push("hedgedoc_live");
} else {
warn!("⚠️ HedgeDoc leer — Fallback zu Gist");
}
}
Err(e) => {
warn!("🔴 HedgeDoc fehlgeschlagen: {} — Fallback zu Gist", e);
}
Err(e) => warn!("Konnte Atomic Arch Gist nicht laden: {}", e),
}
// 2. GIST (Fallback, versioniert)
if sources_used.is_empty() {
match self.fetch_atomic_arch_gist().await {
Ok(threats) => {
info!("🟡 {} IOCs von Gist (Fallback)", threats.len());
all_threats.extend(threats);
sources_used.push("atomic_arch_gist");
}
Err(e) => warn!("🔴 Gist fehlgeschlagen: {}", e),
}
}
// 3. ARCH SECURITY (Offiziell, langsam)
match self.fetch_arch_security().await {
Ok(threats) => {
info!("🟢 {} IOCs von Arch Security", threats.len());
all_threats.extend(threats);
sources_used.push("arch_security");
}
Err(e) => warn!("🔴 Arch Security fehlgeschlagen: {}", e),
}
// 4. AUR RPC (Suspicious Maintainer)
match self.fetch_suspicious_from_aur().await {
Ok(threats) => {
info!("{} suspicious Pakete von AUR RPC", threats.len());
all_threats.extend(threats);
if !threats.is_empty() {
info!("🟡 {} suspicious Pakete von AUR RPC", threats.len());
all_threats.extend(threats);
sources_used.push("aur_rpc");
}
}
Err(e) => warn!("Konnte AUR RPC nicht abfragen: {}", e),
Err(e) => debug!("AUR RPC fehlgeschlagen: {}", e),
}
match self.fetch_community_blocklist().await {
Ok(threats) => {
info!("{} Einträge von Community Blocklist", threats.len());
all_threats.extend(threats);
}
Err(e) => warn!("Konnte Community Blocklist nicht laden: {}", e),
}
info!("📊 Gesamt: {} IOCs aus Quellen: {:?}", all_threats.len(), sources_used);
// Cache speichern
self.save_cache(&all_threats).await?;
Ok(all_threats)
}
/// Prüft Cache auf Aktualität
pub async fn get_cached_iocs(&self) -> Result<Vec<IocEntry>> {
let cache_file = self.cache_dir.join("iocs.json");
@@ -132,56 +201,70 @@ impl IocFetcher {
let content = fs::read_to_string(&cache_file).await?;
let iocs: Vec<IocEntry> = serde_json::from_str(&content)
.context("Konnte Cache nicht parsen")?;
info!("{} IOCs aus Cache geladen", iocs.len());
debug!("{} IOCs aus Cache geladen (Alter: {}s)", iocs.len(), age.num_seconds());
return Ok(iocs);
}
}
// Cache veraltet oder nicht vorhanden — Live holen
self.fetch_all_iocs().await
}
async fn fetch_atomic_arch_list(&self) -> Result<Vec<IocEntry>> {
let url = "https://gist.githubusercontent.com/Kidev/85756c3dcad3623ca5604a8135bafd14/raw";
// === INDIVIDUELLE FETCHER ===
/// 1. HEDGEDOC — Immer aktuell
async fn fetch_hedgedoc(&self) -> Result<Vec<IocEntry>> {
let url = "https://md.archlinux.org/s/SxbqukK6IA";
let response = self.client
.get(url)
.send()
.await
.context("HTTP Request fehlgeschlagen")?;
.context("HedgeDoc Request fehlgeschlagen")?;
if !response.status().is_success() {
anyhow::bail!("HTTP {} von {}", response.status(), url);
anyhow::bail!("HedgeDoc HTTP {}", response.status());
}
let text = response.text().await?;
let mut threats = Vec::new();
// HedgeDoc Format: Markdown-Liste mit Paketnamen
// Format: `- paketname` oder `paketname` pro Zeile
for line in text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
// Überspringe Markdown-Header, Leerzeilen, Kommentare
if trimmed.is_empty()
|| trimmed.starts_with('#')
|| trimmed.starts_with("---")
|| trimmed.starts_with("Arch Linux")
|| trimmed.starts_with("Liste der")
|| trimmed.starts_with("**")
|| trimmed.starts_with("Quelle:")
|| trimmed.starts_with("Aktualisiert:") {
continue;
}
if trimmed.starts_with('[') || trimmed.starts_with('{') {
if let Ok(json_list) = serde_json::from_str::<Vec<String>>(trimmed) {
for pkg in json_list {
threats.push(IocEntry {
package_name: pkg,
threat_type: ThreatType::MaliciousBuildScript,
source: "atomic_arch_gist".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Atomic Arch Supply Chain Attack".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
}
// Extrahiere Paketnamen (alles nach `- ` oder einfach die Zeile)
let pkg = if trimmed.starts_with("- ") {
trimmed[2..].trim()
} else if trimmed.starts_with("* ") {
trimmed[2..].trim()
} else {
trimmed
};
// Validiere: Paketnamen sind lowercase, alphanumerisch, -, _, .
if !pkg.is_empty()
&& pkg.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_' || c == '.')
&& pkg.len() < 100 {
threats.push(IocEntry {
package_name: trimmed.to_string(),
package_name: pkg.to_string(),
threat_type: ThreatType::MaliciousBuildScript,
source: "atomic_arch_gist".to_string(),
source: "hedgedoc_live".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Atomic Arch Supply Chain Attack".to_string(),
description: "Atomic Arch Supply Chain Attack — Live HedgeDoc".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
@@ -190,48 +273,168 @@ impl IocFetcher {
Ok(threats)
}
async fn fetch_suspicious_from_aur(&self) -> Result<Vec<IocEntry>> {
Ok(Vec::new())
}
async fn fetch_community_blocklist(&self) -> Result<Vec<IocEntry>> {
let url = "https://raw.githubusercontent.com/Kidev/AUR-Blocklist/main/blocklist.txt";
let response = match self.client.get(url).send().await {
Ok(r) => r,
Err(_) => {
debug!("Community Blocklist nicht erreichbar");
return Ok(Vec::new());
/// 2. ATOMIC ARCH GIST — Versionierter Fallback
async fn fetch_atomic_arch_gist(&self) -> Result<Vec<IocEntry>> {
// Gist-History für "Latest" holen
let history_url = "https://api.github.com/gists/85756c3dcad3623ca5604a8135bafd14";
let latest_version = match self.client.get(history_url).send().await {
Ok(resp) => {
if resp.status().is_success() {
let gist: serde_json::Value = resp.json().await?;
gist["history"][0]["version"].as_str().map(|s| s.to_string())
} else {
None
}
}
Err(_) => None,
};
// Raw URL mit Version für Cache-Busting
let raw_url = match latest_version {
Some(v) => format!(
"https://gist.githubusercontent.com/Kidev/85756c3dcad3623ca5604a8135bafd14/raw?version={}",
v
),
None => "https://gist.githubusercontent.com/Kidev/85756c3dcad3623ca5604a8135bafd14/raw".to_string(),
};
let response = self.client
.get(&raw_url)
.send()
.await
.context("Gist Request fehlgeschlagen")?;
if !response.status().is_success() {
return Ok(Vec::new());
anyhow::bail!("Gist HTTP {}", response.status());
}
let text = response.text().await?;
let mut threats = Vec::new();
// Gist ist ein Script — wir parsen die Paketliste
for line in text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with("echo") {
continue;
}
threats.push(IocEntry {
package_name: trimmed.to_string(),
threat_type: ThreatType::Unknown("community_reported".to_string()),
source: "community_blocklist".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Community-reported suspicious package".to_string(),
confidence: ConfidenceLevel::Medium,
});
// Extrahiere Paketnamen (ähnlich wie HedgeDoc)
if trimmed.starts_with("[") || trimmed.starts_with("{") {
if let Ok(json_list) = serde_json::from_str::<Vec<String>>(trimmed) {
for pkg in json_list {
threats.push(IocEntry {
package_name: pkg,
threat_type: ThreatType::MaliciousBuildScript,
source: "atomic_arch_gist".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Atomic Arch Supply Chain Attack — Gist Fallback".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
}
} else if !trimmed.contains(" ") && !trimmed.contains("/") && trimmed.len() < 100 {
// Einzelne Paketnamen
threats.push(IocEntry {
package_name: trimmed.to_string(),
threat_type: ThreatType::MaliciousBuildScript,
source: "atomic_arch_gist".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Atomic Arch Supply Chain Attack — Gist Fallback".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
}
Ok(threats)
}
async fn save_cache(&self,
/// 3. ARCH SECURITY TRACKER — Offiziell
async fn fetch_arch_security(&self) -> Result<Vec<IocEntry>> {
let url = "https://security.archlinux.org/advisory/atomic-arch/json";
let response = self.client
.get(url)
.send()
.await;
match response {
Ok(resp) => {
if resp.status().is_success() {
let advisory: serde_json::Value = resp.json().await?;
let mut threats = Vec::new();
// Parsen der Advisory-JSON
if let Some(packages) = advisory["packages"].as_array() {
for pkg in packages {
if let Some(name) = pkg.as_str() {
threats.push(IocEntry {
package_name: name.to_string(),
threat_type: ThreatType::MaliciousBuildScript,
source: "arch_security".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "Arch Linux Security Advisory — Atomic Arch".to_string(),
confidence: ConfidenceLevel::Critical,
});
}
}
}
Ok(threats)
} else {
Ok(Vec::new())
}
}
Err(_) => Ok(Vec::new()),
}
}
/// 4. AUR RPC — Suspicious Maintainer
async fn fetch_suspicious_from_aur(&self) -> Result<Vec<IocEntry>> {
// Suche nach orphaned Paketen die kürzlich übernommen wurden
// Dies ist eine Heuristik für mögliche Orphan-Takeover
let url = "https://aur.archlinux.org/rpc/v5/search?by=maintainer&arg=orphan";
let response = self.client.get(url).send().await;
match response {
Ok(resp) => {
if resp.status().is_success() {
let rpc: serde_json::Value = resp.json().await?;
let mut threats = Vec::new();
if let Some(results) = rpc["results"].as_array() {
for pkg in results.iter().take(50) {
if let Some(name) = pkg["Name"].as_str() {
// Hohe Heuristik-Scores
let votes = pkg["NumVotes"].as_u64().unwrap_or(0);
if votes < 10 {
threats.push(IocEntry {
package_name: name.to_string(),
threat_type: ThreatType::OrphanTakeover,
source: "aur_rpc".to_string(),
discovered_date: chrono::Local::now().format("%Y-%m-%d").to_string(),
description: "AUR Orphaned Package — niedrige Votes".to_string(),
confidence: ConfidenceLevel::Medium,
});
}
}
}
}
Ok(threats)
} else {
Ok(Vec::new())
}
}
Err(_) => Ok(Vec::new()),
}
}
// === CACHE ===
async fn save_cache(
&self,
iocs: &[IocEntry]
) -> Result<()> {
let cache_file = self.cache_dir.join("iocs.json");
@@ -240,7 +443,10 @@ impl IocFetcher {
Ok(())
}
pub fn check_package(&self,
// === PUBLIC HELPERS ===
pub fn check_package(
&self,
package: &str,
iocs: &[IocEntry]
) -> Vec<IocEntry> {
@@ -295,4 +501,23 @@ mod tests {
assert_eq!(cached.len(), 1);
assert_eq!(cached[0].package_name, "test-pkg");
}
#[tokio::test]
async fn test_hedgedoc_fetch() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = IocFetcher::new(tmp.path().to_path_buf()).await.unwrap();
// Live-Test (kann fehlschlagen wenn HedgeDoc down)
let threats = fetcher.fetch_hedgedoc().await;
// Sollte entweder IOCs liefern oder fehlschlagen
match threats {
Ok(list) => {
println!("HedgeDoc lieferte {} IOCs", list.len());
}
Err(e) => {
println!("HedgeDoc fehlgeschlagen: {} (normal wenn offline)", e);
}
}
}
}