From 7d67da9dfee5ee9a1c3e77768a35813098ffe226 Mon Sep 17 00:00:00 2001 From: Eric Wendland Date: Sat, 18 Jul 2026 16:00:13 +0200 Subject: [PATCH] Degrade gracefully when Iroh modules fail --- crates/geth-node/src/daemon.rs | 173 +++++++++++++++++++++++---------- crates/geth-node/src/lib.rs | 59 ++++++++++- docs/architecture.md | 5 + docs/roadmap.md | 11 +++ 4 files changed, 193 insertions(+), 55 deletions(-) diff --git a/crates/geth-node/src/daemon.rs b/crates/geth-node/src/daemon.rs index 1e60c27..71dcc0c 100644 --- a/crates/geth-node/src/daemon.rs +++ b/crates/geth-node/src/daemon.rs @@ -88,6 +88,14 @@ async fn shutdown_iroh_runtime( node: &LocalNode, endpoint: Option, ) -> Result<(), NodeError> { + clear_iroh_runtime_handles(node)?; + if let Some(endpoint) = endpoint { + endpoint.shutdown().await; + } + Ok(()) +} + +fn clear_iroh_runtime_handles(node: &LocalNode) -> Result<(), NodeError> { node.iroh_docs .lock() .map_err(|_| NodeError::RuntimeLockPoisoned)? @@ -104,9 +112,6 @@ async fn shutdown_iroh_runtime( .lock() .map_err(|_| NodeError::RuntimeLockPoisoned)? .take(); - if let Some(endpoint) = endpoint { - endpoint.shutdown().await; - } Ok(()) } @@ -198,55 +203,24 @@ pub(crate) async fn start_daemon_iroh_endpoint( match geth_iroh::start_endpoint(&config).await { Ok(endpoint) => { let status = endpoint.status(); - if let Some(endpoint_id) = &status.endpoint_id { - let store = Store::open(&node.paths.metadata_db())?; - store.upsert_node_endpoint(endpoint_id, &node.node_id, &node.agent_id, "iroh")?; + match initialize_daemon_iroh_runtime(node, &endpoint, status).await { + Ok(()) => Ok(Some(endpoint)), + Err(error) => { + clear_iroh_runtime_handles(node)?; + endpoint.shutdown().await; + node.iroh_status = EndpointStatus { + enabled: false, + endpoint_id: None, + relay_mode: relay_mode_label, + local_discovery, + note: format!( + "Iroh endpoint started but native runtime initialization failed: {error}" + ), + }; + tracing::warn!(%error, "Iroh runtime unavailable; continuing with local control"); + Ok(None) + } } - let iroh_blobs_path = node.paths.cas_dir().join("iroh-blobs"); - std::fs::create_dir_all(&iroh_blobs_path)?; - let iroh_docs_path = node.paths.home().join("iroh-docs"); - std::fs::create_dir_all(&iroh_docs_path)?; - let blob_store = iroh_blobs::store::fs::FsStore::load(iroh_blobs_path) - .await - .map_err(|error| { - NodeError::IrohPeer(format!("failed to open iroh-blobs store: {error}")) - })?; - let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.endpoint()); - let blob_api: iroh_blobs::api::Store = blob_store.clone().into(); - let docs = iroh_docs::protocol::Docs::persistent(iroh_docs_path) - .spawn(endpoint.endpoint(), blob_api, gossip.clone()) - .await - .map_err(|error| { - NodeError::IrohPeer(format!("failed to start iroh-docs: {error}")) - })?; - let mirrored = mirror_local_cas_to_iroh_blobs(&node.paths, &blob_store).await?; - tracing::info!( - mirrored_blobs = mirrored, - "native iroh-blobs store is ready" - ); - node.iroh_status = status; - *node - .iroh_endpoint - .lock() - .map_err(|_| NodeError::RuntimeLockPoisoned)? = Some(endpoint.clone()); - *node - .iroh_blob_store - .lock() - .map_err(|_| NodeError::RuntimeLockPoisoned)? = Some(blob_store); - *node - .iroh_gossip - .lock() - .map_err(|_| NodeError::RuntimeLockPoisoned)? = Some(gossip); - *node - .iroh_docs - .lock() - .map_err(|_| NodeError::RuntimeLockPoisoned)? = Some(docs); - let mirrored_kv = mirror_all_kv_stores_to_iroh_docs(node).await?; - tracing::info!( - mirrored_kv_stores = mirrored_kv, - "native iroh-docs KV store is ready" - ); - Ok(Some(endpoint)) } Err(error) => { node.iroh_status = EndpointStatus { @@ -261,6 +235,60 @@ pub(crate) async fn start_daemon_iroh_endpoint( } } +async fn initialize_daemon_iroh_runtime( + node: &mut LocalNode, + endpoint: &GethIrohEndpoint, + status: EndpointStatus, +) -> Result<(), NodeError> { + if let Some(endpoint_id) = &status.endpoint_id { + let store = Store::open(&node.paths.metadata_db())?; + store.upsert_node_endpoint(endpoint_id, &node.node_id, &node.agent_id, "iroh")?; + } + let iroh_blobs_path = node.paths.cas_dir().join("iroh-blobs"); + std::fs::create_dir_all(&iroh_blobs_path)?; + let iroh_docs_path = node.paths.home().join("iroh-docs"); + std::fs::create_dir_all(&iroh_docs_path)?; + let blob_store = iroh_blobs::store::fs::FsStore::load(iroh_blobs_path) + .await + .map_err(|error| { + NodeError::IrohPeer(format!("failed to open iroh-blobs store: {error}")) + })?; + let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.endpoint()); + let blob_api: iroh_blobs::api::Store = blob_store.clone().into(); + let docs = iroh_docs::protocol::Docs::persistent(iroh_docs_path) + .spawn(endpoint.endpoint(), blob_api, gossip.clone()) + .await + .map_err(|error| NodeError::IrohPeer(format!("failed to start iroh-docs: {error}")))?; + let mirrored = mirror_local_cas_to_iroh_blobs(&node.paths, &blob_store).await?; + tracing::info!( + mirrored_blobs = mirrored, + "native iroh-blobs store is ready" + ); + node.iroh_status = status; + *node + .iroh_endpoint + .lock() + .map_err(|_| NodeError::RuntimeLockPoisoned)? = Some(endpoint.clone()); + *node + .iroh_blob_store + .lock() + .map_err(|_| NodeError::RuntimeLockPoisoned)? = Some(blob_store); + *node + .iroh_gossip + .lock() + .map_err(|_| NodeError::RuntimeLockPoisoned)? = Some(gossip); + *node + .iroh_docs + .lock() + .map_err(|_| NodeError::RuntimeLockPoisoned)? = Some(docs); + let mirrored_kv = mirror_all_kv_stores_to_iroh_docs(node).await?; + tracing::info!( + mirrored_kv_stores = mirrored_kv, + "native iroh-docs KV store is ready" + ); + Ok(()) +} + fn config_relay_mode_to_iroh( mode: &RelayMode, relay_maps: &std::collections::BTreeMap, @@ -338,6 +366,49 @@ mod tests { ); } + #[tokio::test] + async fn native_runtime_failure_degrades_to_local_only_status() { + if std::env::var_os("GETH_TEST_SKIP_IROH").is_some() { + eprintln!("skipping Iroh degradation test because GETH_TEST_SKIP_IROH is set"); + return; + } + + let dir = tempfile::tempdir().expect("tempdir"); + let paths = GethPaths::from_home(dir.path()); + let mut node = init_node(&paths).expect("init node"); + std::fs::write( + paths.config_file(), + "[iroh]\nrelay_mode = \"disabled\"\nlocal_discovery = false\n", + ) + .expect("offline config"); + std::fs::write(paths.cas_dir().join("iroh-blobs"), b"not a directory") + .expect("invalid native store fixture"); + + let endpoint = start_daemon_iroh_endpoint(&mut node) + .await + .expect("runtime failure is degraded"); + if node + .iroh_status + .note + .starts_with("Iroh endpoint failed to start") + { + eprintln!("skipping native runtime assertion because UDP bind is unavailable"); + return; + } + + assert!(endpoint.is_none()); + assert!(!node.iroh_status.enabled); + assert!( + node.iroh_status + .note + .contains("native runtime initialization failed") + ); + assert!(node.iroh_endpoint.lock().expect("endpoint lock").is_none()); + assert!(node.iroh_blob_store.lock().expect("blob lock").is_none()); + assert!(node.iroh_docs.lock().expect("docs lock").is_none()); + assert!(node.iroh_gossip.lock().expect("gossip lock").is_none()); + } + #[tokio::test] async fn control_socket_binding_rejects_a_second_daemon_and_recovers_stale_paths() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/geth-node/src/lib.rs b/crates/geth-node/src/lib.rs index 8b99810..17ba0d6 100644 --- a/crates/geth-node/src/lib.rs +++ b/crates/geth-node/src/lib.rs @@ -5566,7 +5566,7 @@ pub fn handle_request( iroh_relay_mode: node.iroh_status.relay_mode.clone(), iroh_local_discovery: node.iroh_status.local_discovery, iroh: node.iroh_status.note.clone(), - native_backends: native_backend_statuses(), + native_backends: native_backend_statuses(node)?, })) } ControlRequest::NodeId => Ok(ControlResponse::NodeId(NodeIdResponse { @@ -7579,10 +7579,46 @@ pub fn handle_request( } } -fn native_backend_statuses() -> Vec { - geth_iroh::native_iroh_libraries() +fn native_backend_statuses(node: &LocalNode) -> Result, NodeError> { + let endpoint_ready = node + .iroh_endpoint + .lock() + .map_err(|_| NodeError::RuntimeLockPoisoned)? + .is_some(); + let blobs_ready = node + .iroh_blob_store + .lock() + .map_err(|_| NodeError::RuntimeLockPoisoned)? + .is_some(); + let docs_ready = node + .iroh_docs + .lock() + .map_err(|_| NodeError::RuntimeLockPoisoned)? + .is_some(); + let gossip_ready = node + .iroh_gossip + .lock() + .map_err(|_| NodeError::RuntimeLockPoisoned)? + .is_some(); + Ok(geth_iroh::native_iroh_libraries() .into_iter() .map(|library| { + let runtime_ready = match library.module.as_str() { + "cas" => endpoint_ready && blobs_ready, + "kv" => endpoint_ready && docs_ready, + "pubsub" => endpoint_ready && gossip_ready, + _ => endpoint_ready, + }; + if !runtime_ready { + return NativeBackendStatus { + module: library.module, + current_backend: "unavailable".to_owned(), + target_crate: library.crate_name, + target_version: library.crate_version, + status: "unavailable".to_owned(), + blocker: node.iroh_status.note.clone(), + }; + } let (status, blocker) = match library.module.as_str() { "cas" => ( "wired", @@ -7623,7 +7659,7 @@ fn native_backend_statuses() -> Vec { blocker, } }) - .collect() + .collect()) } fn stored_resource_to_descriptor(stored: StoredResource) -> Result { @@ -11151,6 +11187,21 @@ mod tests { ); } + #[test] + fn native_backend_status_reflects_runtime_availability() { + let dir = tempfile::tempdir().expect("tempdir"); + let node = init_node(&GethPaths::from_home(dir.path())).expect("init node"); + let statuses = native_backend_statuses(&node).expect("backend status"); + + assert!(!statuses.is_empty()); + assert!(statuses.iter().all(|status| status.status == "unavailable")); + assert!( + statuses + .iter() + .all(|status| status.current_backend == "unavailable") + ); + } + #[derive(Debug, PartialEq, Eq)] enum RemoteGuardKind { Capability, diff --git a/docs/architecture.md b/docs/architecture.md index 27f4541..e7885c6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -145,6 +145,11 @@ the daemon lifetime. When endpoint startup succeeds, the Iroh EndpointID is recorded as a transport binding for the stable geth node identity. If local UDP binding is unavailable, the daemon keeps local control running and reports the Iroh startup error through status output. +The same degraded-local behavior applies when the endpoint starts but a native +blob, document, gossip, or initial mirroring step fails. Partially initialized +native handles are released, the endpoint is closed, and `geth status` reports +the runtime backends as unavailable with the concrete startup blocker. Local +metadata, diagnosis, backup, and other non-network control remain available. SSH keys are not transport keys. They are admin trust anchors and signing identities for keychain and authorization operations. The bootstrap `geth ssh diff --git a/docs/roadmap.md b/docs/roadmap.md index 3f32ed8..9ae6517 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -26,6 +26,17 @@ For deployment-readiness work that cuts across feature areas, see - `[x]` Stale, unreachable socket paths are recovered automatically. - `[x]` Tests cover stale recovery and live second-daemon rejection. +- `[x]` Keep local control available when Iroh-native startup degrades. + Acceptance criteria: + - `[x]` Endpoint startup and native blob/docs/gossip initialization failures + do not prevent the local control daemon from serving. + - `[x]` Partial native runtime handles are released and a partially started + endpoint is closed. + - `[x]` Status reports each native backend as unavailable with the runtime + failure note instead of claiming a compiled backend is healthy. + - `[x]` Tests inject a post-endpoint native-store failure and verify clean + degradation where UDP endpoint binding is available. + - `[x]` Make startup modes and the daemon lifecycle discoverable. Acceptance criteria: - `[x]` Base and nested CLI help explain every command family instead of