Degrade gracefully when Iroh modules fail
This commit is contained in:
parent
ed007a0632
commit
7d67da9dfe
4 changed files with 193 additions and 55 deletions
|
|
@ -88,6 +88,14 @@ async fn shutdown_iroh_runtime(
|
||||||
node: &LocalNode,
|
node: &LocalNode,
|
||||||
endpoint: Option<GethIrohEndpoint>,
|
endpoint: Option<GethIrohEndpoint>,
|
||||||
) -> Result<(), NodeError> {
|
) -> Result<(), NodeError> {
|
||||||
|
clear_iroh_runtime_handles(node)?;
|
||||||
|
if let Some(endpoint) = endpoint {
|
||||||
|
endpoint.shutdown().await;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clear_iroh_runtime_handles(node: &LocalNode) -> Result<(), NodeError> {
|
||||||
node.iroh_docs
|
node.iroh_docs
|
||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| NodeError::RuntimeLockPoisoned)?
|
.map_err(|_| NodeError::RuntimeLockPoisoned)?
|
||||||
|
|
@ -104,9 +112,6 @@ async fn shutdown_iroh_runtime(
|
||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| NodeError::RuntimeLockPoisoned)?
|
.map_err(|_| NodeError::RuntimeLockPoisoned)?
|
||||||
.take();
|
.take();
|
||||||
if let Some(endpoint) = endpoint {
|
|
||||||
endpoint.shutdown().await;
|
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -198,6 +203,43 @@ pub(crate) async fn start_daemon_iroh_endpoint(
|
||||||
match geth_iroh::start_endpoint(&config).await {
|
match geth_iroh::start_endpoint(&config).await {
|
||||||
Ok(endpoint) => {
|
Ok(endpoint) => {
|
||||||
let status = endpoint.status();
|
let status = endpoint.status();
|
||||||
|
match initialize_daemon_iroh_runtime(node, &endpoint, status).await {
|
||||||
|
Ok(()) => Ok(Some(endpoint)),
|
||||||
|
Err(error) => {
|
||||||
|
clear_iroh_runtime_handles(node)?;
|
||||||
|
endpoint.shutdown().await;
|
||||||
|
node.iroh_status = EndpointStatus {
|
||||||
|
enabled: false,
|
||||||
|
endpoint_id: None,
|
||||||
|
relay_mode: relay_mode_label,
|
||||||
|
local_discovery,
|
||||||
|
note: format!(
|
||||||
|
"Iroh endpoint started but native runtime initialization failed: {error}"
|
||||||
|
),
|
||||||
|
};
|
||||||
|
tracing::warn!(%error, "Iroh runtime unavailable; continuing with local control");
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
node.iroh_status = EndpointStatus {
|
||||||
|
enabled: false,
|
||||||
|
endpoint_id: None,
|
||||||
|
relay_mode: relay_mode_label,
|
||||||
|
local_discovery,
|
||||||
|
note: format!("Iroh endpoint failed to start: {error}"),
|
||||||
|
};
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn initialize_daemon_iroh_runtime(
|
||||||
|
node: &mut LocalNode,
|
||||||
|
endpoint: &GethIrohEndpoint,
|
||||||
|
status: EndpointStatus,
|
||||||
|
) -> Result<(), NodeError> {
|
||||||
if let Some(endpoint_id) = &status.endpoint_id {
|
if let Some(endpoint_id) = &status.endpoint_id {
|
||||||
let store = Store::open(&node.paths.metadata_db())?;
|
let store = Store::open(&node.paths.metadata_db())?;
|
||||||
store.upsert_node_endpoint(endpoint_id, &node.node_id, &node.agent_id, "iroh")?;
|
store.upsert_node_endpoint(endpoint_id, &node.node_id, &node.agent_id, "iroh")?;
|
||||||
|
|
@ -216,9 +258,7 @@ pub(crate) async fn start_daemon_iroh_endpoint(
|
||||||
let docs = iroh_docs::protocol::Docs::persistent(iroh_docs_path)
|
let docs = iroh_docs::protocol::Docs::persistent(iroh_docs_path)
|
||||||
.spawn(endpoint.endpoint(), blob_api, gossip.clone())
|
.spawn(endpoint.endpoint(), blob_api, gossip.clone())
|
||||||
.await
|
.await
|
||||||
.map_err(|error| {
|
.map_err(|error| NodeError::IrohPeer(format!("failed to start iroh-docs: {error}")))?;
|
||||||
NodeError::IrohPeer(format!("failed to start iroh-docs: {error}"))
|
|
||||||
})?;
|
|
||||||
let mirrored = mirror_local_cas_to_iroh_blobs(&node.paths, &blob_store).await?;
|
let mirrored = mirror_local_cas_to_iroh_blobs(&node.paths, &blob_store).await?;
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
mirrored_blobs = mirrored,
|
mirrored_blobs = mirrored,
|
||||||
|
|
@ -246,19 +286,7 @@ pub(crate) async fn start_daemon_iroh_endpoint(
|
||||||
mirrored_kv_stores = mirrored_kv,
|
mirrored_kv_stores = mirrored_kv,
|
||||||
"native iroh-docs KV store is ready"
|
"native iroh-docs KV store is ready"
|
||||||
);
|
);
|
||||||
Ok(Some(endpoint))
|
Ok(())
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
node.iroh_status = EndpointStatus {
|
|
||||||
enabled: false,
|
|
||||||
endpoint_id: None,
|
|
||||||
relay_mode: relay_mode_label,
|
|
||||||
local_discovery,
|
|
||||||
note: format!("Iroh endpoint failed to start: {error}"),
|
|
||||||
};
|
|
||||||
Ok(None)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn config_relay_mode_to_iroh(
|
fn config_relay_mode_to_iroh(
|
||||||
|
|
@ -338,6 +366,49 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn native_runtime_failure_degrades_to_local_only_status() {
|
||||||
|
if std::env::var_os("GETH_TEST_SKIP_IROH").is_some() {
|
||||||
|
eprintln!("skipping Iroh degradation test because GETH_TEST_SKIP_IROH is set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let paths = GethPaths::from_home(dir.path());
|
||||||
|
let mut node = init_node(&paths).expect("init node");
|
||||||
|
std::fs::write(
|
||||||
|
paths.config_file(),
|
||||||
|
"[iroh]\nrelay_mode = \"disabled\"\nlocal_discovery = false\n",
|
||||||
|
)
|
||||||
|
.expect("offline config");
|
||||||
|
std::fs::write(paths.cas_dir().join("iroh-blobs"), b"not a directory")
|
||||||
|
.expect("invalid native store fixture");
|
||||||
|
|
||||||
|
let endpoint = start_daemon_iroh_endpoint(&mut node)
|
||||||
|
.await
|
||||||
|
.expect("runtime failure is degraded");
|
||||||
|
if node
|
||||||
|
.iroh_status
|
||||||
|
.note
|
||||||
|
.starts_with("Iroh endpoint failed to start")
|
||||||
|
{
|
||||||
|
eprintln!("skipping native runtime assertion because UDP bind is unavailable");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(endpoint.is_none());
|
||||||
|
assert!(!node.iroh_status.enabled);
|
||||||
|
assert!(
|
||||||
|
node.iroh_status
|
||||||
|
.note
|
||||||
|
.contains("native runtime initialization failed")
|
||||||
|
);
|
||||||
|
assert!(node.iroh_endpoint.lock().expect("endpoint lock").is_none());
|
||||||
|
assert!(node.iroh_blob_store.lock().expect("blob lock").is_none());
|
||||||
|
assert!(node.iroh_docs.lock().expect("docs lock").is_none());
|
||||||
|
assert!(node.iroh_gossip.lock().expect("gossip lock").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn control_socket_binding_rejects_a_second_daemon_and_recovers_stale_paths() {
|
async fn control_socket_binding_rejects_a_second_daemon_and_recovers_stale_paths() {
|
||||||
let dir = tempfile::tempdir().expect("tempdir");
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
|
|
||||||
|
|
@ -5566,7 +5566,7 @@ pub fn handle_request(
|
||||||
iroh_relay_mode: node.iroh_status.relay_mode.clone(),
|
iroh_relay_mode: node.iroh_status.relay_mode.clone(),
|
||||||
iroh_local_discovery: node.iroh_status.local_discovery,
|
iroh_local_discovery: node.iroh_status.local_discovery,
|
||||||
iroh: node.iroh_status.note.clone(),
|
iroh: node.iroh_status.note.clone(),
|
||||||
native_backends: native_backend_statuses(),
|
native_backends: native_backend_statuses(node)?,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
ControlRequest::NodeId => Ok(ControlResponse::NodeId(NodeIdResponse {
|
ControlRequest::NodeId => Ok(ControlResponse::NodeId(NodeIdResponse {
|
||||||
|
|
@ -7579,10 +7579,46 @@ pub fn handle_request(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn native_backend_statuses() -> Vec<NativeBackendStatus> {
|
fn native_backend_statuses(node: &LocalNode) -> Result<Vec<NativeBackendStatus>, NodeError> {
|
||||||
geth_iroh::native_iroh_libraries()
|
let endpoint_ready = node
|
||||||
|
.iroh_endpoint
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| NodeError::RuntimeLockPoisoned)?
|
||||||
|
.is_some();
|
||||||
|
let blobs_ready = node
|
||||||
|
.iroh_blob_store
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| NodeError::RuntimeLockPoisoned)?
|
||||||
|
.is_some();
|
||||||
|
let docs_ready = node
|
||||||
|
.iroh_docs
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| NodeError::RuntimeLockPoisoned)?
|
||||||
|
.is_some();
|
||||||
|
let gossip_ready = node
|
||||||
|
.iroh_gossip
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| NodeError::RuntimeLockPoisoned)?
|
||||||
|
.is_some();
|
||||||
|
Ok(geth_iroh::native_iroh_libraries()
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|library| {
|
.map(|library| {
|
||||||
|
let runtime_ready = match library.module.as_str() {
|
||||||
|
"cas" => endpoint_ready && blobs_ready,
|
||||||
|
"kv" => endpoint_ready && docs_ready,
|
||||||
|
"pubsub" => endpoint_ready && gossip_ready,
|
||||||
|
_ => endpoint_ready,
|
||||||
|
};
|
||||||
|
if !runtime_ready {
|
||||||
|
return NativeBackendStatus {
|
||||||
|
module: library.module,
|
||||||
|
current_backend: "unavailable".to_owned(),
|
||||||
|
target_crate: library.crate_name,
|
||||||
|
target_version: library.crate_version,
|
||||||
|
status: "unavailable".to_owned(),
|
||||||
|
blocker: node.iroh_status.note.clone(),
|
||||||
|
};
|
||||||
|
}
|
||||||
let (status, blocker) = match library.module.as_str() {
|
let (status, blocker) = match library.module.as_str() {
|
||||||
"cas" => (
|
"cas" => (
|
||||||
"wired",
|
"wired",
|
||||||
|
|
@ -7623,7 +7659,7 @@ fn native_backend_statuses() -> Vec<NativeBackendStatus> {
|
||||||
blocker,
|
blocker,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect()
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stored_resource_to_descriptor(stored: StoredResource) -> Result<ResourceDescriptor, NodeError> {
|
fn stored_resource_to_descriptor(stored: StoredResource) -> Result<ResourceDescriptor, NodeError> {
|
||||||
|
|
@ -11151,6 +11187,21 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn native_backend_status_reflects_runtime_availability() {
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let node = init_node(&GethPaths::from_home(dir.path())).expect("init node");
|
||||||
|
let statuses = native_backend_statuses(&node).expect("backend status");
|
||||||
|
|
||||||
|
assert!(!statuses.is_empty());
|
||||||
|
assert!(statuses.iter().all(|status| status.status == "unavailable"));
|
||||||
|
assert!(
|
||||||
|
statuses
|
||||||
|
.iter()
|
||||||
|
.all(|status| status.current_backend == "unavailable")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq)]
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
enum RemoteGuardKind {
|
enum RemoteGuardKind {
|
||||||
Capability,
|
Capability,
|
||||||
|
|
|
||||||
|
|
@ -145,6 +145,11 @@ the daemon lifetime. When endpoint startup succeeds, the Iroh EndpointID is
|
||||||
recorded as a transport binding for the stable geth node identity. If local UDP
|
recorded as a transport binding for the stable geth node identity. If local UDP
|
||||||
binding is unavailable, the daemon keeps local control running and reports the
|
binding is unavailable, the daemon keeps local control running and reports the
|
||||||
Iroh startup error through status output.
|
Iroh startup error through status output.
|
||||||
|
The same degraded-local behavior applies when the endpoint starts but a native
|
||||||
|
blob, document, gossip, or initial mirroring step fails. Partially initialized
|
||||||
|
native handles are released, the endpoint is closed, and `geth status` reports
|
||||||
|
the runtime backends as unavailable with the concrete startup blocker. Local
|
||||||
|
metadata, diagnosis, backup, and other non-network control remain available.
|
||||||
|
|
||||||
SSH keys are not transport keys. They are admin trust anchors and signing
|
SSH keys are not transport keys. They are admin trust anchors and signing
|
||||||
identities for keychain and authorization operations. The bootstrap `geth ssh
|
identities for keychain and authorization operations. The bootstrap `geth ssh
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,17 @@ For deployment-readiness work that cuts across feature areas, see
|
||||||
- `[x]` Stale, unreachable socket paths are recovered automatically.
|
- `[x]` Stale, unreachable socket paths are recovered automatically.
|
||||||
- `[x]` Tests cover stale recovery and live second-daemon rejection.
|
- `[x]` Tests cover stale recovery and live second-daemon rejection.
|
||||||
|
|
||||||
|
- `[x]` Keep local control available when Iroh-native startup degrades.
|
||||||
|
Acceptance criteria:
|
||||||
|
- `[x]` Endpoint startup and native blob/docs/gossip initialization failures
|
||||||
|
do not prevent the local control daemon from serving.
|
||||||
|
- `[x]` Partial native runtime handles are released and a partially started
|
||||||
|
endpoint is closed.
|
||||||
|
- `[x]` Status reports each native backend as unavailable with the runtime
|
||||||
|
failure note instead of claiming a compiled backend is healthy.
|
||||||
|
- `[x]` Tests inject a post-endpoint native-store failure and verify clean
|
||||||
|
degradation where UDP endpoint binding is available.
|
||||||
|
|
||||||
- `[x]` Make startup modes and the daemon lifecycle discoverable.
|
- `[x]` Make startup modes and the daemon lifecycle discoverable.
|
||||||
Acceptance criteria:
|
Acceptance criteria:
|
||||||
- `[x]` Base and nested CLI help explain every command family instead of
|
- `[x]` Base and nested CLI help explain every command family instead of
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue