eric/geth
Watch
1
0
Fork
You've already forked geth
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fetch CAS blobs over Iroh

Browse source
This commit is contained in:
Eric Wendland 2026-05-18 17:18:25 +02:00
commit e4b788fec2
10 changed files with 425 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crates/geth-cas/src/lib.rs
View file

@ -244,6 +244,14 @@ impl LocalCas {
std::fs::copy(path, out).map_err(CasError::from)
}
pub fn read_bytes(&self, hash: &BlobHash) -> Result<Vec<u8>, CasError> {
let path = self.blob_path(hash)?;
if !path.exists() {
return Err(CasError::NotFound(hash.to_string()));
}
Ok(std::fs::read(path)?)
}
pub fn has(&self, hash: &BlobHash) -> Result<bool, CasError> {
Ok(self.blob_path(hash)?.exists())
}
@ -547,6 +555,10 @@ mod tests {
let out = dir.path().join("out.txt");
cas.get_to_path(&info.hash, &out).expect("get");
assert_eq!(std::fs::read(out).expect("read"), b"hello geth");
assert_eq!(
cas.read_bytes(&info.hash).expect("read bytes"),
b"hello geth"
);
}
#[test]

29
crates/geth-cli/src/lib.rs
View file

@ -223,6 +223,10 @@ pub enum CasCommand {
#[arg(long)]
out: PathBuf,
},
Fetch {
node: String,
hash: String,
},
Hash {
path: PathBuf,
},
@ -548,6 +552,10 @@ fn request_for_command(command: Command) -> Result<ControlRequest> {
hash: hash.into(),
out,
},
CasCommand::Fetch { node, hash } => ControlRequest::CasFetch {
node,
hash: hash.into(),
},
CasCommand::Hash { path } => ControlRequest::CasHash { path },
CasCommand::Has { hash } => ControlRequest::CasHas { hash: hash.into() },
CasCommand::Pin { hash } => ControlRequest::CasPin { hash: hash.into() },
@ -879,6 +887,27 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
} => {
println!("wrote {hash} to {} ({size_bytes} bytes)", out.display());
}
ControlResponse::CasFetched {
peer_node_id,
peer_agent_id,
endpoint_id,
hash,
size_bytes,
allowed,
reason,
note,
} => {
if allowed {
println!("fetched {hash} from {peer_node_id} ({size_bytes} bytes)");
} else {
println!("fetch denied for {hash} from {peer_node_id}");
}
println!("agent: {peer_agent_id}");
println!("endpoint: {endpoint_id}");
println!("allowed: {allowed}");
println!("reason: {reason}");
println!("note: {note}");
}
ControlResponse::CasHash { hash } => println!("{hash}"),
ControlResponse::CasHas { hash, present } => println!("{hash}: {present}"),
ControlResponse::CasPinned { hash, pinned } => {

62
crates/geth-control/src/lib.rs
View file

@ -48,6 +48,10 @@ pub enum ControlRequest {
hash: BlobHash,
out: PathBuf,
},
CasFetch {
node: String,
hash: BlobHash,
},
CasHash {
path: PathBuf,
},
@ -269,6 +273,16 @@ pub enum ControlResponse {
out: PathBuf,
size_bytes: u64,
},
CasFetched {
peer_node_id: String,
peer_agent_id: String,
endpoint_id: String,
hash: BlobHash,
size_bytes: u64,
allowed: bool,
reason: String,
note: String,
},
CasHash {
hash: BlobHash,
},
@ -467,6 +481,11 @@ pub enum PeerControlRequest {
capability: String,
nonce: String,
},
CasFetch {
peer_card: PeerCard,
hash: BlobHash,
nonce: String,
},
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -494,6 +513,20 @@ pub enum PeerControlResponse {
nonce: String,
note: String,
},
CasFetched {
node_id: String,
agent_id: String,
endpoint_id: String,
remote_endpoint_id: String,
hash: BlobHash,
size_bytes: u64,
content_base64: Option<String>,
allowed: bool,
reason: String,
evaluated_ops: usize,
nonce: String,
note: String,
},
Error {
message: String,
},
@ -568,6 +601,15 @@ mod tests {
response
);
let request = ControlRequest::CasFetch {
node: "node:peer".to_owned(),
hash: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".into(),
};
assert_eq!(
decode_request(&encode_request(&request).expect("encode")).expect("decode"),
request
);
let request = ControlRequest::SshRevocationExport {
out: PathBuf::from("revocations.krl-spec"),
format: "openssh-krl-spec".to_owned(),
@ -700,5 +742,25 @@ mod tests {
.expect("decode"),
response
);
let response = PeerControlResponse::CasFetched {
node_id: "node:peer".to_owned(),
agent_id: "agent:peer".to_owned(),
endpoint_id: "endpoint:peer".to_owned(),
remote_endpoint_id: "endpoint:caller".to_owned(),
hash: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".into(),
size_bytes: 5,
content_base64: Some("aGVsbG8".to_owned()),
allowed: true,
reason: "direct grant".to_owned(),
evaluated_ops: 1,
nonce: "nonce".to_owned(),
note: "cas fetch".to_owned(),
};
assert_eq!(
decode_peer_response(&encode_peer_response(&response).expect("encode"))
.expect("decode"),
response
);
}
}

1
crates/geth-node/Cargo.toml
View file

@ -6,6 +6,7 @@ rust-version.workspace = true
license.workspace = true
[dependencies]
base64.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tokio.workspace = true

281
crates/geth-node/src/lib.rs
View file

@ -1,5 +1,6 @@
pub mod service;
use base64::Engine;
use geth_auth::{AuthExplanation, AuthOp, AuthOpKind};
use geth_cas::{
BlobInfoSummary, FileConflict, FileConflictKind, FileConflictResolution, FileConflictStatus,
@ -36,8 +37,8 @@ use geth_store::{
StoredResourceSecret, StoredSshCertRequest, StoredSshCertificate, StoredSshRevocation,
};
use geth_types::{
AuthOpId, Capability, KeyId, NodeId, PrincipalId, ResourceId, ResourceKind, ResourceName,
SshCertId, SshCertRequestId, UnixMillis,
AuthOpId, BlobHash, Capability, KeyId, NodeId, PrincipalId, ResourceId, ResourceKind,
ResourceName, SshCertId, SshCertRequestId, UnixMillis,
};
use std::collections::{BTreeMap, VecDeque};
use std::path::Path;
@ -239,6 +240,10 @@ pub async fn handle_request_async(
resource,
capability,
} => peer_auth_check(node, &peer_node, resource, capability).await,
ControlRequest::CasFetch {
node: peer_node,
hash,
} => cas_fetch_from_peer(node, &peer_node, hash).await,
other => handle_request(node, other),
}
}
@ -512,6 +517,9 @@ async fn peer_ping(node: &LocalNode, peer_node: &str) -> Result<ControlResponse,
PeerControlResponse::AuthChecked { .. } => Err(NodeError::IrohPeer(
"peer returned auth-check response to ping request".to_owned(),
)),
PeerControlResponse::CasFetched { .. } => Err(NodeError::IrohPeer(
"peer returned CAS fetch response to ping request".to_owned(),
)),
}
}
@ -614,6 +622,142 @@ async fn peer_auth_check(
PeerControlResponse::Pong { .. } => Err(NodeError::IrohPeer(
"peer returned pong to auth-check request".to_owned(),
)),
PeerControlResponse::CasFetched { .. } => Err(NodeError::IrohPeer(
"peer returned CAS fetch response to auth-check request".to_owned(),
)),
}
}
async fn cas_fetch_from_peer(
node: &LocalNode,
peer_node: &str,
hash: BlobHash,
) -> Result<ControlResponse, NodeError> {
let store = Store::open(&node.paths.metadata_db())?;
let stored = store
.get_peer_card(peer_node)?
.ok_or_else(|| NodeError::PeerNotFound(peer_node.to_owned()))?;
let peer_card: PeerCard = serde_json::from_str(&stored.card_json)?;
peer_card.validate_candidate()?;
let candidate = peer_card
.endpoints
.first()
.ok_or(geth_discovery::DiscoveryError::MissingEndpoint)?;
ensure_peer_card_matches_endpoint(&peer_card, &candidate.endpoint_id)?;
let node_addr = iroh_node_addr_from_candidate(candidate)?;
let endpoint = node
.iroh_endpoint
.lock()
.map_err(|_| NodeError::RuntimeLockPoisoned)?
.clone()
.ok_or(NodeError::IrohEndpointUnavailable)?;
let self_card = local_peer_card(node, DiscoverySource::PeerExchange, true).await?;
let nonce = geth_crypto::blake3_hex(
format!(
"{}\0{}\0{}\0{}",
node.node_id,
peer_node,
hash.as_str(),
geth_store::now_ms()
)
.as_bytes(),
);
let request = PeerControlRequest::CasFetch {
peer_card: self_card,
hash: hash.clone(),
nonce: nonce.clone(),
};
let conn = endpoint
.endpoint()
.connect(node_addr, geth_iroh::ALPN_CONTROL)
.await
.map_err(|error| NodeError::IrohPeer(error.to_string()))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|error| NodeError::IrohPeer(error.to_string()))?;
send.write_all(geth_control::encode_peer_request(&request)?.as_bytes())
.await
.map_err(|error| NodeError::IrohPeer(error.to_string()))?;
send.finish()
.map_err(|error| NodeError::IrohPeer(error.to_string()))?;
let bytes = recv
.read_to_end(64 * 1024 * 1024)
.await
.map_err(|error| NodeError::IrohPeer(error.to_string()))?;
let text =
std::str::from_utf8(&bytes).map_err(|error| NodeError::IrohPeer(error.to_string()))?;
match geth_control::decode_peer_response(text)? {
PeerControlResponse::CasFetched {
node_id,
agent_id,
endpoint_id,
hash: response_hash,
size_bytes,
content_base64,
allowed,
reason,
nonce: response_nonce,
note,
..
} if response_nonce == nonce && response_hash == hash => {
if !allowed {
return Ok(ControlResponse::CasFetched {
peer_node_id: node_id,
peer_agent_id: agent_id,
endpoint_id,
hash: response_hash,
size_bytes: 0,
allowed,
reason,
note,
});
}
let content = content_base64
.ok_or_else(|| NodeError::IrohPeer("peer omitted CAS content".to_owned()))
.and_then(|content| {
base64::engine::general_purpose::STANDARD
.decode(content)
.map_err(|error| NodeError::IrohPeer(error.to_string()))
})?;
if content.len() as u64 != size_bytes {
return Err(NodeError::IrohPeer(format!(
"peer announced {size_bytes} CAS bytes but returned {} bytes",
content.len()
)));
}
let cas = LocalCas::new(node.paths.cas_dir());
let info = cas.add_bytes(&content)?;
if info.hash != response_hash {
return Err(NodeError::IrohPeer(format!(
"peer returned content hash {} for requested {}",
info.hash, response_hash
)));
}
store.record_cas_object(
info.hash.as_str(),
info.size_bytes,
&info.path.to_string_lossy(),
)?;
Ok(ControlResponse::CasFetched {
peer_node_id: node_id,
peer_agent_id: agent_id,
endpoint_id,
hash: info.hash,
size_bytes: info.size_bytes,
allowed,
reason,
note,
})
}
PeerControlResponse::CasFetched { .. } => Err(NodeError::IrohPeer(
"peer CAS fetch response did not match request".to_owned(),
)),
PeerControlResponse::Error { message } => Err(NodeError::IrohPeer(message)),
PeerControlResponse::Pong { .. } | PeerControlResponse::AuthChecked { .. } => Err(
NodeError::IrohPeer("peer returned wrong response type to CAS fetch".to_owned()),
),
}
}
@ -720,6 +864,74 @@ async fn handle_iroh_control_connection(
note: "protected peer request authenticated endpoint/card binding before resource capability evaluation".to_owned(),
}
}
PeerControlRequest::CasFetch {
peer_card,
hash,
nonce,
} => {
peer_card.validate_candidate()?;
ensure_peer_card_matches_endpoint(&peer_card, &remote_endpoint_id)?;
let discovered = DiscoveredPeer::candidate(
peer_card.clone(),
UnixMillis(geth_store::now_ms()),
DiscoverySource::PeerExchange,
)?;
let store = Store::open(&node.paths.metadata_db())?;
store.upsert_peer_card(&StoredPeerCard {
peer_id: peer_card.node_id.to_string(),
card_json: serde_json::to_string(&peer_card)?,
updated_at_ms: discovered.discovered_at.0,
})?;
let resource = "resource:cas:local".to_owned();
let capability = "cas.fetch".to_owned();
let explanation = geth_auth::explain_auth_ops(
&load_auth_ops_for_resource(&store, &resource)?,
PrincipalId::new(peer_card.node_id.to_string()),
ResourceId::new(resource),
Capability::new(capability),
);
if explanation.allowed {
match LocalCas::new(node.paths.cas_dir()).read_bytes(&hash) {
Ok(content) => {
let size_bytes = content.len() as u64;
PeerControlResponse::CasFetched {
node_id: node.node_id.clone(),
agent_id: node.agent_id.clone(),
endpoint_id: node.iroh_status.endpoint_id.clone().unwrap_or_default(),
remote_endpoint_id,
hash,
size_bytes,
content_base64: Some(
base64::engine::general_purpose::STANDARD.encode(content),
),
allowed: true,
reason: explanation.reason,
evaluated_ops: explanation.evaluated_ops,
nonce,
note: "CAS fetch authenticated endpoint/card binding and required cas.fetch on resource:cas:local".to_owned(),
}
}
Err(error) => PeerControlResponse::Error {
message: format!("CAS blob {hash} is not available: {error}"),
},
}
} else {
PeerControlResponse::CasFetched {
node_id: node.node_id.clone(),
agent_id: node.agent_id.clone(),
endpoint_id: node.iroh_status.endpoint_id.clone().unwrap_or_default(),
remote_endpoint_id,
hash,
size_bytes: 0,
content_base64: None,
allowed: false,
reason: explanation.reason,
evaluated_ops: explanation.evaluated_ops,
nonce,
note: "CAS fetch authenticated endpoint/card binding and required cas.fetch on resource:cas:local".to_owned(),
}
}
}
};
send.write_all(geth_control::encode_peer_response(&response)?.as_bytes())
.await
@ -855,6 +1067,7 @@ pub fn handle_request(
}
ControlRequest::PeerPing { .. } => Err(NodeError::IrohEndpointUnavailable),
ControlRequest::PeerAuthCheck { .. } => Err(NodeError::IrohEndpointUnavailable),
ControlRequest::CasFetch { .. } => Err(NodeError::IrohEndpointUnavailable),
ControlRequest::ResourceList => Ok(ControlResponse::ResourceList {
resources: store
.list_resources()?
@ -2330,6 +2543,9 @@ mod tests {
updated_at_ms: geth_store::now_ms(),
})
.expect("insert right peer");
let right_blob = LocalCas::new(right.paths.cas_dir())
.add_bytes(b"remote cas bytes")
.expect("right cas add");
let ping = handle_request_async(
&left,
@ -2380,6 +2596,34 @@ mod tests {
other => panic!("unexpected denied auth response: {other:?}"),
}
let denied_fetch = handle_request_async(
&left,
ControlRequest::CasFetch {
node: right_card.node_id.to_string(),
hash: right_blob.hash.clone(),
},
)
.await
.expect("denied cas fetch");
match denied_fetch {
ControlResponse::CasFetched {
allowed,
reason,
size_bytes,
..
} => {
assert!(!allowed);
assert_eq!(size_bytes, 0);
assert!(reason.contains("no active direct or group grant"));
assert!(
!LocalCas::new(left.paths.cas_dir())
.has(&right_blob.hash)
.expect("left cas has after denied fetch")
);
}
other => panic!("unexpected denied CAS fetch response: {other:?}"),
}
handle_request(
&right,
ControlRequest::AuthGrant {
@ -2415,6 +2659,39 @@ mod tests {
other => panic!("unexpected allowed auth response: {other:?}"),
}
let fetched = handle_request_async(
&left,
ControlRequest::CasFetch {
node: right_card.node_id.to_string(),
hash: right_blob.hash.clone(),
},
)
.await
.expect("allowed cas fetch");
match fetched {
ControlResponse::CasFetched {
allowed,
hash,
size_bytes,
reason,
note,
..
} => {
assert!(allowed);
assert_eq!(hash, right_blob.hash);
assert_eq!(size_bytes, right_blob.size_bytes);
assert!(reason.contains("direct grant"));
assert!(note.contains("required cas.fetch"));
assert_eq!(
LocalCas::new(left.paths.cas_dir())
.read_bytes(&right_blob.hash)
.expect("left cas read fetched blob"),
b"remote cas bytes"
);
}
other => panic!("unexpected allowed CAS fetch response: {other:?}"),
}
left_endpoint.shutdown().await;
right_endpoint.shutdown().await;
}