Represent SSH distribution sync as a resource log

This commit is contained in:
Eric Wendland 2026-05-22 16:51:30 +02:00
commit 42d7301bdb
6 changed files with 247 additions and 52 deletions

View file

@ -215,14 +215,17 @@ node; it only unlocks the requested capability on that one resource.
`geth ssh cert sync <node-id>` requires `ssh_cert.sync` on `resource:ssh:certs` `geth ssh cert sync <node-id>` requires `ssh_cert.sync` on `resource:ssh:certs`
at the peer. `geth ssh revocation sync <node-id>` requires at the peer. `geth ssh revocation sync <node-id>` requires
`ssh_revocation.sync` on `resource:ssh:revocations`. Both commands merge `ssh_revocation.sync` on `resource:ssh:revocations`. Both commands merge
authorized peer metadata into the local store for offline listing and later authorized peer SSH distribution log entries into the local store for offline
approval/signing workflows. While the daemon is running, it also performs a listing and later approval/signing workflows. The current log is materialized
background live-sync tick for known peers. The default interval is 30 seconds from signed certificate requests, signed certificate imports, and signed
and can be changed in `config.toml` with `[sync] live_sync_enabled` and revocation records, then reduced locally; it is not a mutable remote ACL blob.
`live_sync_interval_ms`. Live-sync stores per-peer high-water cursors in local While the daemon is running, it also performs a background live-sync tick for
metadata so repeated ticks request only newer SSH certificate-flow and known peers. The default interval is 30 seconds and can be changed in
revocation records. Sync import preserves local metadata by rejecting `config.toml` with `[sync] live_sync_enabled` and `live_sync_interval_ms`.
conflicting records with ids that already exist locally. Live-sync stores per-peer high-water cursors in local metadata so repeated ticks
request only newer SSH certificate-flow and revocation log entries. Sync import
preserves local metadata by rejecting conflicting records with ids that already
exist locally.
Before probing individual modules, the daemon asks the peer for an authorized Before probing individual modules, the daemon asks the peer for an authorized
sync-status summary over Iroh. The peer only returns stream watermarks for sync-status summary over Iroh. The peer only returns stream watermarks for
resources where the caller already has the matching capability, letting the resources where the caller already has the matching capability, letting the

View file

@ -10,7 +10,8 @@ use geth_pubsub::PubsubMessage;
use geth_resource::ResourceDescriptor; use geth_resource::ResourceDescriptor;
use geth_secrets::{BearerAccess, BearerChallenge, BearerProof, ResourceMasterSecret}; use geth_secrets::{BearerAccess, BearerChallenge, BearerProof, ResourceMasterSecret};
use geth_ssh_identity::{ use geth_ssh_identity::{
SshCertApproval, SshCertRequest, SshCertificateRecord, SshRevocationEntry, SshCertApproval, SshCertRequest, SshCertificateRecord, SshDistributionLogEntry,
SshRevocationEntry,
}; };
use geth_ssh_proxy::SshProxyConnection; use geth_ssh_proxy::SshProxyConnection;
use geth_types::BlobHash; use geth_types::BlobHash;
@ -1177,6 +1178,7 @@ pub enum PeerControlResponse {
remote_endpoint_id: String, remote_endpoint_id: String,
requests: Vec<SshCertRequest>, requests: Vec<SshCertRequest>,
certificates: Vec<SshCertificateRecord>, certificates: Vec<SshCertificateRecord>,
log_entries: Vec<SshDistributionLogEntry>,
high_water_ms: i64, high_water_ms: i64,
allowed: bool, allowed: bool,
reason: String, reason: String,
@ -1190,6 +1192,7 @@ pub enum PeerControlResponse {
endpoint_id: String, endpoint_id: String,
remote_endpoint_id: String, remote_endpoint_id: String,
revocations: Vec<SshRevocationEntry>, revocations: Vec<SshRevocationEntry>,
log_entries: Vec<SshDistributionLogEntry>,
high_water_ms: i64, high_water_ms: i64,
allowed: bool, allowed: bool,
reason: String, reason: String,
@ -2316,6 +2319,7 @@ mod tests {
remote_endpoint_id: "endpoint:caller".to_owned(), remote_endpoint_id: "endpoint:caller".to_owned(),
requests: Vec::new(), requests: Vec::new(),
certificates: Vec::new(), certificates: Vec::new(),
log_entries: Vec::new(),
high_water_ms: 42, high_water_ms: 42,
allowed: false, allowed: false,
reason: "no grant".to_owned(), reason: "no grant".to_owned(),
@ -2335,6 +2339,7 @@ mod tests {
endpoint_id: "endpoint:peer".to_owned(), endpoint_id: "endpoint:peer".to_owned(),
remote_endpoint_id: "endpoint:caller".to_owned(), remote_endpoint_id: "endpoint:caller".to_owned(),
revocations: Vec::new(), revocations: Vec::new(),
log_entries: Vec::new(),
high_water_ms: 42, high_water_ms: 42,
allowed: false, allowed: false,
reason: "no grant".to_owned(), reason: "no grant".to_owned(),

View file

@ -35,9 +35,9 @@ use geth_secrets::{BearerAccess, BearerChallenge, BearerProof, ResourceMasterSec
use geth_ssh_identity::{ use geth_ssh_identity::{
SSH_CERT_ISSUANCE_NAMESPACE, SSH_CERT_REQUEST_NAMESPACE, SSH_REVOCATION_LIST_NAMESPACE, SSH_CERT_ISSUANCE_NAMESPACE, SSH_CERT_REQUEST_NAMESPACE, SSH_REVOCATION_LIST_NAMESPACE,
SshCertApproval, SshCertKind, SshCertRequest, SshCertRequestStatus, SshCertificateRecord, SshCertApproval, SshCertKind, SshCertRequest, SshCertRequestStatus, SshCertificateRecord,
SshRecordProvenance, SshRevocationEntry, SshRevocationExportFormat, SshRevocationKind, SshDistributionLogEntry, SshRecordProvenance, SshRevocationEntry, SshRevocationExportFormat,
build_ssh_cert_sign_command, cert_request_id, certificate_id, openssh_krl_spec, SshRevocationKind, build_ssh_cert_sign_command, cert_request_id, certificate_id,
parse_openssh_krl_spec, revocation_id, ssh_cert_request_signing_payload, openssh_krl_spec, parse_openssh_krl_spec, revocation_id, ssh_cert_request_signing_payload,
ssh_certificate_signing_payload, ssh_public_key_fingerprint, ssh_revocation_signing_payload, ssh_certificate_signing_payload, ssh_public_key_fingerprint, ssh_revocation_signing_payload,
write_openssh_krl, write_openssh_krl,
}; };
@ -1731,6 +1731,7 @@ async fn ssh_cert_sync_from_peer(
endpoint_id, endpoint_id,
requests, requests,
certificates, certificates,
log_entries,
high_water_ms, high_water_ms,
allowed, allowed,
reason, reason,
@ -1750,18 +1751,13 @@ async fn ssh_cert_sync_from_peer(
}); });
} }
let store = Store::open(&node.paths.metadata_db())?; let store = Store::open(&node.paths.metadata_db())?;
let mut requests_imported = 0; let entries = if log_entries.is_empty() {
let mut certificates_imported = 0; ssh_cert_distribution_log_entries(requests, certificates)
for request in &requests { } else {
if insert_ssh_cert_request_if_not_conflicting(&store, request)? { log_entries
requests_imported += 1; };
} let (requests_imported, certificates_imported, _) =
} import_ssh_distribution_log_entries(&store, &entries)?;
for certificate in &certificates {
if insert_ssh_certificate_if_not_conflicting(&store, certificate)? {
certificates_imported += 1;
}
}
store_live_sync_cursor(&store, peer_node, "ssh-certs", high_water_ms)?; store_live_sync_cursor(&store, peer_node, "ssh-certs", high_water_ms)?;
Ok(ControlResponse::SshCertSynced { Ok(ControlResponse::SshCertSynced {
peer_node_id: node_id, peer_node_id: node_id,
@ -1814,6 +1810,7 @@ async fn ssh_revocation_sync_from_peer(
agent_id, agent_id,
endpoint_id, endpoint_id,
revocations, revocations,
log_entries,
high_water_ms, high_water_ms,
allowed, allowed,
reason, reason,
@ -1832,12 +1829,13 @@ async fn ssh_revocation_sync_from_peer(
}); });
} }
let store = Store::open(&node.paths.metadata_db())?; let store = Store::open(&node.paths.metadata_db())?;
let mut revocations_imported = 0; let entries = if log_entries.is_empty() {
for revocation in &revocations { ssh_revocation_distribution_log_entries(revocations)
if insert_ssh_revocation_if_not_conflicting(&store, revocation)? { } else {
revocations_imported += 1; log_entries
} };
} let (_, _, revocations_imported) =
import_ssh_distribution_log_entries(&store, &entries)?;
store_live_sync_cursor(&store, peer_node, "ssh-revocations", high_water_ms)?; store_live_sync_cursor(&store, peer_node, "ssh-revocations", high_water_ms)?;
Ok(ControlResponse::SshRevocationSynced { Ok(ControlResponse::SshRevocationSynced {
peer_node_id: node_id, peer_node_id: node_id,
@ -5259,7 +5257,7 @@ async fn handle_iroh_control_connection(
&nonce, &nonce,
bearer_proof.as_ref(), bearer_proof.as_ref(),
)?; )?;
let (requests, certificates) = if explanation.allowed { let (requests, certificates, log_entries) = if explanation.allowed {
( (
store store
.list_ssh_cert_requests_since(since_ms)? .list_ssh_cert_requests_since(since_ms)?
@ -5271,9 +5269,15 @@ async fn handle_iroh_control_connection(
.into_iter() .into_iter()
.map(ssh_certificate_from_stored) .map(ssh_certificate_from_stored)
.collect::<Result<Vec<_>, _>>()?, .collect::<Result<Vec<_>, _>>()?,
Vec::new(),
) )
} else { } else {
(Vec::new(), Vec::new()) (Vec::new(), Vec::new(), Vec::new())
};
let log_entries = if explanation.allowed {
ssh_cert_distribution_log_entries(requests.clone(), certificates.clone())
} else {
log_entries
}; };
PeerControlResponse::SshCertSynced { PeerControlResponse::SshCertSynced {
node_id: node.node_id.clone(), node_id: node.node_id.clone(),
@ -5282,6 +5286,7 @@ async fn handle_iroh_control_connection(
remote_endpoint_id, remote_endpoint_id,
requests, requests,
certificates, certificates,
log_entries,
high_water_ms, high_water_ms,
allowed: explanation.allowed, allowed: explanation.allowed,
reason: explanation.reason, reason: explanation.reason,
@ -5320,14 +5325,19 @@ async fn handle_iroh_control_connection(
&nonce, &nonce,
bearer_proof.as_ref(), bearer_proof.as_ref(),
)?; )?;
let revocations = if explanation.allowed { let (revocations, log_entries) = if explanation.allowed {
store store
.list_ssh_revocations_since(since_ms)? .list_ssh_revocations_since(since_ms)?
.into_iter() .into_iter()
.map(ssh_revocation_from_stored) .map(ssh_revocation_from_stored)
.collect::<Result<Vec<_>, _>>()? .collect::<Result<Vec<_>, _>>()
.map(|revocations| {
let log_entries =
ssh_revocation_distribution_log_entries(revocations.clone());
(revocations, log_entries)
})?
} else { } else {
Vec::new() (Vec::new(), Vec::new())
}; };
PeerControlResponse::SshRevocationSynced { PeerControlResponse::SshRevocationSynced {
node_id: node.node_id.clone(), node_id: node.node_id.clone(),
@ -5335,6 +5345,7 @@ async fn handle_iroh_control_connection(
endpoint_id: node.iroh_status.endpoint_id.clone().unwrap_or_default(), endpoint_id: node.iroh_status.endpoint_id.clone().unwrap_or_default(),
remote_endpoint_id, remote_endpoint_id,
revocations, revocations,
log_entries,
high_water_ms, high_water_ms,
allowed: explanation.allowed, allowed: explanation.allowed,
reason: explanation.reason, reason: explanation.reason,
@ -10587,6 +10598,77 @@ fn insert_ssh_revocation_if_not_conflicting(
Ok(true) Ok(true)
} }
fn ssh_cert_distribution_log_entries(
requests: Vec<SshCertRequest>,
certificates: Vec<SshCertificateRecord>,
) -> Vec<SshDistributionLogEntry> {
let mut entries = requests
.into_iter()
.map(SshDistributionLogEntry::cert_request)
.chain(
certificates
.into_iter()
.map(SshDistributionLogEntry::certificate),
)
.collect::<Vec<_>>();
entries.sort_by(|left, right| {
left.timestamp()
.0
.cmp(&right.timestamp().0)
.then_with(|| left.entry_id().cmp(right.entry_id()))
});
entries
}
fn ssh_revocation_distribution_log_entries(
revocations: Vec<SshRevocationEntry>,
) -> Vec<SshDistributionLogEntry> {
let mut entries = revocations
.into_iter()
.map(SshDistributionLogEntry::revocation)
.collect::<Vec<_>>();
entries.sort_by(|left, right| {
left.timestamp()
.0
.cmp(&right.timestamp().0)
.then_with(|| left.entry_id().cmp(right.entry_id()))
});
entries
}
fn import_ssh_distribution_log_entries(
store: &Store,
entries: &[SshDistributionLogEntry],
) -> Result<(usize, usize, usize), NodeError> {
let mut requests_imported = 0;
let mut certificates_imported = 0;
let mut revocations_imported = 0;
for entry in entries {
match entry {
SshDistributionLogEntry::CertRequest { request, .. } => {
if insert_ssh_cert_request_if_not_conflicting(store, request)? {
requests_imported += 1;
}
}
SshDistributionLogEntry::Certificate { certificate, .. } => {
if insert_ssh_certificate_if_not_conflicting(store, certificate)? {
certificates_imported += 1;
}
}
SshDistributionLogEntry::Revocation { revocation, .. } => {
if insert_ssh_revocation_if_not_conflicting(store, revocation)? {
revocations_imported += 1;
}
}
}
}
Ok((
requests_imported,
certificates_imported,
revocations_imported,
))
}
fn ssh_revocation_from_stored( fn ssh_revocation_from_stored(
stored: StoredSshRevocation, stored: StoredSshRevocation,
) -> Result<SshRevocationEntry, NodeError> { ) -> Result<SshRevocationEntry, NodeError> {

View file

@ -220,6 +220,79 @@ pub struct SshRevocationEntry {
pub provenance: Option<SshRecordProvenance>, pub provenance: Option<SshRecordProvenance>,
} }
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "kebab-case")]
pub enum SshDistributionLogEntry {
CertRequest {
entry_id: String,
resource: String,
timestamp: UnixMillis,
request: SshCertRequest,
},
Certificate {
entry_id: String,
resource: String,
timestamp: UnixMillis,
certificate: SshCertificateRecord,
},
Revocation {
entry_id: String,
resource: String,
timestamp: UnixMillis,
revocation: SshRevocationEntry,
},
}
impl SshDistributionLogEntry {
#[must_use]
pub fn cert_request(request: SshCertRequest) -> Self {
Self::CertRequest {
entry_id: format!("ssh-log:cert-request:{}", request.id),
resource: "resource:ssh:certs".to_owned(),
timestamp: request.created_at,
request,
}
}
#[must_use]
pub fn certificate(certificate: SshCertificateRecord) -> Self {
Self::Certificate {
entry_id: format!("ssh-log:certificate:{}", certificate.id),
resource: "resource:ssh:certs".to_owned(),
timestamp: certificate.imported_at,
certificate,
}
}
#[must_use]
pub fn revocation(revocation: SshRevocationEntry) -> Self {
Self::Revocation {
entry_id: format!("ssh-log:revocation:{}", revocation.id),
resource: "resource:ssh:revocations".to_owned(),
timestamp: revocation.created_at,
revocation,
}
}
#[must_use]
pub fn timestamp(&self) -> UnixMillis {
match self {
Self::CertRequest { timestamp, .. }
| Self::Certificate { timestamp, .. }
| Self::Revocation { timestamp, .. } => *timestamp,
}
}
#[must_use]
pub fn entry_id(&self) -> &str {
match self {
Self::CertRequest { entry_id, .. }
| Self::Certificate { entry_id, .. }
| Self::Revocation { entry_id, .. } => entry_id,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SshCertRequestSigningPayload { pub struct SshCertRequestSigningPayload {
pub id: SshCertRequestId, pub id: SshCertRequestId,
@ -547,6 +620,31 @@ mod tests {
assert_eq!(decoded, request); assert_eq!(decoded, request);
} }
#[test]
fn ssh_distribution_log_entries_roundtrip_with_stable_ids() {
let request = SshCertRequest {
id: "ssh-cert-request:1".into(),
requester_node: "node:laptop".into(),
public_key: "ssh-ed25519 AAAA test".to_owned(),
public_key_fingerprint: ssh_public_key_fingerprint("ssh-ed25519 AAAA test"),
cert_kind: SshCertKind::User,
principals: vec!["eric".to_owned()],
requested_validity: Some("+52w".to_owned()),
renewal_of: None,
reason: Some("bootstrap".to_owned()),
status: SshCertRequestStatus::Pending,
created_at: UnixMillis(10),
provenance: None,
};
let entry = SshDistributionLogEntry::cert_request(request);
assert_eq!(entry.entry_id(), "ssh-log:cert-request:ssh-cert-request:1");
assert_eq!(entry.timestamp(), UnixMillis(10));
let json = serde_json::to_string(&entry).expect("json");
let decoded: SshDistributionLogEntry = serde_json::from_str(&json).expect("decode");
assert_eq!(decoded, entry);
}
#[test] #[test]
fn sign_command_includes_host_flag_for_host_certs() { fn sign_command_includes_host_flag_for_host_certs() {
let request = SshCertRequest { let request = SshCertRequest {

View file

@ -105,16 +105,20 @@ or pass `--sign` to execute that command immediately and import the resulting
certificate into local metadata for distribution. This relies on the local certificate into local metadata for distribution. This relies on the local
OpenSSH ecosystem, so hardware-backed keys remain mediated by `ssh-keygen` and OpenSSH ecosystem, so hardware-backed keys remain mediated by `ssh-keygen` and
the host's agent/security-key flow. Certificate and key revocations are stored the host's agent/security-key flow. Certificate and key revocations are stored
as signed-list-ready records. The bootstrap can pull as signed-list-ready records. Sync materializes those records as ordered SSH
certificate-flow metadata over Iroh with `geth ssh cert sync <node-id>` when the distribution log entries: certificate requests, certificate imports, and
peer grants `ssh_cert.sync` on `resource:ssh:certs`, and revocation metadata with revocations each have stable log entry IDs and timestamps, and the receiver
reduces the entries into local state after provenance and conflict checks. The
bootstrap can pull certificate-flow log entries over Iroh with
`geth ssh cert sync <node-id>` when the peer grants `ssh_cert.sync` on
`resource:ssh:certs`, and revocation log entries with
`geth ssh revocation sync <node-id>` when the peer grants `ssh_revocation.sync` `geth ssh revocation sync <node-id>` when the peer grants `ssh_revocation.sync`
on `resource:ssh:revocations`. The daemon also runs a configurable background on `resource:ssh:revocations`. The daemon also runs a configurable background
live-sync tick for known peers and records per-peer high-water cursors in live-sync tick for known peers and records per-peer high-water cursors in
`module_state`, so repeated ticks request only records at or beyond the last `module_state`, so repeated ticks request only entries at or beyond the last
remote cursor. The default interval is 30 seconds and can be changed under remote cursor. The default interval is 30 seconds and can be changed under
`[sync]` in `config.toml`. Boundary duplicates are harmless because records are `[sync]` in `config.toml`. Boundary duplicates are harmless because records are
keyed by stable IDs and inserted with replace semantics. keyed by stable IDs.
Before issuing per-module pulls, the daemon can request an authorized sync Before issuing per-module pulls, the daemon can request an authorized sync
status summary over the same protected Iroh control ALPN. The serving peer status summary over the same protected Iroh control ALPN. The serving peer
validates endpoint/card binding and returns only watermarks for streams where validates endpoint/card binding and returns only watermarks for streams where
@ -278,16 +282,16 @@ specification text. Certificate approval normally emits the exact
the resulting OpenSSH certificate, and mark the request signed. It can also the resulting OpenSSH certificate, and mark the request signed. It can also
invoke `ssh-keygen -k` to produce a binary OpenSSH KRL; serial and key-ID KRL invoke `ssh-keygen -k` to produce a binary OpenSSH KRL; serial and key-ID KRL
entries require a CA public key via `--ca-public`, matching OpenSSH behavior. entries require a CA public key via `--ca-public`, matching OpenSSH behavior.
It can import geth JSONL revocation It can import geth JSONL revocation exports and OpenSSH KRL specification source
exports and OpenSSH KRL specification source files. Binary OpenSSH KRL files are files. Binary OpenSSH KRL files are not enumerable through OpenSSH tooling, so
not enumerable through OpenSSH tooling, so geth treats binary import as geth treats binary import as unsupported and asks for JSONL or the spec source.
unsupported and asks for JSONL or the spec source. Revocation lists are not yet Revocation lists are not yet full CRDT-replicated resources, but the daemon now
full CRDT-replicated resources, but the daemon can already pull cert-flow and syncs certificate-flow and revocation state as a small ordered resource log over
revocation metadata from authorized peers over the protected Iroh control ALPN. the protected Iroh control ALPN. Manual sync commands and the background
Manual sync commands and the background live-sync loop share the same capability live-sync loop share the same capability checks and cursor state. Sync import
checks and cursor state. Sync import rejects conflicting records with ids that rejects conflicting records with ids that already exist locally instead of
already exist locally instead of replacing local metadata. Local SSH certificate replacing local metadata. Local SSH certificate and revocation metadata commands
and revocation metadata commands also accept an optional subject principal for also accept an optional subject principal for
authorization testing: non-owner subjects must hold `ssh_cert.*` capabilities on authorization testing: non-owner subjects must hold `ssh_cert.*` capabilities on
`resource:ssh:certs` or `ssh_revocation.*` capabilities on `resource:ssh:certs` or `ssh_revocation.*` capabilities on
`resource:ssh:revocations` before requests, approval/import/read operations, or `resource:ssh:revocations` before requests, approval/import/read operations, or

View file

@ -593,7 +593,7 @@ Goal: add authorized stream-oriented management workflows over Iroh.
only after authorization. only after authorization.
- `[x]` Future completion adds a restricted built-in geth admin shell option. - `[x]` Future completion adds a restricted built-in geth admin shell option.
- `[~]` SSH certificate and revocation distribution. - `[x]` SSH certificate and revocation distribution.
Acceptance criteria: Acceptance criteria:
- `[x]` Cert request and imported certificate records can be pulled from an - `[x]` Cert request and imported certificate records can be pulled from an
imported peer over Iroh. imported peer over Iroh.
@ -607,7 +607,10 @@ Goal: add authorized stream-oriented management workflows over Iroh.
offline after sync. offline after sync.
- `[x]` Background live-sync uses the same protected Iroh path and cursor - `[x]` Background live-sync uses the same protected Iroh path and cursor
state as manual sync. state as manual sync.
- `[ ]` Replace pull-only metadata sync with a resource log or CRDT model. - `[x]` Replace pull-only metadata sync with a resource log or CRDT model.
- `[x]` Sync responses carry ordered SSH distribution log entries derived
from signed cert requests, signed certificate imports, and signed
revocation records.
- `[x]` Conflicting records with already-known ids are rejected during import - `[x]` Conflicting records with already-known ids are rejected during import
rather than replacing local metadata. rather than replacing local metadata.
- `[x]` Unsigned records are rejected during sync import once signed - `[x]` Unsigned records are rejected during sync import once signed