Add sync health and explicit sync trigger
This commit is contained in:
parent
1336fa38e8
commit
3c5e95fb87
8 changed files with 1085 additions and 98 deletions
|
|
@ -109,9 +109,10 @@ Roadmap items should be actionable and checkable:
|
|||
and `geth peer auth-check` over Iroh, signed peer-card LAN discovery payloads,
|
||||
authorized `geth cas fetch`, `geth ssh cert sync`, and
|
||||
`geth ssh revocation sync` over the Iroh control ALPN, a background SSH
|
||||
metadata and KV live-sync loop with per-peer cursors in `module_state` and an
|
||||
authorized sync-status summary that lets peers disclose only permitted stream
|
||||
watermarks before module pulls,
|
||||
metadata, keychain/auth, KV, DB, document, and file-root live-sync loop with
|
||||
per-peer cursors in `module_state`, `geth sync status`/`geth sync now`, and
|
||||
an authorized sync-status summary that lets peers disclose only permitted
|
||||
stream watermarks before module pulls,
|
||||
untrusted discovery-backend trait, custom relay-map config, and Iroh
|
||||
local-network discovery toggle exist.
|
||||
- Canonical signed-operation envelopes exist for keychain/auth signature
|
||||
|
|
|
|||
|
|
@ -116,6 +116,8 @@ The bootstrap implementation provides:
|
|||
- `geth keychain status`
|
||||
- `geth keychain sync <node-id-or-name>`
|
||||
- `geth auth sync <node-id-or-name>`
|
||||
- `geth sync status`
|
||||
- `geth sync now [node-id-or-name]`
|
||||
- `geth secret status`
|
||||
- `geth secret create <resource>`
|
||||
- `geth secret rotate <resource>`
|
||||
|
|
@ -412,6 +414,13 @@ from a currently trusted admin key over the canonical keychain payload. This is
|
|||
the current replicated device-management substrate. It is still a pull-based
|
||||
operation log, not yet a CRDT or Keyhive-style convergent authority.
|
||||
|
||||
The daemon also runs best-effort live sync for imported peers. `geth sync now
|
||||
[node]` triggers the same sync pass immediately, and `geth sync status` reports
|
||||
the last local attempt, success, cursor, import count, rejection count, and
|
||||
error per peer stream. Keychain and auth sync now use per-peer high-water
|
||||
cursors, while receivers still verify every imported signed operation before it
|
||||
can affect the reduced keychain or authorization views.
|
||||
|
||||
## Authorization Direction
|
||||
|
||||
The MVP defines the split between:
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ pub enum Command {
|
|||
command: DaemonCommand,
|
||||
},
|
||||
Status,
|
||||
Sync {
|
||||
#[command(subcommand)]
|
||||
command: SyncCommand,
|
||||
},
|
||||
Node {
|
||||
#[command(subcommand)]
|
||||
command: NodeCommand,
|
||||
|
|
@ -134,6 +138,12 @@ pub enum ServiceCommand {
|
|||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
pub enum SyncCommand {
|
||||
Status,
|
||||
Now { node: Option<String> },
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
pub enum NodeCommand {
|
||||
Id,
|
||||
|
|
@ -825,6 +835,12 @@ pub async fn run() -> Result<()> {
|
|||
fn request_for_command(command: Command) -> Result<ControlRequest> {
|
||||
Ok(match command {
|
||||
Command::Status => ControlRequest::Status,
|
||||
Command::Sync {
|
||||
command: SyncCommand::Status,
|
||||
} => ControlRequest::SyncStatus,
|
||||
Command::Sync {
|
||||
command: SyncCommand::Now { node },
|
||||
} => ControlRequest::SyncNow { node },
|
||||
Command::Node {
|
||||
command: NodeCommand::Id,
|
||||
} => ControlRequest::NodeId,
|
||||
|
|
@ -1851,6 +1867,7 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
|
|||
ops_imported,
|
||||
signatures_imported,
|
||||
invalid_ops_rejected,
|
||||
high_water_ms,
|
||||
note,
|
||||
} => {
|
||||
println!("synced keychain from: {peer_node_id}");
|
||||
|
|
@ -1859,6 +1876,7 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
|
|||
println!("ops_imported: {ops_imported}");
|
||||
println!("signatures_imported: {signatures_imported}");
|
||||
println!("invalid_ops_rejected: {invalid_ops_rejected}");
|
||||
println!("high_water_ms: {high_water_ms}");
|
||||
println!("note: {note}");
|
||||
}
|
||||
ControlResponse::AuthSynced {
|
||||
|
|
@ -1868,6 +1886,7 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
|
|||
ops_imported,
|
||||
signatures_imported,
|
||||
invalid_ops_rejected,
|
||||
high_water_ms,
|
||||
note,
|
||||
} => {
|
||||
println!("synced auth from: {peer_node_id}");
|
||||
|
|
@ -1876,6 +1895,66 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
|
|||
println!("ops_imported: {ops_imported}");
|
||||
println!("signatures_imported: {signatures_imported}");
|
||||
println!("invalid_ops_rejected: {invalid_ops_rejected}");
|
||||
println!("high_water_ms: {high_water_ms}");
|
||||
println!("note: {note}");
|
||||
}
|
||||
ControlResponse::SyncStatus { peers, note } => {
|
||||
if peers.is_empty() {
|
||||
println!("no sync peers");
|
||||
} else {
|
||||
for peer in peers {
|
||||
println!("peer: {}", peer.peer_node_id);
|
||||
if peer.streams.is_empty() {
|
||||
println!(" no sync attempts recorded");
|
||||
}
|
||||
for stream in peer.streams {
|
||||
println!(
|
||||
" {}\tcursor={}\tlast_attempt={}\tlast_success={}\timported={}\trejected={}\terror={}",
|
||||
stream.stream,
|
||||
stream.cursor_ms,
|
||||
stream
|
||||
.last_attempt_ms
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "never".to_owned()),
|
||||
stream
|
||||
.last_success_ms
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "never".to_owned()),
|
||||
stream.last_imported,
|
||||
stream.last_rejected,
|
||||
stream.last_error.unwrap_or_else(|| "-".to_owned())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("note: {note}");
|
||||
}
|
||||
ControlResponse::SyncRan { peers, note } => {
|
||||
if peers.is_empty() {
|
||||
println!("no sync peers");
|
||||
} else {
|
||||
for peer in peers {
|
||||
println!("peer: {}", peer.peer_node_id);
|
||||
for stream in peer.streams {
|
||||
let state = if !stream.attempted {
|
||||
"skipped"
|
||||
} else if stream.success {
|
||||
"ok"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
println!(
|
||||
" {}\t{}\tcursor={}\timported={}\trejected={}\terror={}",
|
||||
stream.stream,
|
||||
state,
|
||||
stream.cursor_ms,
|
||||
stream.imported,
|
||||
stream.rejected,
|
||||
stream.error.unwrap_or_else(|| "-".to_owned())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("note: {note}");
|
||||
}
|
||||
ControlResponse::SecretStatus { secrets } => {
|
||||
|
|
|
|||
|
|
@ -189,6 +189,10 @@ pub enum ControlRequest {
|
|||
AuthSync {
|
||||
node: String,
|
||||
},
|
||||
SyncStatus,
|
||||
SyncNow {
|
||||
node: Option<String>,
|
||||
},
|
||||
SecretStatus,
|
||||
SecretCreate {
|
||||
resource: String,
|
||||
|
|
@ -576,6 +580,7 @@ pub enum ControlResponse {
|
|||
ops_imported: usize,
|
||||
signatures_imported: usize,
|
||||
invalid_ops_rejected: usize,
|
||||
high_water_ms: i64,
|
||||
note: String,
|
||||
},
|
||||
AuthSynced {
|
||||
|
|
@ -585,6 +590,15 @@ pub enum ControlResponse {
|
|||
ops_imported: usize,
|
||||
signatures_imported: usize,
|
||||
invalid_ops_rejected: usize,
|
||||
high_water_ms: i64,
|
||||
note: String,
|
||||
},
|
||||
SyncStatus {
|
||||
peers: Vec<SyncPeerStatus>,
|
||||
note: String,
|
||||
},
|
||||
SyncRan {
|
||||
peers: Vec<SyncPeerRun>,
|
||||
note: String,
|
||||
},
|
||||
SecretStatus {
|
||||
|
|
@ -953,10 +967,12 @@ pub enum PeerControlRequest {
|
|||
},
|
||||
KeychainSync {
|
||||
peer_card: PeerCard,
|
||||
since_ms: i64,
|
||||
nonce: String,
|
||||
},
|
||||
AuthSync {
|
||||
peer_card: PeerCard,
|
||||
since_ms: i64,
|
||||
nonce: String,
|
||||
},
|
||||
NodeEnrollmentSubmit {
|
||||
|
|
@ -1089,6 +1105,7 @@ pub enum PeerControlResponse {
|
|||
remote_endpoint_id: String,
|
||||
ops: Vec<KeychainOp>,
|
||||
signatures: Vec<KeychainOpSignature>,
|
||||
high_water_ms: i64,
|
||||
nonce: String,
|
||||
note: String,
|
||||
},
|
||||
|
|
@ -1099,6 +1116,7 @@ pub enum PeerControlResponse {
|
|||
remote_endpoint_id: String,
|
||||
ops: Vec<AuthOp>,
|
||||
signatures: Vec<AuthOpSignature>,
|
||||
high_water_ms: i64,
|
||||
nonce: String,
|
||||
note: String,
|
||||
},
|
||||
|
|
@ -1351,6 +1369,40 @@ pub struct SyncWatermark {
|
|||
pub high_water: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SyncPeerStatus {
|
||||
pub peer_node_id: String,
|
||||
pub streams: Vec<SyncStreamStatus>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SyncStreamStatus {
|
||||
pub stream: String,
|
||||
pub cursor_ms: i64,
|
||||
pub last_attempt_ms: Option<i64>,
|
||||
pub last_success_ms: Option<i64>,
|
||||
pub last_error: Option<String>,
|
||||
pub last_imported: usize,
|
||||
pub last_rejected: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SyncPeerRun {
|
||||
pub peer_node_id: String,
|
||||
pub streams: Vec<SyncStreamRun>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SyncStreamRun {
|
||||
pub stream: String,
|
||||
pub attempted: bool,
|
||||
pub success: bool,
|
||||
pub imported: usize,
|
||||
pub rejected: usize,
|
||||
pub cursor_ms: i64,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ControlError {
|
||||
#[error("json error: {0}")]
|
||||
|
|
@ -1507,6 +1559,7 @@ mod tests {
|
|||
ops_imported: 2,
|
||||
signatures_imported: 2,
|
||||
invalid_ops_rejected: 1,
|
||||
high_water_ms: 42,
|
||||
note: "trusted admin signatures only".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
|
|
@ -1535,6 +1588,7 @@ mod tests {
|
|||
ops_imported: 1,
|
||||
signatures_imported: 1,
|
||||
invalid_ops_rejected: 0,
|
||||
high_water_ms: 42,
|
||||
note: "trusted admin signatures only".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
|
|
@ -1542,6 +1596,46 @@ mod tests {
|
|||
response
|
||||
);
|
||||
|
||||
let response = ControlResponse::SyncStatus {
|
||||
peers: vec![SyncPeerStatus {
|
||||
peer_node_id: "node:peer".to_owned(),
|
||||
streams: vec![SyncStreamStatus {
|
||||
stream: "keychain".to_owned(),
|
||||
cursor_ms: 42,
|
||||
last_attempt_ms: Some(43),
|
||||
last_success_ms: Some(43),
|
||||
last_error: None,
|
||||
last_imported: 2,
|
||||
last_rejected: 0,
|
||||
}],
|
||||
}],
|
||||
note: "local health".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
decode_response(&encode_response(&response).expect("encode")).expect("decode"),
|
||||
response
|
||||
);
|
||||
|
||||
let response = ControlResponse::SyncRan {
|
||||
peers: vec![SyncPeerRun {
|
||||
peer_node_id: "node:peer".to_owned(),
|
||||
streams: vec![SyncStreamRun {
|
||||
stream: "auth".to_owned(),
|
||||
attempted: true,
|
||||
success: true,
|
||||
imported: 1,
|
||||
rejected: 0,
|
||||
cursor_ms: 44,
|
||||
error: None,
|
||||
}],
|
||||
}],
|
||||
note: "ran".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
decode_response(&encode_response(&response).expect("encode")).expect("decode"),
|
||||
response
|
||||
);
|
||||
|
||||
let request = ControlRequest::SecretBearerVerify {
|
||||
secret: "bearer:test".to_owned(),
|
||||
resource: "resource:cas:local".to_owned(),
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -905,6 +905,28 @@ impl Store {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn list_module_states_with_prefix(
|
||||
&self,
|
||||
prefix: &str,
|
||||
) -> Result<Vec<StoredModuleState>, StoreError> {
|
||||
let like = format!("{prefix}%");
|
||||
let mut stmt = self.conn.prepare(
|
||||
r#"SELECT module, state_json, updated_at_ms
|
||||
FROM module_state
|
||||
WHERE module LIKE ?1
|
||||
ORDER BY module"#,
|
||||
)?;
|
||||
let rows = stmt.query_map(params![like], |row| {
|
||||
Ok(StoredModuleState {
|
||||
module: row.get(0)?,
|
||||
state_json: row.get(1)?,
|
||||
updated_at_ms: row.get(2)?,
|
||||
})
|
||||
})?;
|
||||
rows.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(StoreError::from)
|
||||
}
|
||||
|
||||
pub fn insert_auth_op(&self, op: &StoredAuthOp) -> Result<(), StoreError> {
|
||||
self.conn.execute(
|
||||
r#"INSERT OR REPLACE INTO auth_ops(op_id, resource_id, op_json, created_at_ms)
|
||||
|
|
@ -1672,12 +1694,26 @@ mod tests {
|
|||
updated_at_ms: 43,
|
||||
};
|
||||
store.put_module_state(&state).expect("put state");
|
||||
store
|
||||
.put_module_state(&StoredModuleState {
|
||||
module: "live-sync-status:node:laptop:ssh-certs".to_owned(),
|
||||
state_json: r#"{"last_attempt_ms":44}"#.to_owned(),
|
||||
updated_at_ms: 44,
|
||||
})
|
||||
.expect("put status state");
|
||||
assert_eq!(
|
||||
store
|
||||
.get_module_state("live-sync:node:laptop:ssh-certs")
|
||||
.expect("get state"),
|
||||
Some(state)
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.list_module_states_with_prefix("live-sync-status:node:laptop:")
|
||||
.expect("list prefix")
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -106,6 +106,12 @@ status summary over the same protected Iroh control ALPN. The serving peer
|
|||
validates endpoint/card binding and returns only watermarks for streams where
|
||||
the caller already has the required resource capability, which reduces blind
|
||||
polling without letting discovery reveal private resource names.
|
||||
Keychain and auth operation logs are also advertised through this watermark
|
||||
path. Pulls are delta-style by per-peer cursor, but every received operation is
|
||||
still verified against trusted-admin OpenSSH signatures before import.
|
||||
Operators can run `geth sync now [node]` to trigger the same best-effort pass
|
||||
immediately and `geth sync status` to inspect locally recorded last-attempt,
|
||||
last-success, cursor, import/rejection counts, and errors for each peer stream.
|
||||
|
||||
## Resource Model
|
||||
|
||||
|
|
|
|||
|
|
@ -150,7 +150,12 @@ geth-to-geth connections without granting trust from discovery alone.
|
|||
the relevant resource capability.
|
||||
- `[x]` Background live-sync skips per-module pulls when the authorized remote
|
||||
watermark has not advanced.
|
||||
- `[x]` `geth sync status` reports last local attempt, success, cursor,
|
||||
import/rejection counts, and error for each recorded peer stream.
|
||||
- `[x]` `geth sync now [node]` triggers the same best-effort sync pass that
|
||||
background live sync uses.
|
||||
- `[x]` Tests verify unauthorized streams are omitted from summary output.
|
||||
- `[x]` Tests verify persisted stream health is exposed in local sync status.
|
||||
|
||||
## Phase 2: Trust And Authorization
|
||||
|
||||
|
|
@ -188,6 +193,8 @@ resource-scoped capability decisions.
|
|||
- `[x]` `geth node rename/revoke` require an admin signing key.
|
||||
- `[x]` `geth keychain sync <node>` verifies signatures from currently
|
||||
trusted admin keys before accepting keychain ops.
|
||||
- `[x]` Keychain live sync advertises and consumes per-peer high-water
|
||||
cursors instead of blindly re-requesting the full log on every tick.
|
||||
- `[x]` `geth node enroll request` creates an agent-key-signed enrollment
|
||||
request with requested node name and capabilities.
|
||||
- `[x]` `geth node enroll submit/import/list` moves pending enrollment
|
||||
|
|
@ -217,6 +224,8 @@ resource-scoped capability decisions.
|
|||
- `[x]` Enrollment approval signs capability grants as auth ops.
|
||||
- `[x]` `geth auth sync <node>` imports only auth ops signed by currently
|
||||
trusted admin keys.
|
||||
- `[x]` Auth live sync advertises and consumes per-peer high-water cursors
|
||||
instead of blindly re-requesting the full log on every tick.
|
||||
- `[x]` `geth auth grant/revoke --signing-key` records signed auth ops.
|
||||
|
||||
- `[x]` Resource auth operation reducer.
|
||||
|
|
|
|||
Loading…
Reference in a new issue