eric/geth
Watch
1
0
Fork
You've already forked geth
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add sync health and explicit sync trigger

Browse source
This commit is contained in:
Eric Wendland 2026-05-21 19:31:05 +02:00
commit 3c5e95fb87
8 changed files with 1085 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

79
crates/geth-cli/src/lib.rs
View file

@ -37,6 +37,10 @@ pub enum Command {
command: DaemonCommand,
},
Status,
Sync {
#[command(subcommand)]
command: SyncCommand,
},
Node {
#[command(subcommand)]
command: NodeCommand,
@ -134,6 +138,12 @@ pub enum ServiceCommand {
},
}
#[derive(Debug, Subcommand)]
pub enum SyncCommand {
Status,
Now { node: Option<String> },
}
#[derive(Debug, Subcommand)]
pub enum NodeCommand {
Id,
@ -825,6 +835,12 @@ pub async fn run() -> Result<()> {
fn request_for_command(command: Command) -> Result<ControlRequest> {
Ok(match command {
Command::Status => ControlRequest::Status,
Command::Sync {
command: SyncCommand::Status,
} => ControlRequest::SyncStatus,
Command::Sync {
command: SyncCommand::Now { node },
} => ControlRequest::SyncNow { node },
Command::Node {
command: NodeCommand::Id,
} => ControlRequest::NodeId,
@ -1851,6 +1867,7 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
ops_imported,
signatures_imported,
invalid_ops_rejected,
high_water_ms,
note,
} => {
println!("synced keychain from: {peer_node_id}");
@ -1859,6 +1876,7 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
println!("ops_imported: {ops_imported}");
println!("signatures_imported: {signatures_imported}");
println!("invalid_ops_rejected: {invalid_ops_rejected}");
println!("high_water_ms: {high_water_ms}");
println!("note: {note}");
}
ControlResponse::AuthSynced {
@ -1868,6 +1886,7 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
ops_imported,
signatures_imported,
invalid_ops_rejected,
high_water_ms,
note,
} => {
println!("synced auth from: {peer_node_id}");
@ -1876,6 +1895,66 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
println!("ops_imported: {ops_imported}");
println!("signatures_imported: {signatures_imported}");
println!("invalid_ops_rejected: {invalid_ops_rejected}");
println!("high_water_ms: {high_water_ms}");
println!("note: {note}");
}
ControlResponse::SyncStatus { peers, note } => {
if peers.is_empty() {
println!("no sync peers");
} else {
for peer in peers {
println!("peer: {}", peer.peer_node_id);
if peer.streams.is_empty() {
println!(" no sync attempts recorded");
}
for stream in peer.streams {
println!(
" {}\tcursor={}\tlast_attempt={}\tlast_success={}\timported={}\trejected={}\terror={}",
stream.stream,
stream.cursor_ms,
stream
.last_attempt_ms
.map(|value| value.to_string())
.unwrap_or_else(|| "never".to_owned()),
stream
.last_success_ms
.map(|value| value.to_string())
.unwrap_or_else(|| "never".to_owned()),
stream.last_imported,
stream.last_rejected,
stream.last_error.unwrap_or_else(|| "-".to_owned())
);
}
}
}
println!("note: {note}");
}
ControlResponse::SyncRan { peers, note } => {
if peers.is_empty() {
println!("no sync peers");
} else {
for peer in peers {
println!("peer: {}", peer.peer_node_id);
for stream in peer.streams {
let state = if !stream.attempted {
"skipped"
} else if stream.success {
"ok"
} else {
"failed"
};
println!(
" {}\t{}\tcursor={}\timported={}\trejected={}\terror={}",
stream.stream,
state,
stream.cursor_ms,
stream.imported,
stream.rejected,
stream.error.unwrap_or_else(|| "-".to_owned())
);
}
}
}
println!("note: {note}");
}
ControlResponse::SecretStatus { secrets } => {

94
crates/geth-control/src/lib.rs
View file

@ -189,6 +189,10 @@ pub enum ControlRequest {
AuthSync {
node: String,
},
SyncStatus,
SyncNow {
node: Option<String>,
},
SecretStatus,
SecretCreate {
resource: String,
@ -576,6 +580,7 @@ pub enum ControlResponse {
ops_imported: usize,
signatures_imported: usize,
invalid_ops_rejected: usize,
high_water_ms: i64,
note: String,
},
AuthSynced {
@ -585,6 +590,15 @@ pub enum ControlResponse {
ops_imported: usize,
signatures_imported: usize,
invalid_ops_rejected: usize,
high_water_ms: i64,
note: String,
},
SyncStatus {
peers: Vec<SyncPeerStatus>,
note: String,
},
SyncRan {
peers: Vec<SyncPeerRun>,
note: String,
},
SecretStatus {
@ -953,10 +967,12 @@ pub enum PeerControlRequest {
},
KeychainSync {
peer_card: PeerCard,
since_ms: i64,
nonce: String,
},
AuthSync {
peer_card: PeerCard,
since_ms: i64,
nonce: String,
},
NodeEnrollmentSubmit {
@ -1089,6 +1105,7 @@ pub enum PeerControlResponse {
remote_endpoint_id: String,
ops: Vec<KeychainOp>,
signatures: Vec<KeychainOpSignature>,
high_water_ms: i64,
nonce: String,
note: String,
},
@ -1099,6 +1116,7 @@ pub enum PeerControlResponse {
remote_endpoint_id: String,
ops: Vec<AuthOp>,
signatures: Vec<AuthOpSignature>,
high_water_ms: i64,
nonce: String,
note: String,
},
@ -1351,6 +1369,40 @@ pub struct SyncWatermark {
pub high_water: i64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SyncPeerStatus {
pub peer_node_id: String,
pub streams: Vec<SyncStreamStatus>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SyncStreamStatus {
pub stream: String,
pub cursor_ms: i64,
pub last_attempt_ms: Option<i64>,
pub last_success_ms: Option<i64>,
pub last_error: Option<String>,
pub last_imported: usize,
pub last_rejected: usize,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SyncPeerRun {
pub peer_node_id: String,
pub streams: Vec<SyncStreamRun>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SyncStreamRun {
pub stream: String,
pub attempted: bool,
pub success: bool,
pub imported: usize,
pub rejected: usize,
pub cursor_ms: i64,
pub error: Option<String>,
}
#[derive(Debug, thiserror::Error)]
pub enum ControlError {
#[error("json error: {0}")]
@ -1507,6 +1559,7 @@ mod tests {
ops_imported: 2,
signatures_imported: 2,
invalid_ops_rejected: 1,
high_water_ms: 42,
note: "trusted admin signatures only".to_owned(),
};
assert_eq!(
@ -1535,6 +1588,7 @@ mod tests {
ops_imported: 1,
signatures_imported: 1,
invalid_ops_rejected: 0,
high_water_ms: 42,
note: "trusted admin signatures only".to_owned(),
};
assert_eq!(
@ -1542,6 +1596,46 @@ mod tests {
response
);
let response = ControlResponse::SyncStatus {
peers: vec![SyncPeerStatus {
peer_node_id: "node:peer".to_owned(),
streams: vec![SyncStreamStatus {
stream: "keychain".to_owned(),
cursor_ms: 42,
last_attempt_ms: Some(43),
last_success_ms: Some(43),
last_error: None,
last_imported: 2,
last_rejected: 0,
}],
}],
note: "local health".to_owned(),
};
assert_eq!(
decode_response(&encode_response(&response).expect("encode")).expect("decode"),
response
);
let response = ControlResponse::SyncRan {
peers: vec![SyncPeerRun {
peer_node_id: "node:peer".to_owned(),
streams: vec![SyncStreamRun {
stream: "auth".to_owned(),
attempted: true,
success: true,
imported: 1,
rejected: 0,
cursor_ms: 44,
error: None,
}],
}],
note: "ran".to_owned(),
};
assert_eq!(
decode_response(&encode_response(&response).expect("encode")).expect("decode"),
response
);
let request = ControlRequest::SecretBearerVerify {
secret: "bearer:test".to_owned(),
resource: "resource:cas:local".to_owned(),

955
crates/geth-node/src/lib.rs
View file

File diff suppressed because it is too large Load diff

36
crates/geth-store/src/lib.rs
View file

@ -905,6 +905,28 @@ impl Store {
}
}
pub fn list_module_states_with_prefix(
&self,
prefix: &str,
) -> Result<Vec<StoredModuleState>, StoreError> {
let like = format!("{prefix}%");
let mut stmt = self.conn.prepare(
r#"SELECT module, state_json, updated_at_ms
FROM module_state
WHERE module LIKE ?1
ORDER BY module"#,
)?;
let rows = stmt.query_map(params![like], |row| {
Ok(StoredModuleState {
module: row.get(0)?,
state_json: row.get(1)?,
updated_at_ms: row.get(2)?,
})
})?;
rows.collect::<Result<Vec<_>, _>>()
.map_err(StoreError::from)
}
pub fn insert_auth_op(&self, op: &StoredAuthOp) -> Result<(), StoreError> {
self.conn.execute(
r#"INSERT OR REPLACE INTO auth_ops(op_id, resource_id, op_json, created_at_ms)
@ -1672,12 +1694,26 @@ mod tests {
updated_at_ms: 43,
};
store.put_module_state(&state).expect("put state");
store
.put_module_state(&StoredModuleState {
module: "live-sync-status:node:laptop:ssh-certs".to_owned(),
state_json: r#"{"last_attempt_ms":44}"#.to_owned(),
updated_at_ms: 44,
})
.expect("put status state");
assert_eq!(
store
.get_module_state("live-sync:node:laptop:ssh-certs")
.expect("get state"),
Some(state)
);
assert_eq!(
store
.list_module_states_with_prefix("live-sync-status:node:laptop:")
.expect("list prefix")
.len(),
1
);
}
#[test]