feat: add automation wait commands
This commit is contained in:
parent
448a831a98
commit
e49b945259
5 changed files with 261 additions and 7 deletions
|
|
@ -99,6 +99,7 @@ The bootstrap implementation provides:
|
||||||
- `geth daemon run`
|
- `geth daemon run`
|
||||||
- `geth daemon service install|uninstall|start|stop|status|print`
|
- `geth daemon service install|uninstall|start|stop|status|print`
|
||||||
- `geth status`
|
- `geth status`
|
||||||
|
- `geth wait daemon|peer|sync --timeout-ms <ms>`
|
||||||
- `geth node id`
|
- `geth node id`
|
||||||
- `geth node list`
|
- `geth node list`
|
||||||
- `geth node enroll request --node-name <name> --capability <resource=capability> [--out <path>]`
|
- `geth node enroll request --node-name <name> --capability <resource=capability> [--out <path>]`
|
||||||
|
|
@ -152,6 +153,9 @@ The bootstrap implementation provides:
|
||||||
- `geth auth sync <node-id-or-name>`
|
- `geth auth sync <node-id-or-name>`
|
||||||
- `geth sync status`
|
- `geth sync status`
|
||||||
- `geth sync now [node-id-or-name]`
|
- `geth sync now [node-id-or-name]`
|
||||||
|
- `geth wait daemon --timeout-ms <ms>`
|
||||||
|
- `geth wait peer <node-id-or-name> --timeout-ms <ms>`
|
||||||
|
- `geth wait sync <node-id-or-name> --timeout-ms <ms>`
|
||||||
- `geth secret status`
|
- `geth secret status`
|
||||||
- `geth secret create <resource>`
|
- `geth secret create <resource>`
|
||||||
- `geth secret rotate <resource>`
|
- `geth secret rotate <resource>`
|
||||||
|
|
|
||||||
|
|
@ -3,10 +3,11 @@ use base64::Engine;
|
||||||
use clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};
|
use clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};
|
||||||
use clap_complete::{Shell, generate};
|
use clap_complete::{Shell, generate};
|
||||||
use geth_config::GethPaths;
|
use geth_config::GethPaths;
|
||||||
use geth_control::{ControlRequest, ControlResponse};
|
use geth_control::{ControlRequest, ControlResponse, SyncStreamStatus};
|
||||||
use geth_node::service::{ServiceInstallOptions, ServiceManager, ServiceReport};
|
use geth_node::service::{ServiceInstallOptions, ServiceManager, ServiceReport};
|
||||||
use std::io::{Read, stdout};
|
use std::io::{Read, stdout};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
const TOP_LEVEL_AFTER_HELP: &str = r#"Common starts:
|
const TOP_LEVEL_AFTER_HELP: &str = r#"Common starts:
|
||||||
geth guide init
|
geth guide init
|
||||||
|
|
@ -363,6 +364,10 @@ pub enum Command {
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
command: SyncCommand,
|
command: SyncCommand,
|
||||||
},
|
},
|
||||||
|
Wait {
|
||||||
|
#[command(subcommand)]
|
||||||
|
command: WaitCommand,
|
||||||
|
},
|
||||||
Node {
|
Node {
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
command: NodeCommand,
|
command: NodeCommand,
|
||||||
|
|
@ -482,6 +487,30 @@ pub enum SyncCommand {
|
||||||
Now { node: Option<String> },
|
Now { node: Option<String> },
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Subcommand)]
|
||||||
|
pub enum WaitCommand {
|
||||||
|
Daemon {
|
||||||
|
#[arg(long, default_value_t = 30_000)]
|
||||||
|
timeout_ms: u64,
|
||||||
|
#[arg(long, default_value_t = 250)]
|
||||||
|
interval_ms: u64,
|
||||||
|
},
|
||||||
|
Peer {
|
||||||
|
node: String,
|
||||||
|
#[arg(long, default_value_t = 30_000)]
|
||||||
|
timeout_ms: u64,
|
||||||
|
#[arg(long, default_value_t = 500)]
|
||||||
|
interval_ms: u64,
|
||||||
|
},
|
||||||
|
Sync {
|
||||||
|
node: String,
|
||||||
|
#[arg(long, default_value_t = 30_000)]
|
||||||
|
timeout_ms: u64,
|
||||||
|
#[arg(long, default_value_t = 500)]
|
||||||
|
interval_ms: u64,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Subcommand)]
|
#[derive(Debug, Subcommand)]
|
||||||
pub enum NodeCommand {
|
pub enum NodeCommand {
|
||||||
Id,
|
Id,
|
||||||
|
|
@ -1284,6 +1313,13 @@ async fn run_inner(cli: Cli) -> Result<()> {
|
||||||
run_service_command(&paths, command).context("manage geth user service")?;
|
run_service_command(&paths, command).context("manage geth user service")?;
|
||||||
print_service_report(report, cli.json || cli.jsonl)?;
|
print_service_report(report, cli.json || cli.jsonl)?;
|
||||||
}
|
}
|
||||||
|
Command::Wait { command } => {
|
||||||
|
let report = run_wait_command(&paths, command).await?;
|
||||||
|
print_wait_report(&report, cli.json || cli.jsonl)?;
|
||||||
|
if !report.ready {
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
Command::Ssh {
|
Command::Ssh {
|
||||||
command:
|
command:
|
||||||
SshCommand::Proxy {
|
SshCommand::Proxy {
|
||||||
|
|
@ -2180,12 +2216,183 @@ fn request_for_command(command: Command) -> Result<ControlRequest> {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Command::Guide { .. } | Command::Init { .. } | Command::Daemon { .. } => {
|
Command::Guide { .. }
|
||||||
|
| Command::Init { .. }
|
||||||
|
| Command::Daemon { .. }
|
||||||
|
| Command::Wait { .. } => {
|
||||||
bail!("command is handled directly")
|
bail!("command is handled directly")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct WaitReport {
|
||||||
|
target: String,
|
||||||
|
ready: bool,
|
||||||
|
elapsed_ms: u128,
|
||||||
|
attempts: u64,
|
||||||
|
reason: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_wait_command(paths: &GethPaths, command: WaitCommand) -> Result<WaitReport> {
|
||||||
|
match command {
|
||||||
|
WaitCommand::Daemon {
|
||||||
|
timeout_ms,
|
||||||
|
interval_ms,
|
||||||
|
} => wait_for_daemon(paths, timeout_ms, interval_ms).await,
|
||||||
|
WaitCommand::Peer {
|
||||||
|
node,
|
||||||
|
timeout_ms,
|
||||||
|
interval_ms,
|
||||||
|
} => wait_for_peer(paths, node, timeout_ms, interval_ms).await,
|
||||||
|
WaitCommand::Sync {
|
||||||
|
node,
|
||||||
|
timeout_ms,
|
||||||
|
interval_ms,
|
||||||
|
} => wait_for_sync(paths, node, timeout_ms, interval_ms).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_daemon(
|
||||||
|
paths: &GethPaths,
|
||||||
|
timeout_ms: u64,
|
||||||
|
interval_ms: u64,
|
||||||
|
) -> Result<WaitReport> {
|
||||||
|
wait_loop("daemon".to_owned(), timeout_ms, interval_ms, || async {
|
||||||
|
match geth_node::send_control(paths, ControlRequest::Status).await {
|
||||||
|
Ok(ControlResponse::Status(_)) => Ok(Some("daemon control is ready".to_owned())),
|
||||||
|
Ok(other) => Ok(Some(format!("unexpected response: {:?}", other))),
|
||||||
|
Err(error) => Err(anyhow::anyhow!(error.to_string())),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_peer(
|
||||||
|
paths: &GethPaths,
|
||||||
|
node: String,
|
||||||
|
timeout_ms: u64,
|
||||||
|
interval_ms: u64,
|
||||||
|
) -> Result<WaitReport> {
|
||||||
|
wait_loop(format!("peer:{node}"), timeout_ms, interval_ms, || {
|
||||||
|
let node = node.clone();
|
||||||
|
async move {
|
||||||
|
match geth_node::send_control(paths, ControlRequest::PeerPing { node }).await {
|
||||||
|
Ok(ControlResponse::PeerPinged { note, .. }) => Ok(Some(note)),
|
||||||
|
Ok(ControlResponse::Error { message }) => Err(anyhow::anyhow!(message)),
|
||||||
|
Ok(_) => Ok(None),
|
||||||
|
Err(error) => Err(anyhow::anyhow!(error.to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_sync(
|
||||||
|
paths: &GethPaths,
|
||||||
|
node: String,
|
||||||
|
timeout_ms: u64,
|
||||||
|
interval_ms: u64,
|
||||||
|
) -> Result<WaitReport> {
|
||||||
|
wait_loop(format!("sync:{node}"), timeout_ms, interval_ms, || {
|
||||||
|
let node = node.clone();
|
||||||
|
async move {
|
||||||
|
match geth_node::send_control(paths, ControlRequest::SyncStatus).await {
|
||||||
|
Ok(ControlResponse::SyncStatus { peers, .. }) => {
|
||||||
|
let Some(peer) = peers.into_iter().find(|peer| peer.peer_node_id == node)
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
if peer.streams.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
if peer.streams.iter().all(sync_stream_ready) {
|
||||||
|
Ok(Some("sync streams are healthy or stale".to_owned()))
|
||||||
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(ControlResponse::Error { message }) => Err(anyhow::anyhow!(message)),
|
||||||
|
Ok(_) => Ok(None),
|
||||||
|
Err(error) => Err(anyhow::anyhow!(error.to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sync_stream_ready(stream: &SyncStreamStatus) -> bool {
|
||||||
|
stream.state == "ok" || stream.stale
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_loop<F, Fut>(
|
||||||
|
target: String,
|
||||||
|
timeout_ms: u64,
|
||||||
|
interval_ms: u64,
|
||||||
|
mut check: F,
|
||||||
|
) -> Result<WaitReport>
|
||||||
|
where
|
||||||
|
F: FnMut() -> Fut,
|
||||||
|
Fut: std::future::Future<Output = Result<Option<String>>>,
|
||||||
|
{
|
||||||
|
let started = Instant::now();
|
||||||
|
let timeout = Duration::from_millis(timeout_ms);
|
||||||
|
let interval = Duration::from_millis(interval_ms.max(1));
|
||||||
|
let mut attempts = 0;
|
||||||
|
let mut last_error: Option<String> = None;
|
||||||
|
loop {
|
||||||
|
attempts += 1;
|
||||||
|
match check().await {
|
||||||
|
Ok(Some(reason)) => {
|
||||||
|
return Ok(WaitReport {
|
||||||
|
target,
|
||||||
|
ready: true,
|
||||||
|
elapsed_ms: started.elapsed().as_millis(),
|
||||||
|
attempts,
|
||||||
|
reason,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(error) => {
|
||||||
|
last_error = Some(error.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if started.elapsed() >= timeout {
|
||||||
|
return Ok(WaitReport {
|
||||||
|
target,
|
||||||
|
ready: false,
|
||||||
|
elapsed_ms: started.elapsed().as_millis(),
|
||||||
|
attempts,
|
||||||
|
reason: last_error.unwrap_or_else(|| "timeout waiting for readiness".to_owned()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
tokio::time::sleep(interval).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn print_wait_report(report: &WaitReport, json: bool) -> Result<()> {
|
||||||
|
if json {
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
serde_json::to_string_pretty(&serde_json::json!({
|
||||||
|
"type": "wait",
|
||||||
|
"target": report.target,
|
||||||
|
"ready": report.ready,
|
||||||
|
"elapsed_ms": report.elapsed_ms,
|
||||||
|
"attempts": report.attempts,
|
||||||
|
"reason": report.reason,
|
||||||
|
}))?
|
||||||
|
);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
println!("target: {}", report.target);
|
||||||
|
println!("ready: {}", report.ready);
|
||||||
|
println!("elapsed_ms: {}", report.elapsed_ms);
|
||||||
|
println!("attempts: {}", report.attempts);
|
||||||
|
println!("reason: {}", report.reason);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn run_service_command(paths: &GethPaths, command: ServiceCommand) -> Result<ServiceReport> {
|
fn run_service_command(paths: &GethPaths, command: ServiceCommand) -> Result<ServiceReport> {
|
||||||
Ok(match command {
|
Ok(match command {
|
||||||
ServiceCommand::Install {
|
ServiceCommand::Install {
|
||||||
|
|
|
||||||
|
|
@ -258,6 +258,18 @@ fn geth_status_against_running_daemon() {
|
||||||
|
|
||||||
let output = run_geth(home.path(), &["status"]);
|
let output = run_geth(home.path(), &["status"]);
|
||||||
let json_output = run_geth(home.path(), &["--json", "status"]);
|
let json_output = run_geth(home.path(), &["--json", "status"]);
|
||||||
|
let wait_output = run_geth(
|
||||||
|
home.path(),
|
||||||
|
&[
|
||||||
|
"--json",
|
||||||
|
"wait",
|
||||||
|
"daemon",
|
||||||
|
"--timeout-ms",
|
||||||
|
"1000",
|
||||||
|
"--interval-ms",
|
||||||
|
"10",
|
||||||
|
],
|
||||||
|
);
|
||||||
let _ = daemon.kill();
|
let _ = daemon.kill();
|
||||||
let _ = daemon.wait();
|
let _ = daemon.wait();
|
||||||
|
|
||||||
|
|
@ -305,6 +317,11 @@ fn geth_status_against_running_daemon() {
|
||||||
assert_eq!(status_json["store_synchronous"], "normal");
|
assert_eq!(status_json["store_synchronous"], "normal");
|
||||||
assert_eq!(status_json["store_status"], "ok");
|
assert_eq!(status_json["store_status"], "ok");
|
||||||
assert!(status_json["native_backends"].is_array());
|
assert!(status_json["native_backends"].is_array());
|
||||||
|
|
||||||
|
let wait_json = command_json(&wait_output);
|
||||||
|
assert_eq!(wait_json["type"], "wait");
|
||||||
|
assert_eq!(wait_json["target"], "daemon");
|
||||||
|
assert_eq!(wait_json["ready"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -362,6 +379,31 @@ fn json_errors_include_stable_code_for_common_failures() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn wait_daemon_reports_json_timeout_without_daemon() {
|
||||||
|
let home = tempfile::tempdir().expect("tempdir");
|
||||||
|
let output = run_geth(
|
||||||
|
home.path(),
|
||||||
|
&[
|
||||||
|
"--json",
|
||||||
|
"wait",
|
||||||
|
"daemon",
|
||||||
|
"--timeout-ms",
|
||||||
|
"1",
|
||||||
|
"--interval-ms",
|
||||||
|
"1",
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(!output.status.success());
|
||||||
|
let report =
|
||||||
|
serde_json::from_slice::<serde_json::Value>(&output.stdout).expect("decode wait report");
|
||||||
|
assert_eq!(report["type"], "wait");
|
||||||
|
assert_eq!(report["target"], "daemon");
|
||||||
|
assert_eq!(report["ready"], false);
|
||||||
|
assert!(report["attempts"].as_u64().expect("attempts") >= 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn peer_ping_uses_daemon_owned_iroh_endpoint() {
|
fn peer_ping_uses_daemon_owned_iroh_endpoint() {
|
||||||
if skip_iroh_integration_tests() {
|
if skip_iroh_integration_tests() {
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,7 @@ The following command families are intended to be stable automation surfaces:
|
||||||
- `geth keychain sigchain|verify-sigchain|import-sigchain|verify-checkpoint|fetch|explain|explain-signer`
|
- `geth keychain sigchain|verify-sigchain|import-sigchain|verify-checkpoint|fetch|explain|explain-signer`
|
||||||
- `geth auth explain|grant|revoke|sync`
|
- `geth auth explain|grant|revoke|sync`
|
||||||
- `geth sync status|now`
|
- `geth sync status|now`
|
||||||
|
- `geth wait daemon|peer|sync`
|
||||||
- `geth secret status|create|rotate|bearer`
|
- `geth secret status|create|rotate|bearer`
|
||||||
- Local CAS object commands: `geth cas add|get|hash|has|pin|unpin|cleanup|providers|list|fetch`
|
- Local CAS object commands: `geth cas add|get|hash|has|pin|unpin|cleanup|providers|list|fetch`
|
||||||
- File-root metadata commands: `geth cas root add|list|scan|sync|apply`
|
- File-root metadata commands: `geth cas root add|list|scan|sync|apply`
|
||||||
|
|
|
||||||
|
|
@ -243,14 +243,14 @@ Goal: make `geth` ergonomic and stable as a base layer for custom automation.
|
||||||
- `[x]` Tests assert both code and operator-facing hint for representative
|
- `[x]` Tests assert both code and operator-facing hint for representative
|
||||||
failures.
|
failures.
|
||||||
|
|
||||||
- `[ ]` Add wait commands for automation.
|
- `[x]` Add wait commands for automation.
|
||||||
Acceptance criteria:
|
Acceptance criteria:
|
||||||
- `[ ]` `geth wait daemon` can block until local control is ready.
|
- `[x]` `geth wait daemon` can block until local control is ready.
|
||||||
- `[ ]` `geth wait peer <node>` can block until peer control succeeds or
|
- `[x]` `geth wait peer <node>` can block until peer control succeeds or
|
||||||
times out.
|
times out.
|
||||||
- `[ ]` `geth wait sync <node>` can block until required streams are healthy
|
- `[x]` `geth wait sync <node>` can block until required streams are healthy
|
||||||
or stale.
|
or stale.
|
||||||
- `[ ]` Wait commands support JSON output and timeout flags.
|
- `[x]` Wait commands support JSON output and timeout flags.
|
||||||
|
|
||||||
- `[ ]` Make common commands idempotent.
|
- `[ ]` Make common commands idempotent.
|
||||||
Acceptance criteria:
|
Acceptance criteria:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue