diff --git a/AGENTS.md b/AGENTS.md index 48b5a41..b2aeb43 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,8 +116,8 @@ Roadmap items should be actionable and checkable: enforcement are still roadmap work. - The daemon persists local keychain init/admin-key ops and reduces them for `keychain status`. SSH signature capture/verification is still roadmap work. -- Local CAS supports pin/unpin metadata, surfaced through `cas list`. Cache - cleanup and eviction policy are still roadmap work. +- Local CAS supports pin/unpin metadata, surfaced through `cas list`, and + `cas cleanup` evicts unpinned blobs while retaining pinned blobs. - Signed peer-card LAN discovery payloads, peer auth over Iroh, cr-sqlite, iroh-docs, iroh-gossip, iroh-blobs, Automerge sync, real auth enforcement, OpenSSH KRL generation, and Keyhive/BeeKEM-style authorization are future diff --git a/README.md b/README.md index 624846d..8fd63a5 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ The bootstrap implementation provides: - `geth auth grant [--grant-id ]` - `geth auth revoke ` - local filesystem CAS commands: `add`, `get`, `hash`, `has`, `pin`, `unpin`, - `list` + `cleanup`, `list` - SSH certificate flow metadata: - `geth ssh cert request --public-key --principal ` - `geth ssh cert requests` diff --git a/crates/geth-cas/src/lib.rs b/crates/geth-cas/src/lib.rs index ceee527..4f6d5fe 100644 --- a/crates/geth-cas/src/lib.rs +++ b/crates/geth-cas/src/lib.rs @@ -67,6 +67,15 @@ impl LocalCas { Ok(self.blob_path(hash)?.exists()) } + pub fn remove(&self, hash: &BlobHash) -> Result { + let path = self.blob_path(hash)?; + if !path.exists() { + return Ok(false); + } + std::fs::remove_file(path)?; + Ok(true) + } + pub fn list(&self) -> Result, CasError> { let blobs = self.root.join("blobs"); if !blobs.exists() { @@ -154,4 +163,15 @@ mod tests { cas.get_to_path(&info.hash, &out).expect("get"); assert_eq!(std::fs::read(out).expect("read"), b"hello geth"); } + + #[test] + fn cas_remove_deletes_blob_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let cas = LocalCas::new(dir.path()); + let info = cas.add_bytes(b"remove me").expect("add"); + + assert!(cas.remove(&info.hash).expect("remove")); + assert!(!cas.has(&info.hash).expect("has after remove")); + assert!(!cas.remove(&info.hash).expect("remove missing")); + } } diff --git a/crates/geth-cli/src/lib.rs b/crates/geth-cli/src/lib.rs index aaa51c6..95a023c 100644 --- a/crates/geth-cli/src/lib.rs +++ b/crates/geth-cli/src/lib.rs @@ -185,6 +185,10 @@ pub enum CasCommand { Unpin { hash: String, }, + Cleanup { + #[arg(long)] + dry_run: bool, + }, List, } @@ -400,6 +404,7 @@ fn request_for_command(command: Command) -> Result { CasCommand::Has { hash } => ControlRequest::CasHas { hash: hash.into() }, CasCommand::Pin { hash } => ControlRequest::CasPin { hash: hash.into() }, CasCommand::Unpin { hash } => ControlRequest::CasUnpin { hash: hash.into() }, + CasCommand::Cleanup { dry_run } => ControlRequest::CasCleanup { dry_run }, CasCommand::List => ControlRequest::CasList, }, Command::Kv { command } => ControlRequest::ModuleStub { @@ -596,6 +601,21 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> { ControlResponse::CasPinned { hash, pinned } => { println!("{hash}: pinned={pinned}"); } + ControlResponse::CasCleanup { + removed, + retained_pinned, + dry_run, + } => { + let action = if dry_run { "would remove" } else { "removed" }; + println!("{action}: {}", removed.len()); + for hash in removed { + println!(" {hash}"); + } + println!("retained_pinned: {}", retained_pinned.len()); + for hash in retained_pinned { + println!(" {hash}"); + } + } ControlResponse::CasList { blobs } => { for blob in blobs { let pin = if blob.pinned { "pinned" } else { "unpinned" }; diff --git a/crates/geth-control/src/lib.rs b/crates/geth-control/src/lib.rs index 31a497e..194596c 100644 --- a/crates/geth-control/src/lib.rs +++ b/crates/geth-control/src/lib.rs @@ -37,6 +37,9 @@ pub enum ControlRequest { CasUnpin { hash: BlobHash, }, + CasCleanup { + dry_run: bool, + }, CasList, KeychainInit { admin_key_path: Option, @@ -124,6 +127,11 @@ pub enum ControlResponse { hash: BlobHash, pinned: bool, }, + CasCleanup { + removed: Vec, + retained_pinned: Vec, + dry_run: bool, + }, CasList { blobs: Vec, }, diff --git a/crates/geth-node/src/lib.rs b/crates/geth-node/src/lib.rs index eb8f2fd..230919e 100644 --- a/crates/geth-node/src/lib.rs +++ b/crates/geth-node/src/lib.rs @@ -243,6 +243,26 @@ pub fn handle_request( pinned: false, }) } + ControlRequest::CasCleanup { dry_run } => { + let cas = LocalCas::new(node.paths.cas_dir()); + let mut removed = Vec::new(); + let mut retained_pinned = Vec::new(); + for blob in cas.list()? { + if store.is_cas_object_pinned(blob.hash.as_str())? { + retained_pinned.push(blob.hash); + continue; + } + if !dry_run && cas.remove(&blob.hash)? { + store.delete_cas_object(blob.hash.as_str())?; + } + removed.push(blob.hash); + } + Ok(ControlResponse::CasCleanup { + removed, + retained_pinned, + dry_run, + }) + } ControlRequest::CasList => { let cas = LocalCas::new(node.paths.cas_dir()); let blobs = cas diff --git a/crates/geth-store/src/lib.rs b/crates/geth-store/src/lib.rs index 0a6434f..121d708 100644 --- a/crates/geth-store/src/lib.rs +++ b/crates/geth-store/src/lib.rs @@ -235,6 +235,14 @@ impl Store { Ok(()) } + pub fn delete_cas_object(&self, hash: &str) -> Result<(), StoreError> { + self.conn + .execute("DELETE FROM cas_objects WHERE hash = ?1", params![hash])?; + self.conn + .execute("DELETE FROM cas_pins WHERE hash = ?1", params![hash])?; + Ok(()) + } + pub fn list_cas_objects(&self) -> Result, StoreError> { let mut stmt = self.conn.prepare( "SELECT hash, size_bytes, path FROM cas_objects ORDER BY created_at_ms, hash", @@ -765,4 +773,19 @@ mod tests { assert!(!store.is_cas_object_pinned(hash).expect("unpinned again")); assert!(store.list_cas_pins().expect("pins").is_empty()); } + + #[test] + fn deleting_cas_object_removes_pin_metadata() { + let store = Store::open_memory().expect("open"); + let hash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + store + .record_cas_object(hash, 10, "/tmp/blob") + .expect("record object"); + store.pin_cas_object(hash).expect("pin"); + + store.delete_cas_object(hash).expect("delete object"); + + assert!(store.list_cas_objects().expect("objects").is_empty()); + assert!(store.list_cas_pins().expect("pins").is_empty()); + } } diff --git a/crates/geth/tests/bootstrap.rs b/crates/geth/tests/bootstrap.rs index bc44778..b90a58e 100644 --- a/crates/geth/tests/bootstrap.rs +++ b/crates/geth/tests/bootstrap.rs @@ -168,6 +168,90 @@ fn cas_pin_unpin_updates_local_pin_metadata() { } } +#[test] +fn cas_cleanup_removes_unpinned_and_retains_pinned_blobs() { + let home = tempfile::tempdir().expect("tempdir"); + let paths = geth_config::GethPaths::from_home(home.path()); + let node = geth_node::init_node(&paths).expect("init node"); + let pinned_path = home.path().join("pinned.txt"); + let unpinned_path = home.path().join("unpinned.txt"); + std::fs::write(&pinned_path, b"keep me").expect("write pinned"); + std::fs::write(&unpinned_path, b"remove me").expect("write unpinned"); + + let pinned_hash = match geth_node::handle_request( + &node, + geth_control::ControlRequest::CasAdd { path: pinned_path }, + ) + .expect("add pinned") + { + geth_control::ControlResponse::CasAdded { hash, .. } => hash, + other => panic!("unexpected response: {other:?}"), + }; + let unpinned_hash = match geth_node::handle_request( + &node, + geth_control::ControlRequest::CasAdd { + path: unpinned_path, + }, + ) + .expect("add unpinned") + { + geth_control::ControlResponse::CasAdded { hash, .. } => hash, + other => panic!("unexpected response: {other:?}"), + }; + geth_node::handle_request( + &node, + geth_control::ControlRequest::CasPin { + hash: pinned_hash.clone(), + }, + ) + .expect("pin blob"); + + let dry_run = geth_node::handle_request( + &node, + geth_control::ControlRequest::CasCleanup { dry_run: true }, + ) + .expect("dry-run cleanup"); + match dry_run { + geth_control::ControlResponse::CasCleanup { + removed, + retained_pinned, + dry_run, + } => { + assert!(dry_run); + assert_eq!(removed, vec![unpinned_hash.clone()]); + assert_eq!(retained_pinned, vec![pinned_hash.clone()]); + } + other => panic!("unexpected response: {other:?}"), + } + assert!( + geth_cas::LocalCas::new(paths.cas_dir()) + .has(&unpinned_hash) + .expect("dry-run keeps unpinned") + ); + + let cleanup = geth_node::handle_request( + &node, + geth_control::ControlRequest::CasCleanup { dry_run: false }, + ) + .expect("cleanup"); + match cleanup { + geth_control::ControlResponse::CasCleanup { + removed, + retained_pinned, + dry_run, + } => { + assert!(!dry_run); + assert_eq!(removed, vec![unpinned_hash.clone()]); + assert_eq!(retained_pinned, vec![pinned_hash.clone()]); + } + other => panic!("unexpected response: {other:?}"), + } + + let cas = geth_cas::LocalCas::new(paths.cas_dir()); + assert!(cas.has(&pinned_hash).expect("pinned retained")); + assert!(!cas.has(&unpinned_hash).expect("unpinned removed")); +} + #[test] fn auth_explain_distinguishes_discovered_peer_candidates() { let home = tempfile::tempdir().expect("tempdir"); diff --git a/docs/architecture.md b/docs/architecture.md index 7b12618..4d89a6a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -89,7 +89,8 @@ Resource kinds: `geth-cas` is implemented locally first using BLAKE3 hashes and filesystem blob storage. Local pin/unpin metadata is tracked in SQLite and surfaced in -`cas list`. Iroh-blobs, providers, encrypted blobs, cache cleanup, manifests, +`cas list`. `cas cleanup` removes unpinned local blobs while retaining pinned +blobs. Iroh-blobs, providers, encrypted blobs, richer cache policies, manifests, and file sync trees are future work. `geth-kv`, `geth-db`, `geth-document`, `geth-pubsub`, `geth-pipe`, and diff --git a/docs/roadmap.md b/docs/roadmap.md index c64c5bf..ff8b14a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -200,14 +200,14 @@ authorization and durable-state boundaries clear. - Provider tracking is recorded locally. - Local add/get/hash/has/list behavior remains backward compatible. -- `[~]` CAS pin and cache policy. +- `[x]` CAS pin and cache policy. Acceptance criteria: - `[x]` `geth cas pin ` persists local pin metadata. - `[x]` `geth cas unpin ` removes local pin metadata. - `[x]` `geth cas list` reports pinned versus unpinned blobs. - - `[ ]` Pinned blobs are retained across cache cleanup. - - `[ ]` Unpinned cached blobs can be evicted by policy. - - `[ ]` Tests cover attempted eviction of pinned data. + - `[x]` Pinned blobs are retained across cache cleanup. + - `[x]` Unpinned cached blobs can be evicted by policy. + - `[x]` Tests cover attempted eviction of pinned data. - `[ ]` Encrypted private blobs. Acceptance criteria: