eric/geth
Watch
1
0
Fork
You've already forked geth
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add local CAS file roots

Browse source
This commit is contained in:
Eric Wendland 2026-05-18 03:50:09 +02:00
commit e1f36ffafb
12 changed files with 581 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

219
crates/geth-cas/src/lib.rs
View file

@ -1,5 +1,6 @@
use geth_types::BlobHash;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
pub const CAS_TREE_OBJECT_VERSION: u16 = 1;
@ -18,6 +19,8 @@ pub enum CasError {
TreeRootNotDirectory(String),
#[error("CAS tree paths must be relative: {0}")]
NonRelativeTreePath(String),
#[error("invalid file root name: {0}")]
InvalidFileRootName(String),
}
#[derive(Clone, Debug)]
@ -60,6 +63,39 @@ pub struct CasTreeStored {
pub object: BlobInfo,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct FileRoot {
pub id: String,
pub resource: String,
pub name: String,
pub path: String,
pub latest_tree: Option<BlobHash>,
pub updated_at_ms: i64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct FileRootScan {
pub root: FileRoot,
pub tree: BlobInfoSummary,
pub changes: Vec<FileRootChange>,
pub note: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BlobInfoSummary {
pub hash: BlobHash,
pub size_bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "kebab-case")]
pub enum FileRootChange {
Created { path: String },
Modified { path: String },
Deleted { path: String },
Renamed { from: String, to: String },
}
impl LocalCas {
#[must_use]
pub fn new(root: impl Into<PathBuf>) -> Self {
@ -188,6 +224,107 @@ pub fn build_tree_object(cas: &LocalCas, root: &Path) -> Result<CasTreeObject, C
})
}
pub fn diff_tree_objects(
previous: Option<&CasTreeObject>,
current: &CasTreeObject,
) -> Vec<FileRootChange> {
let Some(previous) = previous else {
return current
.entries
.iter()
.map(|entry| FileRootChange::Created {
path: entry.path.clone(),
})
.collect();
};
let previous_entries = tree_entry_map(previous);
let current_entries = tree_entry_map(current);
let previous_paths = previous_entries.keys().cloned().collect::<BTreeSet<_>>();
let current_paths = current_entries.keys().cloned().collect::<BTreeSet<_>>();
let deleted = previous_paths
.difference(&current_paths)
.cloned()
.collect::<Vec<_>>();
let created = current_paths
.difference(&previous_paths)
.cloned()
.collect::<Vec<_>>();
let mut changes = Vec::new();
let mut consumed_deleted = BTreeSet::new();
let mut consumed_created = BTreeSet::new();
for deleted_path in &deleted {
let Some(deleted_entry) = previous_entries.get(deleted_path) else {
continue;
};
let Some(deleted_blob) = &deleted_entry.blob else {
continue;
};
if let Some(created_path) = created.iter().find(|created_path| {
!consumed_created.contains(*created_path)
&& current_entries
.get(*created_path)
.and_then(|entry| entry.blob.as_ref())
== Some(deleted_blob)
}) {
changes.push(FileRootChange::Renamed {
from: deleted_path.clone(),
to: created_path.clone(),
});
consumed_deleted.insert(deleted_path.clone());
consumed_created.insert(created_path.clone());
}
}
for path in deleted {
if !consumed_deleted.contains(&path) {
changes.push(FileRootChange::Deleted { path });
}
}
for path in created {
if !consumed_created.contains(&path) {
changes.push(FileRootChange::Created { path });
}
}
for path in previous_paths.intersection(&current_paths) {
let previous = previous_entries.get(path).expect("previous entry");
let current = current_entries.get(path).expect("current entry");
if previous.kind != current.kind
|| previous.blob != current.blob
|| previous.executable != current.executable
|| previous.size_bytes != current.size_bytes
{
changes.push(FileRootChange::Modified { path: path.clone() });
}
}
changes.sort_by_key(change_sort_key);
changes
}
#[must_use]
pub fn file_root_scan_note() -> &'static str {
"local scan only; geth never overwrites file roots without a recorded future sync decision"
}
fn tree_entry_map(tree: &CasTreeObject) -> BTreeMap<String, &CasTreeEntry> {
tree.entries
.iter()
.map(|entry| (entry.path.clone(), entry))
.collect()
}
fn change_sort_key(change: &FileRootChange) -> (u8, String, String) {
match change {
FileRootChange::Created { path } => (0, path.clone(), String::new()),
FileRootChange::Modified { path } => (1, path.clone(), String::new()),
FileRootChange::Deleted { path } => (2, path.clone(), String::new()),
FileRootChange::Renamed { from, to } => (3, from.clone(), to.clone()),
}
}
fn collect_tree_entries(
cas: &LocalCas,
root: &Path,
@ -277,6 +414,17 @@ pub fn is_valid_hash(hash: &str) -> bool {
hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit())
}
pub fn validate_file_root_name(name: &str) -> Result<(), CasError> {
if name.is_empty()
|| !name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
{
return Err(CasError::InvalidFileRootName(name.to_owned()));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@ -351,4 +499,75 @@ mod tests {
Err(CasError::TreeRootNotDirectory(_))
));
}
#[test]
fn cas_tree_diff_detects_create_modify_delete_and_rename() {
let previous = CasTreeObject {
version: CAS_TREE_OBJECT_VERSION,
entries: vec![
tree_file("deleted.txt", "01", 1),
tree_file("modified.txt", "02", 1),
tree_file("old.txt", "03", 1),
],
};
let current = CasTreeObject {
version: CAS_TREE_OBJECT_VERSION,
entries: vec![
tree_file("created.txt", "04", 1),
tree_file("modified.txt", "05", 1),
tree_file("new.txt", "03", 1),
],
};
let diff = diff_tree_objects(Some(&previous), &current);
assert!(diff.contains(&FileRootChange::Created {
path: "created.txt".to_owned()
}));
assert!(diff.contains(&FileRootChange::Modified {
path: "modified.txt".to_owned()
}));
assert!(diff.contains(&FileRootChange::Deleted {
path: "deleted.txt".to_owned()
}));
assert!(diff.contains(&FileRootChange::Renamed {
from: "old.txt".to_owned(),
to: "new.txt".to_owned(),
}));
}
#[test]
fn cas_tree_diff_does_not_reuse_rename_targets() {
let previous = CasTreeObject {
version: CAS_TREE_OBJECT_VERSION,
entries: vec![tree_file("one.txt", "01", 1), tree_file("two.txt", "01", 1)],
};
let current = CasTreeObject {
version: CAS_TREE_OBJECT_VERSION,
entries: vec![tree_file("renamed.txt", "01", 1)],
};
let diff = diff_tree_objects(Some(&previous), &current);
let rename_count = diff
.iter()
.filter(|change| matches!(change, FileRootChange::Renamed { .. }))
.count();
let delete_count = diff
.iter()
.filter(|change| matches!(change, FileRootChange::Deleted { .. }))
.count();
assert_eq!(rename_count, 1);
assert_eq!(delete_count, 1);
}
fn tree_file(path: &str, byte: &str, size_bytes: u64) -> CasTreeEntry {
CasTreeEntry {
path: path.to_owned(),
kind: CasTreeEntryKind::File,
blob: Some(BlobHash::new(byte.repeat(32))),
size_bytes,
executable: false,
}
}
}

48
crates/geth-cli/src/lib.rs
View file

@ -216,6 +216,17 @@ pub enum CasCommand {
dry_run: bool,
},
List,
Root {
#[command(subcommand)]
command: CasRootCommand,
},
}
#[derive(Debug, Subcommand)]
pub enum CasRootCommand {
Add { name: String, path: PathBuf },
List,
Scan { name: String },
}
#[derive(Debug, Subcommand)]
@ -464,6 +475,11 @@ fn request_for_command(command: Command) -> Result<ControlRequest> {
CasCommand::Unpin { hash } => ControlRequest::CasUnpin { hash: hash.into() },
CasCommand::Cleanup { dry_run } => ControlRequest::CasCleanup { dry_run },
CasCommand::List => ControlRequest::CasList,
CasCommand::Root { command } => match command {
CasRootCommand::Add { name, path } => ControlRequest::CasRootAdd { name, path },
CasRootCommand::List => ControlRequest::CasRootList,
CasRootCommand::Scan { name } => ControlRequest::CasRootScan { name },
},
},
Command::Kv { command } => match command {
KvCommand::Create { name } => ControlRequest::KvCreate { name },
@ -696,6 +712,38 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
println!("{}\t{} bytes\t{}", blob.hash, blob.size_bytes, pin);
}
}
ControlResponse::CasRootAdded { root } => {
println!("added file root: {}", root.name);
println!("id: {}", root.id);
println!("resource: {}", root.resource);
println!("path: {}", root.path);
}
ControlResponse::CasRootList { roots } => {
if roots.is_empty() {
println!("no file roots");
} else {
for root in roots {
println!(
"{}\t{}\t{}",
root.name,
root.path,
root.latest_tree
.map(|hash| hash.to_string())
.unwrap_or_else(|| "unscanned".to_owned())
);
}
}
}
ControlResponse::CasRootScanned { scan } => {
println!("file root: {}", scan.root.name);
println!("tree: {}", scan.tree.hash);
println!("tree_bytes: {}", scan.tree.size_bytes);
println!("changes: {}", scan.changes.len());
for change in scan.changes {
println!("{change:?}");
}
println!("note: {}", scan.note);
}
ControlResponse::KeychainStatus(status) => {
println!("initialized: {}", status.initialized);
println!("admin_keys: {}", status.admin_keys);

1
crates/geth-control/Cargo.toml
View file

@ -10,6 +10,7 @@ serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
geth-auth = { path = "../geth-auth" }
geth-cas = { path = "../geth-cas" }
geth-db = { path = "../geth-db" }
geth-document = { path = "../geth-document" }
geth-keychain = { path = "../geth-keychain" }

26
crates/geth-control/src/lib.rs
View file

@ -1,4 +1,5 @@
use geth_auth::{AuthExplanation, AuthOp};
use geth_cas::{FileRoot, FileRootScan};
use geth_db::{CrSqliteChangeBatch, DbResource};
use geth_document::{DocumentResource, DocumentState};
use geth_keychain::KeychainOp;
@ -47,6 +48,14 @@ pub enum ControlRequest {
dry_run: bool,
},
CasList,
CasRootAdd {
name: String,
path: PathBuf,
},
CasRootList,
CasRootScan {
name: String,
},
KeychainInit {
admin_key_path: Option<PathBuf>,
},
@ -209,6 +218,15 @@ pub enum ControlResponse {
CasList {
blobs: Vec<CasBlob>,
},
CasRootAdded {
root: FileRoot,
},
CasRootList {
roots: Vec<FileRoot>,
},
CasRootScanned {
scan: FileRootScan,
},
KeychainStatus(KeychainStatusResponse),
KeychainInitialized {
ops: Vec<KeychainOp>,
@ -446,5 +464,13 @@ mod tests {
decode_request(&encode_request(&request).expect("encode")).expect("decode"),
request
);
let request = ControlRequest::CasRootScan {
name: "notes".to_owned(),
};
assert_eq!(
decode_request(&encode_request(&request).expect("encode")).expect("decode"),
request
);
}
}

89
crates/geth-node/src/lib.rs
View file

@ -1,7 +1,7 @@
pub mod service;
use geth_auth::{AuthExplanation, AuthOp, AuthOpKind};
use geth_cas::{LocalCas, hash_path};
use geth_cas::{BlobInfoSummary, FileRoot, FileRootScan, LocalCas, hash_path};
use geth_config::{GethConfig, GethPaths, RelayMode};
use geth_control::{
CasBlob, ControlRequest, ControlResponse, KeychainStatusResponse, NodeIdResponse,
@ -23,9 +23,9 @@ use geth_ssh_identity::{
cert_request_id, certificate_id, openssh_krl_spec, revocation_id, ssh_public_key_fingerprint,
};
use geth_store::{
Store, StoredAuthOp, StoredDbResource, StoredDocumentResource, StoredKeychainOp, StoredKvEntry,
StoredKvStore, StoredResource, StoredResourceSecret, StoredSshCertRequest,
StoredSshCertificate, StoredSshRevocation,
Store, StoredAuthOp, StoredDbResource, StoredDocumentResource, StoredFileRoot,
StoredKeychainOp, StoredKvEntry, StoredKvStore, StoredResource, StoredResourceSecret,
StoredSshCertRequest, StoredSshCertificate, StoredSshRevocation,
};
use geth_types::{
AuthOpId, Capability, KeyId, NodeId, PrincipalId, ResourceId, ResourceKind, ResourceName,
@ -344,6 +344,76 @@ pub fn handle_request(
.collect::<Result<Vec<_>, NodeError>>()?;
Ok(ControlResponse::CasList { blobs })
}
ControlRequest::CasRootAdd { name, path } => {
geth_cas::validate_file_root_name(&name)?;
if !path.is_dir() {
return Err(NodeError::Cas(geth_cas::CasError::TreeRootNotDirectory(
path.display().to_string(),
)));
}
let path = std::fs::canonicalize(path)?;
let resource_id = format!("resource:cas-tree:{name}");
store.insert_resource(&StoredResource {
resource_id: resource_id.clone(),
kind: ResourceKind::Cas.to_string(),
name: format!("file-root:{name}"),
status: "active".to_owned(),
})?;
let stored = StoredFileRoot {
root_id: format!("file-root:{name}"),
resource_id,
name,
path: path.display().to_string(),
latest_tree_hash: None,
latest_tree_json: None,
updated_at_ms: geth_store::now_ms(),
};
store.upsert_file_root(&stored)?;
Ok(ControlResponse::CasRootAdded {
root: file_root_from_stored(&stored),
})
}
ControlRequest::CasRootList => Ok(ControlResponse::CasRootList {
roots: store
.list_file_roots()?
.iter()
.map(file_root_from_stored)
.collect(),
}),
ControlRequest::CasRootScan { name } => {
geth_cas::validate_file_root_name(&name)?;
let mut stored = store
.get_file_root_by_name(&name)?
.ok_or_else(|| NodeError::ResourceNotFound(format!("file-root:{name}")))?;
let previous_tree = stored
.latest_tree_json
.as_deref()
.map(serde_json::from_str)
.transpose()?;
let cas = LocalCas::new(node.paths.cas_dir());
let scanned = cas.add_tree_path(Path::new(&stored.path))?;
store.record_cas_object(
scanned.object.hash.as_str(),
scanned.object.size_bytes,
&scanned.object.path.to_string_lossy(),
)?;
let changes = geth_cas::diff_tree_objects(previous_tree.as_ref(), &scanned.tree);
stored.latest_tree_hash = Some(scanned.object.hash.to_string());
stored.latest_tree_json = Some(serde_json::to_string(&scanned.tree)?);
stored.updated_at_ms = geth_store::now_ms();
store.upsert_file_root(&stored)?;
Ok(ControlResponse::CasRootScanned {
scan: FileRootScan {
root: file_root_from_stored(&stored),
tree: BlobInfoSummary {
hash: scanned.object.hash,
size_bytes: scanned.object.size_bytes,
},
changes,
note: geth_cas::file_root_scan_note().to_owned(),
},
})
}
ControlRequest::KeychainInit { admin_key_path } => {
let mut ops = Vec::new();
let created_at = UnixMillis(geth_store::now_ms());
@ -1055,6 +1125,17 @@ fn document_state_from_stored(stored: &StoredDocumentResource) -> DocumentState
}
}
fn file_root_from_stored(stored: &StoredFileRoot) -> FileRoot {
FileRoot {
id: stored.root_id.clone(),
resource: stored.resource_id.clone(),
name: stored.name.clone(),
path: stored.path.clone(),
latest_tree: stored.latest_tree_hash.clone().map(Into::into),
updated_at_ms: stored.updated_at_ms,
}
}
fn ensure_resource_exists(store: &Store, resource_id: &str) -> Result<(), NodeError> {
if store
.list_resources()?

103
crates/geth-store/src/lib.rs
View file

@ -124,6 +124,15 @@ impl Store {
state_json TEXT NOT NULL DEFAULT '{}',
updated_at_ms INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS file_roots (
root_id TEXT PRIMARY KEY,
resource_id TEXT NOT NULL,
name TEXT NOT NULL UNIQUE,
path TEXT NOT NULL,
latest_tree_hash TEXT,
latest_tree_json TEXT,
updated_at_ms INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS peer_cards (
peer_id TEXT PRIMARY KEY,
card_json TEXT NOT NULL,
@ -382,6 +391,65 @@ impl Store {
}
}
pub fn upsert_file_root(&self, root: &StoredFileRoot) -> Result<(), StoreError> {
self.conn.execute(
r#"INSERT OR REPLACE INTO file_roots(
root_id, resource_id, name, path, latest_tree_hash, latest_tree_json, updated_at_ms
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"#,
params![
root.root_id,
root.resource_id,
root.name,
root.path,
root.latest_tree_hash,
root.latest_tree_json,
root.updated_at_ms
],
)?;
Ok(())
}
pub fn get_file_root_by_name(&self, name: &str) -> Result<Option<StoredFileRoot>, StoreError> {
let mut stmt = self.conn.prepare(
r#"SELECT root_id, resource_id, name, path, latest_tree_hash, latest_tree_json, updated_at_ms
FROM file_roots WHERE name = ?1"#,
)?;
let mut rows = stmt.query(params![name])?;
if let Some(row) = rows.next()? {
Ok(Some(StoredFileRoot {
root_id: row.get(0)?,
resource_id: row.get(1)?,
name: row.get(2)?,
path: row.get(3)?,
latest_tree_hash: row.get(4)?,
latest_tree_json: row.get(5)?,
updated_at_ms: row.get(6)?,
}))
} else {
Ok(None)
}
}
pub fn list_file_roots(&self) -> Result<Vec<StoredFileRoot>, StoreError> {
let mut stmt = self.conn.prepare(
r#"SELECT root_id, resource_id, name, path, latest_tree_hash, latest_tree_json, updated_at_ms
FROM file_roots ORDER BY name"#,
)?;
let rows = stmt.query_map([], |row| {
Ok(StoredFileRoot {
root_id: row.get(0)?,
resource_id: row.get(1)?,
name: row.get(2)?,
path: row.get(3)?,
latest_tree_hash: row.get(4)?,
latest_tree_json: row.get(5)?,
updated_at_ms: row.get(6)?,
})
})?;
rows.collect::<Result<Vec<_>, _>>()
.map_err(StoreError::from)
}
pub fn insert_resource_secret(&self, secret: &StoredResourceSecret) -> Result<(), StoreError> {
self.conn.execute(
r#"INSERT OR REPLACE INTO resource_secrets(
@ -829,6 +897,17 @@ pub struct StoredDocumentResource {
pub updated_at_ms: i64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StoredFileRoot {
pub root_id: String,
pub resource_id: String,
pub name: String,
pub path: String,
pub latest_tree_hash: Option<String>,
pub latest_tree_json: Option<String>,
pub updated_at_ms: i64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StoredResourceSecret {
pub secret_id: String,
@ -1101,6 +1180,30 @@ mod tests {
);
}
#[test]
fn file_roots_roundtrip_by_name() {
let store = Store::open_memory().expect("open");
let root = StoredFileRoot {
root_id: "file-root:notes".to_owned(),
resource_id: "resource:cas-tree:notes".to_owned(),
name: "notes".to_owned(),
path: "/tmp/notes".to_owned(),
latest_tree_hash: Some(
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(),
),
latest_tree_json: Some("{}".to_owned()),
updated_at_ms: 10,
};
store.upsert_file_root(&root).expect("insert root");
assert_eq!(
store.get_file_root_by_name("notes").expect("get file root"),
Some(root.clone())
);
assert_eq!(store.list_file_roots().expect("list roots"), vec![root]);
}
#[test]
fn kv_store_and_entries_roundtrip() {
let store = Store::open_memory().expect("open");

79
crates/geth/tests/bootstrap.rs
View file

@ -168,6 +168,85 @@ fn cas_pin_unpin_updates_local_pin_metadata() {
}
}
#[test]
fn cas_file_root_add_list_scan_detects_changes() {
let home = tempfile::tempdir().expect("tempdir");
let paths = geth_config::GethPaths::from_home(home.path());
let node = geth_node::init_node(&paths).expect("init node");
let root_path = home.path().join("notes-root");
std::fs::create_dir_all(&root_path).expect("create root");
std::fs::write(root_path.join("a.txt"), b"alpha").expect("write a");
let response = geth_node::handle_request(
&node,
geth_control::ControlRequest::CasRootAdd {
name: "notes".to_owned(),
path: root_path.clone(),
},
)
.expect("add root");
match response {
geth_control::ControlResponse::CasRootAdded { root } => {
assert_eq!(root.name, "notes");
assert!(root.latest_tree.is_none());
}
other => panic!("unexpected response: {other:?}"),
}
let response = geth_node::handle_request(&node, geth_control::ControlRequest::CasRootList)
.expect("list roots");
match response {
geth_control::ControlResponse::CasRootList { roots } => {
assert_eq!(roots.len(), 1);
assert_eq!(roots[0].name, "notes");
}
other => panic!("unexpected response: {other:?}"),
}
let response = geth_node::handle_request(
&node,
geth_control::ControlRequest::CasRootScan {
name: "notes".to_owned(),
},
)
.expect("initial scan");
let first_tree = match response {
geth_control::ControlResponse::CasRootScanned { scan } => {
assert_eq!(scan.changes.len(), 1);
assert!(scan.root.latest_tree.is_some());
assert!(scan.note.contains("never overwrites"));
scan.tree.hash
}
other => panic!("unexpected response: {other:?}"),
};
std::fs::rename(root_path.join("a.txt"), root_path.join("b.txt")).expect("rename a to b");
std::fs::write(root_path.join("c.txt"), b"charlie").expect("write c");
let response = geth_node::handle_request(
&node,
geth_control::ControlRequest::CasRootScan {
name: "notes".to_owned(),
},
)
.expect("second scan");
match response {
geth_control::ControlResponse::CasRootScanned { scan } => {
assert_ne!(scan.tree.hash, first_tree);
assert!(scan.changes.iter().any(|change| {
matches!(
change,
geth_cas::FileRootChange::Renamed { from, to }
if from == "a.txt" && to == "b.txt"
)
}));
assert!(scan.changes.iter().any(|change| {
matches!(change, geth_cas::FileRootChange::Created { path } if path == "c.txt")
}));
}
other => panic!("unexpected response: {other:?}"),
}
}
#[test]
fn cas_cleanup_removes_unpinned_and_retains_pinned_blobs() {
let home = tempfile::tempdir().expect("tempdir");