Add local CAS file roots
This commit is contained in:
parent
64e17ff957
commit
e1f36ffafb
12 changed files with 581 additions and 13 deletions
|
|
@ -119,8 +119,10 @@ Roadmap items should be actionable and checkable:
|
||||||
- Local CAS supports pin/unpin metadata, surfaced through `cas list`, and
|
- Local CAS supports pin/unpin metadata, surfaced through `cas list`, and
|
||||||
`cas cleanup` evicts unpinned blobs while retaining pinned blobs.
|
`cas cleanup` evicts unpinned blobs while retaining pinned blobs.
|
||||||
- The CAS crate can build deterministic tree objects for local file trees and
|
- The CAS crate can build deterministic tree objects for local file trees and
|
||||||
store those manifests as CAS blobs. File roots, scan workflows, and sync
|
store those manifests as CAS blobs. The daemon can register and scan local
|
||||||
conflict handling are still roadmap work.
|
file roots, reporting create/update/delete/rename changes without writing back
|
||||||
|
to the working tree. Cross-node file sync and conflict handling are still
|
||||||
|
roadmap work.
|
||||||
- DB resources can be registered locally and report local-only status plus a
|
- DB resources can be registered locally and report local-only status plus a
|
||||||
read-only SQLite schema summary/hash and `crsql_changes` metadata when
|
read-only SQLite schema summary/hash and `crsql_changes` metadata when
|
||||||
present. The DB crate and daemon can extract typed read-only `crsql_changes`
|
present. The DB crate and daemon can extract typed read-only `crsql_changes`
|
||||||
|
|
|
||||||
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1118,6 +1118,7 @@ name = "geth-control"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"geth-auth",
|
"geth-auth",
|
||||||
|
"geth-cas",
|
||||||
"geth-db",
|
"geth-db",
|
||||||
"geth-document",
|
"geth-document",
|
||||||
"geth-keychain",
|
"geth-keychain",
|
||||||
|
|
|
||||||
|
|
@ -88,6 +88,7 @@ The bootstrap implementation provides:
|
||||||
- local filesystem CAS commands: `add`, `get`, `hash`, `has`, `pin`, `unpin`,
|
- local filesystem CAS commands: `add`, `get`, `hash`, `has`, `pin`, `unpin`,
|
||||||
`cleanup`, `list`
|
`cleanup`, `list`
|
||||||
- local CAS tree objects describe file trees and are stored as CAS blobs
|
- local CAS tree objects describe file trees and are stored as CAS blobs
|
||||||
|
- local file-root commands: `geth cas root add/list/scan`
|
||||||
- local DB resource registration: `geth db add <name> <path>` and
|
- local DB resource registration: `geth db add <name> <path>` and
|
||||||
`geth db status <name>` with schema and `crsql_changes` metadata; the DB
|
`geth db status <name>` with schema and `crsql_changes` metadata; the DB
|
||||||
crate and daemon can extract typed local `crsql_changes` batches through
|
crate and daemon can extract typed local `crsql_changes` batches through
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
use geth_types::BlobHash;
|
use geth_types::BlobHash;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
pub const CAS_TREE_OBJECT_VERSION: u16 = 1;
|
pub const CAS_TREE_OBJECT_VERSION: u16 = 1;
|
||||||
|
|
@ -18,6 +19,8 @@ pub enum CasError {
|
||||||
TreeRootNotDirectory(String),
|
TreeRootNotDirectory(String),
|
||||||
#[error("CAS tree paths must be relative: {0}")]
|
#[error("CAS tree paths must be relative: {0}")]
|
||||||
NonRelativeTreePath(String),
|
NonRelativeTreePath(String),
|
||||||
|
#[error("invalid file root name: {0}")]
|
||||||
|
InvalidFileRootName(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
|
|
@ -60,6 +63,39 @@ pub struct CasTreeStored {
|
||||||
pub object: BlobInfo,
|
pub object: BlobInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct FileRoot {
|
||||||
|
pub id: String,
|
||||||
|
pub resource: String,
|
||||||
|
pub name: String,
|
||||||
|
pub path: String,
|
||||||
|
pub latest_tree: Option<BlobHash>,
|
||||||
|
pub updated_at_ms: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct FileRootScan {
|
||||||
|
pub root: FileRoot,
|
||||||
|
pub tree: BlobInfoSummary,
|
||||||
|
pub changes: Vec<FileRootChange>,
|
||||||
|
pub note: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct BlobInfoSummary {
|
||||||
|
pub hash: BlobHash,
|
||||||
|
pub size_bytes: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(tag = "kind", rename_all = "kebab-case")]
|
||||||
|
pub enum FileRootChange {
|
||||||
|
Created { path: String },
|
||||||
|
Modified { path: String },
|
||||||
|
Deleted { path: String },
|
||||||
|
Renamed { from: String, to: String },
|
||||||
|
}
|
||||||
|
|
||||||
impl LocalCas {
|
impl LocalCas {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn new(root: impl Into<PathBuf>) -> Self {
|
pub fn new(root: impl Into<PathBuf>) -> Self {
|
||||||
|
|
@ -188,6 +224,107 @@ pub fn build_tree_object(cas: &LocalCas, root: &Path) -> Result<CasTreeObject, C
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn diff_tree_objects(
|
||||||
|
previous: Option<&CasTreeObject>,
|
||||||
|
current: &CasTreeObject,
|
||||||
|
) -> Vec<FileRootChange> {
|
||||||
|
let Some(previous) = previous else {
|
||||||
|
return current
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.map(|entry| FileRootChange::Created {
|
||||||
|
path: entry.path.clone(),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
};
|
||||||
|
|
||||||
|
let previous_entries = tree_entry_map(previous);
|
||||||
|
let current_entries = tree_entry_map(current);
|
||||||
|
let previous_paths = previous_entries.keys().cloned().collect::<BTreeSet<_>>();
|
||||||
|
let current_paths = current_entries.keys().cloned().collect::<BTreeSet<_>>();
|
||||||
|
let deleted = previous_paths
|
||||||
|
.difference(¤t_paths)
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let created = current_paths
|
||||||
|
.difference(&previous_paths)
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let mut changes = Vec::new();
|
||||||
|
let mut consumed_deleted = BTreeSet::new();
|
||||||
|
let mut consumed_created = BTreeSet::new();
|
||||||
|
for deleted_path in &deleted {
|
||||||
|
let Some(deleted_entry) = previous_entries.get(deleted_path) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(deleted_blob) = &deleted_entry.blob else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if let Some(created_path) = created.iter().find(|created_path| {
|
||||||
|
!consumed_created.contains(*created_path)
|
||||||
|
&& current_entries
|
||||||
|
.get(*created_path)
|
||||||
|
.and_then(|entry| entry.blob.as_ref())
|
||||||
|
== Some(deleted_blob)
|
||||||
|
}) {
|
||||||
|
changes.push(FileRootChange::Renamed {
|
||||||
|
from: deleted_path.clone(),
|
||||||
|
to: created_path.clone(),
|
||||||
|
});
|
||||||
|
consumed_deleted.insert(deleted_path.clone());
|
||||||
|
consumed_created.insert(created_path.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for path in deleted {
|
||||||
|
if !consumed_deleted.contains(&path) {
|
||||||
|
changes.push(FileRootChange::Deleted { path });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for path in created {
|
||||||
|
if !consumed_created.contains(&path) {
|
||||||
|
changes.push(FileRootChange::Created { path });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for path in previous_paths.intersection(¤t_paths) {
|
||||||
|
let previous = previous_entries.get(path).expect("previous entry");
|
||||||
|
let current = current_entries.get(path).expect("current entry");
|
||||||
|
if previous.kind != current.kind
|
||||||
|
|| previous.blob != current.blob
|
||||||
|
|| previous.executable != current.executable
|
||||||
|
|| previous.size_bytes != current.size_bytes
|
||||||
|
{
|
||||||
|
changes.push(FileRootChange::Modified { path: path.clone() });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
changes.sort_by_key(change_sort_key);
|
||||||
|
changes
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn file_root_scan_note() -> &'static str {
|
||||||
|
"local scan only; geth never overwrites file roots without a recorded future sync decision"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tree_entry_map(tree: &CasTreeObject) -> BTreeMap<String, &CasTreeEntry> {
|
||||||
|
tree.entries
|
||||||
|
.iter()
|
||||||
|
.map(|entry| (entry.path.clone(), entry))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn change_sort_key(change: &FileRootChange) -> (u8, String, String) {
|
||||||
|
match change {
|
||||||
|
FileRootChange::Created { path } => (0, path.clone(), String::new()),
|
||||||
|
FileRootChange::Modified { path } => (1, path.clone(), String::new()),
|
||||||
|
FileRootChange::Deleted { path } => (2, path.clone(), String::new()),
|
||||||
|
FileRootChange::Renamed { from, to } => (3, from.clone(), to.clone()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn collect_tree_entries(
|
fn collect_tree_entries(
|
||||||
cas: &LocalCas,
|
cas: &LocalCas,
|
||||||
root: &Path,
|
root: &Path,
|
||||||
|
|
@ -277,6 +414,17 @@ pub fn is_valid_hash(hash: &str) -> bool {
|
||||||
hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit())
|
hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn validate_file_root_name(name: &str) -> Result<(), CasError> {
|
||||||
|
if name.is_empty()
|
||||||
|
|| !name
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
|
||||||
|
{
|
||||||
|
return Err(CasError::InvalidFileRootName(name.to_owned()));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
@ -351,4 +499,75 @@ mod tests {
|
||||||
Err(CasError::TreeRootNotDirectory(_))
|
Err(CasError::TreeRootNotDirectory(_))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cas_tree_diff_detects_create_modify_delete_and_rename() {
|
||||||
|
let previous = CasTreeObject {
|
||||||
|
version: CAS_TREE_OBJECT_VERSION,
|
||||||
|
entries: vec![
|
||||||
|
tree_file("deleted.txt", "01", 1),
|
||||||
|
tree_file("modified.txt", "02", 1),
|
||||||
|
tree_file("old.txt", "03", 1),
|
||||||
|
],
|
||||||
|
};
|
||||||
|
let current = CasTreeObject {
|
||||||
|
version: CAS_TREE_OBJECT_VERSION,
|
||||||
|
entries: vec![
|
||||||
|
tree_file("created.txt", "04", 1),
|
||||||
|
tree_file("modified.txt", "05", 1),
|
||||||
|
tree_file("new.txt", "03", 1),
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
let diff = diff_tree_objects(Some(&previous), ¤t);
|
||||||
|
|
||||||
|
assert!(diff.contains(&FileRootChange::Created {
|
||||||
|
path: "created.txt".to_owned()
|
||||||
|
}));
|
||||||
|
assert!(diff.contains(&FileRootChange::Modified {
|
||||||
|
path: "modified.txt".to_owned()
|
||||||
|
}));
|
||||||
|
assert!(diff.contains(&FileRootChange::Deleted {
|
||||||
|
path: "deleted.txt".to_owned()
|
||||||
|
}));
|
||||||
|
assert!(diff.contains(&FileRootChange::Renamed {
|
||||||
|
from: "old.txt".to_owned(),
|
||||||
|
to: "new.txt".to_owned(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cas_tree_diff_does_not_reuse_rename_targets() {
|
||||||
|
let previous = CasTreeObject {
|
||||||
|
version: CAS_TREE_OBJECT_VERSION,
|
||||||
|
entries: vec![tree_file("one.txt", "01", 1), tree_file("two.txt", "01", 1)],
|
||||||
|
};
|
||||||
|
let current = CasTreeObject {
|
||||||
|
version: CAS_TREE_OBJECT_VERSION,
|
||||||
|
entries: vec![tree_file("renamed.txt", "01", 1)],
|
||||||
|
};
|
||||||
|
|
||||||
|
let diff = diff_tree_objects(Some(&previous), ¤t);
|
||||||
|
let rename_count = diff
|
||||||
|
.iter()
|
||||||
|
.filter(|change| matches!(change, FileRootChange::Renamed { .. }))
|
||||||
|
.count();
|
||||||
|
let delete_count = diff
|
||||||
|
.iter()
|
||||||
|
.filter(|change| matches!(change, FileRootChange::Deleted { .. }))
|
||||||
|
.count();
|
||||||
|
|
||||||
|
assert_eq!(rename_count, 1);
|
||||||
|
assert_eq!(delete_count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tree_file(path: &str, byte: &str, size_bytes: u64) -> CasTreeEntry {
|
||||||
|
CasTreeEntry {
|
||||||
|
path: path.to_owned(),
|
||||||
|
kind: CasTreeEntryKind::File,
|
||||||
|
blob: Some(BlobHash::new(byte.repeat(32))),
|
||||||
|
size_bytes,
|
||||||
|
executable: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -216,6 +216,17 @@ pub enum CasCommand {
|
||||||
dry_run: bool,
|
dry_run: bool,
|
||||||
},
|
},
|
||||||
List,
|
List,
|
||||||
|
Root {
|
||||||
|
#[command(subcommand)]
|
||||||
|
command: CasRootCommand,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Subcommand)]
|
||||||
|
pub enum CasRootCommand {
|
||||||
|
Add { name: String, path: PathBuf },
|
||||||
|
List,
|
||||||
|
Scan { name: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Subcommand)]
|
#[derive(Debug, Subcommand)]
|
||||||
|
|
@ -464,6 +475,11 @@ fn request_for_command(command: Command) -> Result<ControlRequest> {
|
||||||
CasCommand::Unpin { hash } => ControlRequest::CasUnpin { hash: hash.into() },
|
CasCommand::Unpin { hash } => ControlRequest::CasUnpin { hash: hash.into() },
|
||||||
CasCommand::Cleanup { dry_run } => ControlRequest::CasCleanup { dry_run },
|
CasCommand::Cleanup { dry_run } => ControlRequest::CasCleanup { dry_run },
|
||||||
CasCommand::List => ControlRequest::CasList,
|
CasCommand::List => ControlRequest::CasList,
|
||||||
|
CasCommand::Root { command } => match command {
|
||||||
|
CasRootCommand::Add { name, path } => ControlRequest::CasRootAdd { name, path },
|
||||||
|
CasRootCommand::List => ControlRequest::CasRootList,
|
||||||
|
CasRootCommand::Scan { name } => ControlRequest::CasRootScan { name },
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Command::Kv { command } => match command {
|
Command::Kv { command } => match command {
|
||||||
KvCommand::Create { name } => ControlRequest::KvCreate { name },
|
KvCommand::Create { name } => ControlRequest::KvCreate { name },
|
||||||
|
|
@ -696,6 +712,38 @@ fn print_response(response: ControlResponse, json: bool) -> Result<()> {
|
||||||
println!("{}\t{} bytes\t{}", blob.hash, blob.size_bytes, pin);
|
println!("{}\t{} bytes\t{}", blob.hash, blob.size_bytes, pin);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ControlResponse::CasRootAdded { root } => {
|
||||||
|
println!("added file root: {}", root.name);
|
||||||
|
println!("id: {}", root.id);
|
||||||
|
println!("resource: {}", root.resource);
|
||||||
|
println!("path: {}", root.path);
|
||||||
|
}
|
||||||
|
ControlResponse::CasRootList { roots } => {
|
||||||
|
if roots.is_empty() {
|
||||||
|
println!("no file roots");
|
||||||
|
} else {
|
||||||
|
for root in roots {
|
||||||
|
println!(
|
||||||
|
"{}\t{}\t{}",
|
||||||
|
root.name,
|
||||||
|
root.path,
|
||||||
|
root.latest_tree
|
||||||
|
.map(|hash| hash.to_string())
|
||||||
|
.unwrap_or_else(|| "unscanned".to_owned())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ControlResponse::CasRootScanned { scan } => {
|
||||||
|
println!("file root: {}", scan.root.name);
|
||||||
|
println!("tree: {}", scan.tree.hash);
|
||||||
|
println!("tree_bytes: {}", scan.tree.size_bytes);
|
||||||
|
println!("changes: {}", scan.changes.len());
|
||||||
|
for change in scan.changes {
|
||||||
|
println!("{change:?}");
|
||||||
|
}
|
||||||
|
println!("note: {}", scan.note);
|
||||||
|
}
|
||||||
ControlResponse::KeychainStatus(status) => {
|
ControlResponse::KeychainStatus(status) => {
|
||||||
println!("initialized: {}", status.initialized);
|
println!("initialized: {}", status.initialized);
|
||||||
println!("admin_keys: {}", status.admin_keys);
|
println!("admin_keys: {}", status.admin_keys);
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
geth-auth = { path = "../geth-auth" }
|
geth-auth = { path = "../geth-auth" }
|
||||||
|
geth-cas = { path = "../geth-cas" }
|
||||||
geth-db = { path = "../geth-db" }
|
geth-db = { path = "../geth-db" }
|
||||||
geth-document = { path = "../geth-document" }
|
geth-document = { path = "../geth-document" }
|
||||||
geth-keychain = { path = "../geth-keychain" }
|
geth-keychain = { path = "../geth-keychain" }
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
use geth_auth::{AuthExplanation, AuthOp};
|
use geth_auth::{AuthExplanation, AuthOp};
|
||||||
|
use geth_cas::{FileRoot, FileRootScan};
|
||||||
use geth_db::{CrSqliteChangeBatch, DbResource};
|
use geth_db::{CrSqliteChangeBatch, DbResource};
|
||||||
use geth_document::{DocumentResource, DocumentState};
|
use geth_document::{DocumentResource, DocumentState};
|
||||||
use geth_keychain::KeychainOp;
|
use geth_keychain::KeychainOp;
|
||||||
|
|
@ -47,6 +48,14 @@ pub enum ControlRequest {
|
||||||
dry_run: bool,
|
dry_run: bool,
|
||||||
},
|
},
|
||||||
CasList,
|
CasList,
|
||||||
|
CasRootAdd {
|
||||||
|
name: String,
|
||||||
|
path: PathBuf,
|
||||||
|
},
|
||||||
|
CasRootList,
|
||||||
|
CasRootScan {
|
||||||
|
name: String,
|
||||||
|
},
|
||||||
KeychainInit {
|
KeychainInit {
|
||||||
admin_key_path: Option<PathBuf>,
|
admin_key_path: Option<PathBuf>,
|
||||||
},
|
},
|
||||||
|
|
@ -209,6 +218,15 @@ pub enum ControlResponse {
|
||||||
CasList {
|
CasList {
|
||||||
blobs: Vec<CasBlob>,
|
blobs: Vec<CasBlob>,
|
||||||
},
|
},
|
||||||
|
CasRootAdded {
|
||||||
|
root: FileRoot,
|
||||||
|
},
|
||||||
|
CasRootList {
|
||||||
|
roots: Vec<FileRoot>,
|
||||||
|
},
|
||||||
|
CasRootScanned {
|
||||||
|
scan: FileRootScan,
|
||||||
|
},
|
||||||
KeychainStatus(KeychainStatusResponse),
|
KeychainStatus(KeychainStatusResponse),
|
||||||
KeychainInitialized {
|
KeychainInitialized {
|
||||||
ops: Vec<KeychainOp>,
|
ops: Vec<KeychainOp>,
|
||||||
|
|
@ -446,5 +464,13 @@ mod tests {
|
||||||
decode_request(&encode_request(&request).expect("encode")).expect("decode"),
|
decode_request(&encode_request(&request).expect("encode")).expect("decode"),
|
||||||
request
|
request
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let request = ControlRequest::CasRootScan {
|
||||||
|
name: "notes".to_owned(),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
decode_request(&encode_request(&request).expect("encode")).expect("decode"),
|
||||||
|
request
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
pub mod service;
|
pub mod service;
|
||||||
|
|
||||||
use geth_auth::{AuthExplanation, AuthOp, AuthOpKind};
|
use geth_auth::{AuthExplanation, AuthOp, AuthOpKind};
|
||||||
use geth_cas::{LocalCas, hash_path};
|
use geth_cas::{BlobInfoSummary, FileRoot, FileRootScan, LocalCas, hash_path};
|
||||||
use geth_config::{GethConfig, GethPaths, RelayMode};
|
use geth_config::{GethConfig, GethPaths, RelayMode};
|
||||||
use geth_control::{
|
use geth_control::{
|
||||||
CasBlob, ControlRequest, ControlResponse, KeychainStatusResponse, NodeIdResponse,
|
CasBlob, ControlRequest, ControlResponse, KeychainStatusResponse, NodeIdResponse,
|
||||||
|
|
@ -23,9 +23,9 @@ use geth_ssh_identity::{
|
||||||
cert_request_id, certificate_id, openssh_krl_spec, revocation_id, ssh_public_key_fingerprint,
|
cert_request_id, certificate_id, openssh_krl_spec, revocation_id, ssh_public_key_fingerprint,
|
||||||
};
|
};
|
||||||
use geth_store::{
|
use geth_store::{
|
||||||
Store, StoredAuthOp, StoredDbResource, StoredDocumentResource, StoredKeychainOp, StoredKvEntry,
|
Store, StoredAuthOp, StoredDbResource, StoredDocumentResource, StoredFileRoot,
|
||||||
StoredKvStore, StoredResource, StoredResourceSecret, StoredSshCertRequest,
|
StoredKeychainOp, StoredKvEntry, StoredKvStore, StoredResource, StoredResourceSecret,
|
||||||
StoredSshCertificate, StoredSshRevocation,
|
StoredSshCertRequest, StoredSshCertificate, StoredSshRevocation,
|
||||||
};
|
};
|
||||||
use geth_types::{
|
use geth_types::{
|
||||||
AuthOpId, Capability, KeyId, NodeId, PrincipalId, ResourceId, ResourceKind, ResourceName,
|
AuthOpId, Capability, KeyId, NodeId, PrincipalId, ResourceId, ResourceKind, ResourceName,
|
||||||
|
|
@ -344,6 +344,76 @@ pub fn handle_request(
|
||||||
.collect::<Result<Vec<_>, NodeError>>()?;
|
.collect::<Result<Vec<_>, NodeError>>()?;
|
||||||
Ok(ControlResponse::CasList { blobs })
|
Ok(ControlResponse::CasList { blobs })
|
||||||
}
|
}
|
||||||
|
ControlRequest::CasRootAdd { name, path } => {
|
||||||
|
geth_cas::validate_file_root_name(&name)?;
|
||||||
|
if !path.is_dir() {
|
||||||
|
return Err(NodeError::Cas(geth_cas::CasError::TreeRootNotDirectory(
|
||||||
|
path.display().to_string(),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let path = std::fs::canonicalize(path)?;
|
||||||
|
let resource_id = format!("resource:cas-tree:{name}");
|
||||||
|
store.insert_resource(&StoredResource {
|
||||||
|
resource_id: resource_id.clone(),
|
||||||
|
kind: ResourceKind::Cas.to_string(),
|
||||||
|
name: format!("file-root:{name}"),
|
||||||
|
status: "active".to_owned(),
|
||||||
|
})?;
|
||||||
|
let stored = StoredFileRoot {
|
||||||
|
root_id: format!("file-root:{name}"),
|
||||||
|
resource_id,
|
||||||
|
name,
|
||||||
|
path: path.display().to_string(),
|
||||||
|
latest_tree_hash: None,
|
||||||
|
latest_tree_json: None,
|
||||||
|
updated_at_ms: geth_store::now_ms(),
|
||||||
|
};
|
||||||
|
store.upsert_file_root(&stored)?;
|
||||||
|
Ok(ControlResponse::CasRootAdded {
|
||||||
|
root: file_root_from_stored(&stored),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
ControlRequest::CasRootList => Ok(ControlResponse::CasRootList {
|
||||||
|
roots: store
|
||||||
|
.list_file_roots()?
|
||||||
|
.iter()
|
||||||
|
.map(file_root_from_stored)
|
||||||
|
.collect(),
|
||||||
|
}),
|
||||||
|
ControlRequest::CasRootScan { name } => {
|
||||||
|
geth_cas::validate_file_root_name(&name)?;
|
||||||
|
let mut stored = store
|
||||||
|
.get_file_root_by_name(&name)?
|
||||||
|
.ok_or_else(|| NodeError::ResourceNotFound(format!("file-root:{name}")))?;
|
||||||
|
let previous_tree = stored
|
||||||
|
.latest_tree_json
|
||||||
|
.as_deref()
|
||||||
|
.map(serde_json::from_str)
|
||||||
|
.transpose()?;
|
||||||
|
let cas = LocalCas::new(node.paths.cas_dir());
|
||||||
|
let scanned = cas.add_tree_path(Path::new(&stored.path))?;
|
||||||
|
store.record_cas_object(
|
||||||
|
scanned.object.hash.as_str(),
|
||||||
|
scanned.object.size_bytes,
|
||||||
|
&scanned.object.path.to_string_lossy(),
|
||||||
|
)?;
|
||||||
|
let changes = geth_cas::diff_tree_objects(previous_tree.as_ref(), &scanned.tree);
|
||||||
|
stored.latest_tree_hash = Some(scanned.object.hash.to_string());
|
||||||
|
stored.latest_tree_json = Some(serde_json::to_string(&scanned.tree)?);
|
||||||
|
stored.updated_at_ms = geth_store::now_ms();
|
||||||
|
store.upsert_file_root(&stored)?;
|
||||||
|
Ok(ControlResponse::CasRootScanned {
|
||||||
|
scan: FileRootScan {
|
||||||
|
root: file_root_from_stored(&stored),
|
||||||
|
tree: BlobInfoSummary {
|
||||||
|
hash: scanned.object.hash,
|
||||||
|
size_bytes: scanned.object.size_bytes,
|
||||||
|
},
|
||||||
|
changes,
|
||||||
|
note: geth_cas::file_root_scan_note().to_owned(),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
ControlRequest::KeychainInit { admin_key_path } => {
|
ControlRequest::KeychainInit { admin_key_path } => {
|
||||||
let mut ops = Vec::new();
|
let mut ops = Vec::new();
|
||||||
let created_at = UnixMillis(geth_store::now_ms());
|
let created_at = UnixMillis(geth_store::now_ms());
|
||||||
|
|
@ -1055,6 +1125,17 @@ fn document_state_from_stored(stored: &StoredDocumentResource) -> DocumentState
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn file_root_from_stored(stored: &StoredFileRoot) -> FileRoot {
|
||||||
|
FileRoot {
|
||||||
|
id: stored.root_id.clone(),
|
||||||
|
resource: stored.resource_id.clone(),
|
||||||
|
name: stored.name.clone(),
|
||||||
|
path: stored.path.clone(),
|
||||||
|
latest_tree: stored.latest_tree_hash.clone().map(Into::into),
|
||||||
|
updated_at_ms: stored.updated_at_ms,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn ensure_resource_exists(store: &Store, resource_id: &str) -> Result<(), NodeError> {
|
fn ensure_resource_exists(store: &Store, resource_id: &str) -> Result<(), NodeError> {
|
||||||
if store
|
if store
|
||||||
.list_resources()?
|
.list_resources()?
|
||||||
|
|
|
||||||
|
|
@ -124,6 +124,15 @@ impl Store {
|
||||||
state_json TEXT NOT NULL DEFAULT '{}',
|
state_json TEXT NOT NULL DEFAULT '{}',
|
||||||
updated_at_ms INTEGER NOT NULL DEFAULT 0
|
updated_at_ms INTEGER NOT NULL DEFAULT 0
|
||||||
);
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS file_roots (
|
||||||
|
root_id TEXT PRIMARY KEY,
|
||||||
|
resource_id TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL UNIQUE,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
latest_tree_hash TEXT,
|
||||||
|
latest_tree_json TEXT,
|
||||||
|
updated_at_ms INTEGER NOT NULL
|
||||||
|
);
|
||||||
CREATE TABLE IF NOT EXISTS peer_cards (
|
CREATE TABLE IF NOT EXISTS peer_cards (
|
||||||
peer_id TEXT PRIMARY KEY,
|
peer_id TEXT PRIMARY KEY,
|
||||||
card_json TEXT NOT NULL,
|
card_json TEXT NOT NULL,
|
||||||
|
|
@ -382,6 +391,65 @@ impl Store {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn upsert_file_root(&self, root: &StoredFileRoot) -> Result<(), StoreError> {
|
||||||
|
self.conn.execute(
|
||||||
|
r#"INSERT OR REPLACE INTO file_roots(
|
||||||
|
root_id, resource_id, name, path, latest_tree_hash, latest_tree_json, updated_at_ms
|
||||||
|
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"#,
|
||||||
|
params![
|
||||||
|
root.root_id,
|
||||||
|
root.resource_id,
|
||||||
|
root.name,
|
||||||
|
root.path,
|
||||||
|
root.latest_tree_hash,
|
||||||
|
root.latest_tree_json,
|
||||||
|
root.updated_at_ms
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_file_root_by_name(&self, name: &str) -> Result<Option<StoredFileRoot>, StoreError> {
|
||||||
|
let mut stmt = self.conn.prepare(
|
||||||
|
r#"SELECT root_id, resource_id, name, path, latest_tree_hash, latest_tree_json, updated_at_ms
|
||||||
|
FROM file_roots WHERE name = ?1"#,
|
||||||
|
)?;
|
||||||
|
let mut rows = stmt.query(params![name])?;
|
||||||
|
if let Some(row) = rows.next()? {
|
||||||
|
Ok(Some(StoredFileRoot {
|
||||||
|
root_id: row.get(0)?,
|
||||||
|
resource_id: row.get(1)?,
|
||||||
|
name: row.get(2)?,
|
||||||
|
path: row.get(3)?,
|
||||||
|
latest_tree_hash: row.get(4)?,
|
||||||
|
latest_tree_json: row.get(5)?,
|
||||||
|
updated_at_ms: row.get(6)?,
|
||||||
|
}))
|
||||||
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_file_roots(&self) -> Result<Vec<StoredFileRoot>, StoreError> {
|
||||||
|
let mut stmt = self.conn.prepare(
|
||||||
|
r#"SELECT root_id, resource_id, name, path, latest_tree_hash, latest_tree_json, updated_at_ms
|
||||||
|
FROM file_roots ORDER BY name"#,
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map([], |row| {
|
||||||
|
Ok(StoredFileRoot {
|
||||||
|
root_id: row.get(0)?,
|
||||||
|
resource_id: row.get(1)?,
|
||||||
|
name: row.get(2)?,
|
||||||
|
path: row.get(3)?,
|
||||||
|
latest_tree_hash: row.get(4)?,
|
||||||
|
latest_tree_json: row.get(5)?,
|
||||||
|
updated_at_ms: row.get(6)?,
|
||||||
|
})
|
||||||
|
})?;
|
||||||
|
rows.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map_err(StoreError::from)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn insert_resource_secret(&self, secret: &StoredResourceSecret) -> Result<(), StoreError> {
|
pub fn insert_resource_secret(&self, secret: &StoredResourceSecret) -> Result<(), StoreError> {
|
||||||
self.conn.execute(
|
self.conn.execute(
|
||||||
r#"INSERT OR REPLACE INTO resource_secrets(
|
r#"INSERT OR REPLACE INTO resource_secrets(
|
||||||
|
|
@ -829,6 +897,17 @@ pub struct StoredDocumentResource {
|
||||||
pub updated_at_ms: i64,
|
pub updated_at_ms: i64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct StoredFileRoot {
|
||||||
|
pub root_id: String,
|
||||||
|
pub resource_id: String,
|
||||||
|
pub name: String,
|
||||||
|
pub path: String,
|
||||||
|
pub latest_tree_hash: Option<String>,
|
||||||
|
pub latest_tree_json: Option<String>,
|
||||||
|
pub updated_at_ms: i64,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub struct StoredResourceSecret {
|
pub struct StoredResourceSecret {
|
||||||
pub secret_id: String,
|
pub secret_id: String,
|
||||||
|
|
@ -1101,6 +1180,30 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn file_roots_roundtrip_by_name() {
|
||||||
|
let store = Store::open_memory().expect("open");
|
||||||
|
let root = StoredFileRoot {
|
||||||
|
root_id: "file-root:notes".to_owned(),
|
||||||
|
resource_id: "resource:cas-tree:notes".to_owned(),
|
||||||
|
name: "notes".to_owned(),
|
||||||
|
path: "/tmp/notes".to_owned(),
|
||||||
|
latest_tree_hash: Some(
|
||||||
|
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(),
|
||||||
|
),
|
||||||
|
latest_tree_json: Some("{}".to_owned()),
|
||||||
|
updated_at_ms: 10,
|
||||||
|
};
|
||||||
|
|
||||||
|
store.upsert_file_root(&root).expect("insert root");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
store.get_file_root_by_name("notes").expect("get file root"),
|
||||||
|
Some(root.clone())
|
||||||
|
);
|
||||||
|
assert_eq!(store.list_file_roots().expect("list roots"), vec![root]);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn kv_store_and_entries_roundtrip() {
|
fn kv_store_and_entries_roundtrip() {
|
||||||
let store = Store::open_memory().expect("open");
|
let store = Store::open_memory().expect("open");
|
||||||
|
|
|
||||||
|
|
@ -168,6 +168,85 @@ fn cas_pin_unpin_updates_local_pin_metadata() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cas_file_root_add_list_scan_detects_changes() {
|
||||||
|
let home = tempfile::tempdir().expect("tempdir");
|
||||||
|
let paths = geth_config::GethPaths::from_home(home.path());
|
||||||
|
let node = geth_node::init_node(&paths).expect("init node");
|
||||||
|
let root_path = home.path().join("notes-root");
|
||||||
|
std::fs::create_dir_all(&root_path).expect("create root");
|
||||||
|
std::fs::write(root_path.join("a.txt"), b"alpha").expect("write a");
|
||||||
|
|
||||||
|
let response = geth_node::handle_request(
|
||||||
|
&node,
|
||||||
|
geth_control::ControlRequest::CasRootAdd {
|
||||||
|
name: "notes".to_owned(),
|
||||||
|
path: root_path.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("add root");
|
||||||
|
match response {
|
||||||
|
geth_control::ControlResponse::CasRootAdded { root } => {
|
||||||
|
assert_eq!(root.name, "notes");
|
||||||
|
assert!(root.latest_tree.is_none());
|
||||||
|
}
|
||||||
|
other => panic!("unexpected response: {other:?}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
let response = geth_node::handle_request(&node, geth_control::ControlRequest::CasRootList)
|
||||||
|
.expect("list roots");
|
||||||
|
match response {
|
||||||
|
geth_control::ControlResponse::CasRootList { roots } => {
|
||||||
|
assert_eq!(roots.len(), 1);
|
||||||
|
assert_eq!(roots[0].name, "notes");
|
||||||
|
}
|
||||||
|
other => panic!("unexpected response: {other:?}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
let response = geth_node::handle_request(
|
||||||
|
&node,
|
||||||
|
geth_control::ControlRequest::CasRootScan {
|
||||||
|
name: "notes".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("initial scan");
|
||||||
|
let first_tree = match response {
|
||||||
|
geth_control::ControlResponse::CasRootScanned { scan } => {
|
||||||
|
assert_eq!(scan.changes.len(), 1);
|
||||||
|
assert!(scan.root.latest_tree.is_some());
|
||||||
|
assert!(scan.note.contains("never overwrites"));
|
||||||
|
scan.tree.hash
|
||||||
|
}
|
||||||
|
other => panic!("unexpected response: {other:?}"),
|
||||||
|
};
|
||||||
|
|
||||||
|
std::fs::rename(root_path.join("a.txt"), root_path.join("b.txt")).expect("rename a to b");
|
||||||
|
std::fs::write(root_path.join("c.txt"), b"charlie").expect("write c");
|
||||||
|
let response = geth_node::handle_request(
|
||||||
|
&node,
|
||||||
|
geth_control::ControlRequest::CasRootScan {
|
||||||
|
name: "notes".to_owned(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("second scan");
|
||||||
|
match response {
|
||||||
|
geth_control::ControlResponse::CasRootScanned { scan } => {
|
||||||
|
assert_ne!(scan.tree.hash, first_tree);
|
||||||
|
assert!(scan.changes.iter().any(|change| {
|
||||||
|
matches!(
|
||||||
|
change,
|
||||||
|
geth_cas::FileRootChange::Renamed { from, to }
|
||||||
|
if from == "a.txt" && to == "b.txt"
|
||||||
|
)
|
||||||
|
}));
|
||||||
|
assert!(scan.changes.iter().any(|change| {
|
||||||
|
matches!(change, geth_cas::FileRootChange::Created { path } if path == "c.txt")
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
other => panic!("unexpected response: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cas_cleanup_removes_unpinned_and_retains_pinned_blobs() {
|
fn cas_cleanup_removes_unpinned_and_retains_pinned_blobs() {
|
||||||
let home = tempfile::tempdir().expect("tempdir");
|
let home = tempfile::tempdir().expect("tempdir");
|
||||||
|
|
|
||||||
|
|
@ -92,8 +92,11 @@ storage. Local pin/unpin metadata is tracked in SQLite and surfaced in
|
||||||
`cas list`. `cas cleanup` removes unpinned local blobs while retaining pinned
|
`cas list`. `cas cleanup` removes unpinned local blobs while retaining pinned
|
||||||
blobs. The CAS crate can build deterministic tree objects that describe
|
blobs. The CAS crate can build deterministic tree objects that describe
|
||||||
directories, files, executable bits, and file blob hashes; those tree objects
|
directories, files, executable bits, and file blob hashes; those tree objects
|
||||||
are stored as CAS blobs. Iroh-blobs, providers, encrypted blobs, richer cache
|
are stored as CAS blobs. The daemon can register local file roots and scan them
|
||||||
policies, file roots, and sync conflict handling are future work.
|
into CAS tree objects while reporting create/update/delete/rename changes. These
|
||||||
|
scans are local metadata only and never overwrite the working tree. Iroh-blobs,
|
||||||
|
providers, encrypted blobs, richer cache policies, cross-node file roots, and
|
||||||
|
sync conflict handling are future work.
|
||||||
|
|
||||||
`geth-db` currently registers local SQLite paths as DB resources and reports
|
`geth-db` currently registers local SQLite paths as DB resources and reports
|
||||||
local-only sync status plus a read-only SQLite schema summary/hash. It also
|
local-only sync status plus a read-only SQLite schema summary/hash. It also
|
||||||
|
|
|
||||||
|
|
@ -351,11 +351,15 @@ and future group key evolution.
|
||||||
- `[x]` Tree objects are content-addressed and stored in CAS.
|
- `[x]` Tree objects are content-addressed and stored in CAS.
|
||||||
- `[x]` Tests cover deterministic tree hashing.
|
- `[x]` Tests cover deterministic tree hashing.
|
||||||
|
|
||||||
- `[ ]` File roots.
|
- `[~]` File roots.
|
||||||
Acceptance criteria:
|
Acceptance criteria:
|
||||||
- A file root maps a local path to a CAS tree resource.
|
- `[x]` A file root maps a local path to a CAS tree resource.
|
||||||
- Scan detects create/update/delete/rename changes.
|
- `[x]` `geth cas root add/list/scan` persists local root metadata and latest
|
||||||
- Sync never silently overwrites local changes without a recorded decision.
|
tree state.
|
||||||
|
- `[x]` Scan detects create/update/delete/rename changes.
|
||||||
|
- `[x]` Scan output documents that geth never overwrites file roots without a
|
||||||
|
recorded future sync decision.
|
||||||
|
- `[ ]` File roots can be synced across nodes.
|
||||||
|
|
||||||
- `[ ]` Conflict handling.
|
- `[ ]` Conflict handling.
|
||||||
Acceptance criteria:
|
Acceptance criteria:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue