Add deterministic CAS tree objects
This commit is contained in:
parent
a44eef3126
commit
64e17ff957
7 changed files with 194 additions and 6 deletions
|
|
@ -118,6 +118,9 @@ Roadmap items should be actionable and checkable:
|
|||
`keychain status`. SSH signature capture/verification is still roadmap work.
|
||||
- Local CAS supports pin/unpin metadata, surfaced through `cas list`, and
|
||||
`cas cleanup` evicts unpinned blobs while retaining pinned blobs.
|
||||
- The CAS crate can build deterministic tree objects for local file trees and
|
||||
store those manifests as CAS blobs. File roots, scan workflows, and sync
|
||||
conflict handling are still roadmap work.
|
||||
- DB resources can be registered locally and report local-only status plus a
|
||||
read-only SQLite schema summary/hash and `crsql_changes` metadata when
|
||||
present. The DB crate and daemon can extract typed read-only `crsql_changes`
|
||||
|
|
|
|||
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -1069,8 +1069,10 @@ name = "geth-cas"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"geth-codec",
|
||||
"geth-types",
|
||||
"hex",
|
||||
"serde",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ The bootstrap implementation provides:
|
|||
- `geth auth revoke <resource> <grant-id>`
|
||||
- local filesystem CAS commands: `add`, `get`, `hash`, `has`, `pin`, `unpin`,
|
||||
`cleanup`, `list`
|
||||
- local CAS tree objects describe file trees and are stored as CAS blobs
|
||||
- local DB resource registration: `geth db add <name> <path>` and
|
||||
`geth db status <name>` with schema and `crsql_changes` metadata; the DB
|
||||
crate and daemon can extract typed local `crsql_changes` batches through
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@ license.workspace = true
|
|||
[dependencies]
|
||||
blake3.workspace = true
|
||||
hex.workspace = true
|
||||
serde.workspace = true
|
||||
thiserror.workspace = true
|
||||
geth-codec = { path = "../geth-codec" }
|
||||
geth-types = { path = "../geth-types" }
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -1,14 +1,23 @@
|
|||
use geth_types::BlobHash;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub const CAS_TREE_OBJECT_VERSION: u16 = 1;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CasError {
|
||||
#[error("io error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("canonical codec error: {0}")]
|
||||
Codec(#[from] geth_codec::CodecError),
|
||||
#[error("invalid blob hash: {0}")]
|
||||
InvalidHash(String),
|
||||
#[error("blob not found: {0}")]
|
||||
NotFound(String),
|
||||
#[error("CAS tree root is not a directory: {0}")]
|
||||
TreeRootNotDirectory(String),
|
||||
#[error("CAS tree paths must be relative: {0}")]
|
||||
NonRelativeTreePath(String),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
|
|
@ -23,6 +32,34 @@ pub struct BlobInfo {
|
|||
pub path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct CasTreeObject {
|
||||
pub version: u16,
|
||||
pub entries: Vec<CasTreeEntry>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct CasTreeEntry {
|
||||
pub path: String,
|
||||
pub kind: CasTreeEntryKind,
|
||||
pub blob: Option<BlobHash>,
|
||||
pub size_bytes: u64,
|
||||
pub executable: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum CasTreeEntryKind {
|
||||
Directory,
|
||||
File,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CasTreeStored {
|
||||
pub tree: CasTreeObject,
|
||||
pub object: BlobInfo,
|
||||
}
|
||||
|
||||
impl LocalCas {
|
||||
#[must_use]
|
||||
pub fn new(root: impl Into<PathBuf>) -> Self {
|
||||
|
|
@ -123,6 +160,98 @@ impl LocalCas {
|
|||
.join(&hash[2..4])
|
||||
.join(hash))
|
||||
}
|
||||
|
||||
pub fn add_tree_path(&self, root: &Path) -> Result<CasTreeStored, CasError> {
|
||||
let tree = build_tree_object(self, root)?;
|
||||
let bytes = geth_codec::encode_canonical(&tree)?;
|
||||
let object = self.add_bytes(&bytes)?;
|
||||
Ok(CasTreeStored { tree, object })
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_tree_object(cas: &LocalCas, root: &Path) -> Result<CasTreeObject, CasError> {
|
||||
if !root.is_dir() {
|
||||
return Err(CasError::TreeRootNotDirectory(root.display().to_string()));
|
||||
}
|
||||
|
||||
let mut entries = Vec::new();
|
||||
collect_tree_entries(cas, root, root, &mut entries)?;
|
||||
entries.sort_by(|left, right| {
|
||||
left.path
|
||||
.cmp(&right.path)
|
||||
.then_with(|| kind_order(&left.kind).cmp(&kind_order(&right.kind)))
|
||||
});
|
||||
|
||||
Ok(CasTreeObject {
|
||||
version: CAS_TREE_OBJECT_VERSION,
|
||||
entries,
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_tree_entries(
|
||||
cas: &LocalCas,
|
||||
root: &Path,
|
||||
current: &Path,
|
||||
entries: &mut Vec<CasTreeEntry>,
|
||||
) -> Result<(), CasError> {
|
||||
let mut children = std::fs::read_dir(current)?.collect::<Result<Vec<_>, _>>()?;
|
||||
children.sort_by_key(|entry| entry.file_name());
|
||||
|
||||
for child in children {
|
||||
let path = child.path();
|
||||
let metadata = child.metadata()?;
|
||||
let relative = relative_tree_path(root, &path)?;
|
||||
if metadata.is_dir() {
|
||||
entries.push(CasTreeEntry {
|
||||
path: relative,
|
||||
kind: CasTreeEntryKind::Directory,
|
||||
blob: None,
|
||||
size_bytes: 0,
|
||||
executable: false,
|
||||
});
|
||||
collect_tree_entries(cas, root, &path, entries)?;
|
||||
} else if metadata.is_file() {
|
||||
let blob = cas.add_path(&path)?;
|
||||
entries.push(CasTreeEntry {
|
||||
path: relative,
|
||||
kind: CasTreeEntryKind::File,
|
||||
blob: Some(blob.hash),
|
||||
size_bytes: metadata.len(),
|
||||
executable: is_executable(&metadata),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn relative_tree_path(root: &Path, path: &Path) -> Result<String, CasError> {
|
||||
let relative = path
|
||||
.strip_prefix(root)
|
||||
.map_err(|_| CasError::NonRelativeTreePath(path.display().to_string()))?;
|
||||
Ok(relative
|
||||
.components()
|
||||
.map(|component| component.as_os_str().to_string_lossy())
|
||||
.collect::<Vec<_>>()
|
||||
.join("/"))
|
||||
}
|
||||
|
||||
fn kind_order(kind: &CasTreeEntryKind) -> u8 {
|
||||
match kind {
|
||||
CasTreeEntryKind::Directory => 0,
|
||||
CasTreeEntryKind::File => 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn is_executable(metadata: &std::fs::Metadata) -> bool {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
metadata.permissions().mode() & 0o111 != 0
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn is_executable(_metadata: &std::fs::Metadata) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
|
@ -174,4 +303,52 @@ mod tests {
|
|||
assert!(!cas.has(&info.hash).expect("has after remove"));
|
||||
assert!(!cas.remove(&info.hash).expect("remove missing"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cas_tree_objects_are_deterministic_and_store_file_blobs() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let cas = LocalCas::new(dir.path().join("cas"));
|
||||
let root = dir.path().join("root");
|
||||
std::fs::create_dir_all(root.join("sub")).expect("create subdir");
|
||||
std::fs::write(root.join("b.txt"), b"bravo").expect("write b");
|
||||
std::fs::write(root.join("sub").join("a.txt"), b"alpha").expect("write a");
|
||||
|
||||
let first = cas.add_tree_path(&root).expect("first tree");
|
||||
let second = cas.add_tree_path(&root).expect("second tree");
|
||||
|
||||
assert_eq!(first.object.hash, second.object.hash);
|
||||
assert_eq!(first.tree, second.tree);
|
||||
assert_eq!(
|
||||
first
|
||||
.tree
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| entry.path.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["b.txt", "sub", "sub/a.txt"]
|
||||
);
|
||||
assert!(
|
||||
first
|
||||
.tree
|
||||
.entries
|
||||
.iter()
|
||||
.find(|entry| entry.path == "b.txt")
|
||||
.and_then(|entry| entry.blob.clone())
|
||||
.is_some_and(|hash| cas.has(&hash).expect("has blob"))
|
||||
);
|
||||
assert!(cas.has(&first.object.hash).expect("has tree object"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cas_tree_rejects_file_root() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let cas = LocalCas::new(dir.path().join("cas"));
|
||||
let file = dir.path().join("file.txt");
|
||||
std::fs::write(&file, b"not a directory").expect("write");
|
||||
|
||||
assert!(matches!(
|
||||
cas.add_tree_path(&file),
|
||||
Err(CasError::TreeRootNotDirectory(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,8 +90,10 @@ Resource kinds:
|
|||
`geth-cas` is implemented locally first using BLAKE3 hashes and filesystem blob
|
||||
storage. Local pin/unpin metadata is tracked in SQLite and surfaced in
|
||||
`cas list`. `cas cleanup` removes unpinned local blobs while retaining pinned
|
||||
blobs. Iroh-blobs, providers, encrypted blobs, richer cache policies, manifests,
|
||||
and file sync trees are future work.
|
||||
blobs. The CAS crate can build deterministic tree objects that describe
|
||||
directories, files, executable bits, and file blob hashes; those tree objects
|
||||
are stored as CAS blobs. Iroh-blobs, providers, encrypted blobs, richer cache
|
||||
policies, file roots, and sync conflict handling are future work.
|
||||
|
||||
`geth-db` currently registers local SQLite paths as DB resources and reports
|
||||
local-only sync status plus a read-only SQLite schema summary/hash. It also
|
||||
|
|
|
|||
|
|
@ -344,11 +344,12 @@ Automerge documents.
|
|||
Goal: build higher-level local-first collaboration on CAS trees, resource auth,
|
||||
and future group key evolution.
|
||||
|
||||
- `[ ]` CAS tree objects.
|
||||
- `[x]` CAS tree objects.
|
||||
Acceptance criteria:
|
||||
- Tree objects describe directories, files, executable bits, and blob hashes.
|
||||
- Tree objects are content-addressed and stored in CAS.
|
||||
- Tests cover deterministic tree hashing.
|
||||
- `[x]` Tree objects describe directories, files, executable bits, and blob
|
||||
hashes.
|
||||
- `[x]` Tree objects are content-addressed and stored in CAS.
|
||||
- `[x]` Tests cover deterministic tree hashing.
|
||||
|
||||
- `[ ]` File roots.
|
||||
Acceptance criteria:
|
||||
|
|
|
|||
Loading…
Reference in a new issue