Add deterministic CAS tree objects
This commit is contained in:
parent
a44eef3126
commit
64e17ff957
7 changed files with 194 additions and 6 deletions
|
|
@ -118,6 +118,9 @@ Roadmap items should be actionable and checkable:
|
||||||
`keychain status`. SSH signature capture/verification is still roadmap work.
|
`keychain status`. SSH signature capture/verification is still roadmap work.
|
||||||
- Local CAS supports pin/unpin metadata, surfaced through `cas list`, and
|
- Local CAS supports pin/unpin metadata, surfaced through `cas list`, and
|
||||||
`cas cleanup` evicts unpinned blobs while retaining pinned blobs.
|
`cas cleanup` evicts unpinned blobs while retaining pinned blobs.
|
||||||
|
- The CAS crate can build deterministic tree objects for local file trees and
|
||||||
|
store those manifests as CAS blobs. File roots, scan workflows, and sync
|
||||||
|
conflict handling are still roadmap work.
|
||||||
- DB resources can be registered locally and report local-only status plus a
|
- DB resources can be registered locally and report local-only status plus a
|
||||||
read-only SQLite schema summary/hash and `crsql_changes` metadata when
|
read-only SQLite schema summary/hash and `crsql_changes` metadata when
|
||||||
present. The DB crate and daemon can extract typed read-only `crsql_changes`
|
present. The DB crate and daemon can extract typed read-only `crsql_changes`
|
||||||
|
|
|
||||||
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -1069,8 +1069,10 @@ name = "geth-cas"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"blake3",
|
"blake3",
|
||||||
|
"geth-codec",
|
||||||
"geth-types",
|
"geth-types",
|
||||||
"hex",
|
"hex",
|
||||||
|
"serde",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -87,6 +87,7 @@ The bootstrap implementation provides:
|
||||||
- `geth auth revoke <resource> <grant-id>`
|
- `geth auth revoke <resource> <grant-id>`
|
||||||
- local filesystem CAS commands: `add`, `get`, `hash`, `has`, `pin`, `unpin`,
|
- local filesystem CAS commands: `add`, `get`, `hash`, `has`, `pin`, `unpin`,
|
||||||
`cleanup`, `list`
|
`cleanup`, `list`
|
||||||
|
- local CAS tree objects describe file trees and are stored as CAS blobs
|
||||||
- local DB resource registration: `geth db add <name> <path>` and
|
- local DB resource registration: `geth db add <name> <path>` and
|
||||||
`geth db status <name>` with schema and `crsql_changes` metadata; the DB
|
`geth db status <name>` with schema and `crsql_changes` metadata; the DB
|
||||||
crate and daemon can extract typed local `crsql_changes` batches through
|
crate and daemon can extract typed local `crsql_changes` batches through
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,9 @@ license.workspace = true
|
||||||
[dependencies]
|
[dependencies]
|
||||||
blake3.workspace = true
|
blake3.workspace = true
|
||||||
hex.workspace = true
|
hex.workspace = true
|
||||||
|
serde.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
|
geth-codec = { path = "../geth-codec" }
|
||||||
geth-types = { path = "../geth-types" }
|
geth-types = { path = "../geth-types" }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,23 @@
|
||||||
use geth_types::BlobHash;
|
use geth_types::BlobHash;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
pub const CAS_TREE_OBJECT_VERSION: u16 = 1;
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum CasError {
|
pub enum CasError {
|
||||||
#[error("io error: {0}")]
|
#[error("io error: {0}")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
|
#[error("canonical codec error: {0}")]
|
||||||
|
Codec(#[from] geth_codec::CodecError),
|
||||||
#[error("invalid blob hash: {0}")]
|
#[error("invalid blob hash: {0}")]
|
||||||
InvalidHash(String),
|
InvalidHash(String),
|
||||||
#[error("blob not found: {0}")]
|
#[error("blob not found: {0}")]
|
||||||
NotFound(String),
|
NotFound(String),
|
||||||
|
#[error("CAS tree root is not a directory: {0}")]
|
||||||
|
TreeRootNotDirectory(String),
|
||||||
|
#[error("CAS tree paths must be relative: {0}")]
|
||||||
|
NonRelativeTreePath(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
|
|
@ -23,6 +32,34 @@ pub struct BlobInfo {
|
||||||
pub path: PathBuf,
|
pub path: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct CasTreeObject {
|
||||||
|
pub version: u16,
|
||||||
|
pub entries: Vec<CasTreeEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct CasTreeEntry {
|
||||||
|
pub path: String,
|
||||||
|
pub kind: CasTreeEntryKind,
|
||||||
|
pub blob: Option<BlobHash>,
|
||||||
|
pub size_bytes: u64,
|
||||||
|
pub executable: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub enum CasTreeEntryKind {
|
||||||
|
Directory,
|
||||||
|
File,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct CasTreeStored {
|
||||||
|
pub tree: CasTreeObject,
|
||||||
|
pub object: BlobInfo,
|
||||||
|
}
|
||||||
|
|
||||||
impl LocalCas {
|
impl LocalCas {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn new(root: impl Into<PathBuf>) -> Self {
|
pub fn new(root: impl Into<PathBuf>) -> Self {
|
||||||
|
|
@ -123,6 +160,98 @@ impl LocalCas {
|
||||||
.join(&hash[2..4])
|
.join(&hash[2..4])
|
||||||
.join(hash))
|
.join(hash))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn add_tree_path(&self, root: &Path) -> Result<CasTreeStored, CasError> {
|
||||||
|
let tree = build_tree_object(self, root)?;
|
||||||
|
let bytes = geth_codec::encode_canonical(&tree)?;
|
||||||
|
let object = self.add_bytes(&bytes)?;
|
||||||
|
Ok(CasTreeStored { tree, object })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_tree_object(cas: &LocalCas, root: &Path) -> Result<CasTreeObject, CasError> {
|
||||||
|
if !root.is_dir() {
|
||||||
|
return Err(CasError::TreeRootNotDirectory(root.display().to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
collect_tree_entries(cas, root, root, &mut entries)?;
|
||||||
|
entries.sort_by(|left, right| {
|
||||||
|
left.path
|
||||||
|
.cmp(&right.path)
|
||||||
|
.then_with(|| kind_order(&left.kind).cmp(&kind_order(&right.kind)))
|
||||||
|
});
|
||||||
|
|
||||||
|
Ok(CasTreeObject {
|
||||||
|
version: CAS_TREE_OBJECT_VERSION,
|
||||||
|
entries,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_tree_entries(
|
||||||
|
cas: &LocalCas,
|
||||||
|
root: &Path,
|
||||||
|
current: &Path,
|
||||||
|
entries: &mut Vec<CasTreeEntry>,
|
||||||
|
) -> Result<(), CasError> {
|
||||||
|
let mut children = std::fs::read_dir(current)?.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
children.sort_by_key(|entry| entry.file_name());
|
||||||
|
|
||||||
|
for child in children {
|
||||||
|
let path = child.path();
|
||||||
|
let metadata = child.metadata()?;
|
||||||
|
let relative = relative_tree_path(root, &path)?;
|
||||||
|
if metadata.is_dir() {
|
||||||
|
entries.push(CasTreeEntry {
|
||||||
|
path: relative,
|
||||||
|
kind: CasTreeEntryKind::Directory,
|
||||||
|
blob: None,
|
||||||
|
size_bytes: 0,
|
||||||
|
executable: false,
|
||||||
|
});
|
||||||
|
collect_tree_entries(cas, root, &path, entries)?;
|
||||||
|
} else if metadata.is_file() {
|
||||||
|
let blob = cas.add_path(&path)?;
|
||||||
|
entries.push(CasTreeEntry {
|
||||||
|
path: relative,
|
||||||
|
kind: CasTreeEntryKind::File,
|
||||||
|
blob: Some(blob.hash),
|
||||||
|
size_bytes: metadata.len(),
|
||||||
|
executable: is_executable(&metadata),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn relative_tree_path(root: &Path, path: &Path) -> Result<String, CasError> {
|
||||||
|
let relative = path
|
||||||
|
.strip_prefix(root)
|
||||||
|
.map_err(|_| CasError::NonRelativeTreePath(path.display().to_string()))?;
|
||||||
|
Ok(relative
|
||||||
|
.components()
|
||||||
|
.map(|component| component.as_os_str().to_string_lossy())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("/"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn kind_order(kind: &CasTreeEntryKind) -> u8 {
|
||||||
|
match kind {
|
||||||
|
CasTreeEntryKind::Directory => 0,
|
||||||
|
CasTreeEntryKind::File => 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn is_executable(metadata: &std::fs::Metadata) -> bool {
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
metadata.permissions().mode() & 0o111 != 0
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
fn is_executable(_metadata: &std::fs::Metadata) -> bool {
|
||||||
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
|
|
@ -174,4 +303,52 @@ mod tests {
|
||||||
assert!(!cas.has(&info.hash).expect("has after remove"));
|
assert!(!cas.has(&info.hash).expect("has after remove"));
|
||||||
assert!(!cas.remove(&info.hash).expect("remove missing"));
|
assert!(!cas.remove(&info.hash).expect("remove missing"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cas_tree_objects_are_deterministic_and_store_file_blobs() {
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let cas = LocalCas::new(dir.path().join("cas"));
|
||||||
|
let root = dir.path().join("root");
|
||||||
|
std::fs::create_dir_all(root.join("sub")).expect("create subdir");
|
||||||
|
std::fs::write(root.join("b.txt"), b"bravo").expect("write b");
|
||||||
|
std::fs::write(root.join("sub").join("a.txt"), b"alpha").expect("write a");
|
||||||
|
|
||||||
|
let first = cas.add_tree_path(&root).expect("first tree");
|
||||||
|
let second = cas.add_tree_path(&root).expect("second tree");
|
||||||
|
|
||||||
|
assert_eq!(first.object.hash, second.object.hash);
|
||||||
|
assert_eq!(first.tree, second.tree);
|
||||||
|
assert_eq!(
|
||||||
|
first
|
||||||
|
.tree
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.map(|entry| entry.path.as_str())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["b.txt", "sub", "sub/a.txt"]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
first
|
||||||
|
.tree
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.find(|entry| entry.path == "b.txt")
|
||||||
|
.and_then(|entry| entry.blob.clone())
|
||||||
|
.is_some_and(|hash| cas.has(&hash).expect("has blob"))
|
||||||
|
);
|
||||||
|
assert!(cas.has(&first.object.hash).expect("has tree object"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cas_tree_rejects_file_root() {
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let cas = LocalCas::new(dir.path().join("cas"));
|
||||||
|
let file = dir.path().join("file.txt");
|
||||||
|
std::fs::write(&file, b"not a directory").expect("write");
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
cas.add_tree_path(&file),
|
||||||
|
Err(CasError::TreeRootNotDirectory(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -90,8 +90,10 @@ Resource kinds:
|
||||||
`geth-cas` is implemented locally first using BLAKE3 hashes and filesystem blob
|
`geth-cas` is implemented locally first using BLAKE3 hashes and filesystem blob
|
||||||
storage. Local pin/unpin metadata is tracked in SQLite and surfaced in
|
storage. Local pin/unpin metadata is tracked in SQLite and surfaced in
|
||||||
`cas list`. `cas cleanup` removes unpinned local blobs while retaining pinned
|
`cas list`. `cas cleanup` removes unpinned local blobs while retaining pinned
|
||||||
blobs. Iroh-blobs, providers, encrypted blobs, richer cache policies, manifests,
|
blobs. The CAS crate can build deterministic tree objects that describe
|
||||||
and file sync trees are future work.
|
directories, files, executable bits, and file blob hashes; those tree objects
|
||||||
|
are stored as CAS blobs. Iroh-blobs, providers, encrypted blobs, richer cache
|
||||||
|
policies, file roots, and sync conflict handling are future work.
|
||||||
|
|
||||||
`geth-db` currently registers local SQLite paths as DB resources and reports
|
`geth-db` currently registers local SQLite paths as DB resources and reports
|
||||||
local-only sync status plus a read-only SQLite schema summary/hash. It also
|
local-only sync status plus a read-only SQLite schema summary/hash. It also
|
||||||
|
|
|
||||||
|
|
@ -344,11 +344,12 @@ Automerge documents.
|
||||||
Goal: build higher-level local-first collaboration on CAS trees, resource auth,
|
Goal: build higher-level local-first collaboration on CAS trees, resource auth,
|
||||||
and future group key evolution.
|
and future group key evolution.
|
||||||
|
|
||||||
- `[ ]` CAS tree objects.
|
- `[x]` CAS tree objects.
|
||||||
Acceptance criteria:
|
Acceptance criteria:
|
||||||
- Tree objects describe directories, files, executable bits, and blob hashes.
|
- `[x]` Tree objects describe directories, files, executable bits, and blob
|
||||||
- Tree objects are content-addressed and stored in CAS.
|
hashes.
|
||||||
- Tests cover deterministic tree hashing.
|
- `[x]` Tree objects are content-addressed and stored in CAS.
|
||||||
|
- `[x]` Tests cover deterministic tree hashing.
|
||||||
|
|
||||||
- `[ ]` File roots.
|
- `[ ]` File roots.
|
||||||
Acceptance criteria:
|
Acceptance criteria:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue