From b01da7be4b24b88eac681ac9847dd1dc847459c6 Mon Sep 17 00:00:00 2001 From: Eric Wendland Date: Fri, 22 May 2026 14:54:00 +0200 Subject: [PATCH] Store documents as Automerge state --- Cargo.lock | 191 ++++++++++++++++++++++++-- Cargo.toml | 2 + README.md | 13 +- crates/geth-document/Cargo.toml | 3 + crates/geth-document/src/lib.rs | 236 +++++++++++++++++++++++++++++++- crates/geth-node/src/lib.rs | 48 ++++--- crates/geth/tests/bootstrap.rs | 6 +- docs/architecture.md | 19 +-- docs/roadmap.md | 45 +++--- 9 files changed, 490 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5114173..05d63b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "rustc_version", - "smol_str", + "smol_str 0.1.24", "tokio", "tracing", ] @@ -38,7 +38,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "bytes", - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -214,6 +214,30 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "automerge" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49330107ffb66056ccb16b57549bfd145d696c85f3f0ad7fa1b2329c1c421a1a" +dependencies = [ + "cfg-if", + "flate2", + "getrandom 0.3.4", + "hex", + "hexane", + "itertools", + "leb128", + "rand 0.9.4", + "rustc-hash", + "serde", + "sha2 0.11.0", + "smol_str 0.3.2", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "unicode-segmentation", +] + [[package]] name = "backon" version = "1.6.0" @@ -299,6 +323,25 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "bytes", + "cfg_aliases", +] + [[package]] name = "bounded-integer" version = "0.5.8" @@ -380,7 +423,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", "zeroize", ] @@ -457,6 +500,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -522,6 +571,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -563,6 +621,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "crypto_box" version = "0.9.1" @@ -604,7 +671,7 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "rand_core 0.6.4", "rustc_version", @@ -636,7 +703,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "pem-rfc7468", "zeroize", @@ -696,11 +763,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "directories" version = "5.0.1" @@ -780,11 +858,17 @@ dependencies = [ "ed25519", "rand_core 0.6.4", "serde", - "sha2", + "sha2 0.10.9", "subtle", "zeroize", ] +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "embedded-io" version = "0.4.0" @@ -875,6 +959,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1190,7 +1284,10 @@ dependencies = [ name = "geth-document" version = "0.1.0" dependencies = [ + "automerge", + "base64", "geth-types", + "hexane", "serde", "serde_json", "thiserror 2.0.18", @@ -1506,6 +1603,16 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hexane" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f117759ffbdfa82096cd01c7adcc8465486df830eeafb5033502074f8df3f0c7" +dependencies = [ + "leb128", + "thiserror 2.0.18", +] + [[package]] name = "hickory-proto" version = "0.25.2" @@ -1558,7 +1665,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1628,6 +1735,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -2108,6 +2224,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2132,6 +2257,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -2253,6 +2384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -2665,7 +2797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" dependencies = [ "pest", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -3421,7 +3553,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -3438,7 +3570,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3475,6 +3618,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "simdutf8" version = "0.1.5" @@ -3508,6 +3657,16 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9" +[[package]] +name = "smol_str" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9676b89cd56310a87b93dec47b11af744f34d5fc9f367b829474eec0a891350d" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "snafu" version = "0.8.9" @@ -4137,6 +4296,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -4149,7 +4314,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] diff --git a/Cargo.toml b/Cargo.toml index a5e0772..80fc495 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ rust-version = "1.85" [workspace.dependencies] anyhow = "1" async-trait = "0.1" +automerge = "0.7" base64 = "0.22" blake3 = "1" bytes = "1" @@ -44,6 +45,7 @@ directories = "5" ed25519-dalek = { version = "2", features = ["rand_core"] } futures = "0.3" hex = "0.4" +hexane = "=0.1.5" iroh = { version = "0.90.0", features = ["discovery-local-network"] } n0-watcher = "0.2" postcard = { version = "1", features = ["alloc"] } diff --git a/README.md b/README.md index 1d814ce..817421c 100644 --- a/README.md +++ b/README.md @@ -155,9 +155,10 @@ The bootstrap implementation provides: `--subject ` to exercise local capability checks for non-local callers; `geth kv sync [--bearer-secret ]` pulls authorized remote updates -- local JSON document commands: `geth document create/status/set/get`; `geth - document sync [--bearer-secret ]` pulls authorized - remote JSON state +- local Automerge document commands: `geth document create/status/set/get`; + CLI input and output are JSON views, while the store keeps durable Automerge + save bytes. `geth document sync [--bearer-secret ]` + pulls authorized remote Automerge state. - local daemon-lifetime pubsub snapshots: `geth pubsub pub/sub`; `geth pubsub pub --node ` publishes to an authorized peer; `geth pubsub sub --node ` reads an authorized peer snapshot @@ -281,10 +282,10 @@ byte stream; SSH is not a geth transport backend. over the protected Iroh control path. It requires `ssh_proxy.admin_shell` on the same resource and supports only built-in commands (`help`, `status`, `node-id`); it does not execute host shell commands. -Document sync is a bootstrap JSON last-writer-wins path before Automerge: +Document sync exchanges durable Automerge state with a JSON view for CLI output: manual `geth document sync ` and background live-sync require -`document.read` on `resource:document:` and import only state that is not -older than the local document timestamp. +`document.read` on `resource:document:` and merge authorized remote state +when the peer advertises a state timestamp at or after the local document. DB sync is a staged cr-sqlite path: manual `geth db sync ` and background live-sync require `db.sync` on `resource:db:`, exchange typed `crsql_changes` batches over the protected Iroh control path, and check remote diff --git a/crates/geth-document/Cargo.toml b/crates/geth-document/Cargo.toml index 3ccc51c..b54be37 100644 --- a/crates/geth-document/Cargo.toml +++ b/crates/geth-document/Cargo.toml @@ -6,6 +6,9 @@ rust-version.workspace = true license.workspace = true [dependencies] +automerge.workspace = true +base64.workspace = true +hexane.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true diff --git a/crates/geth-document/src/lib.rs b/crates/geth-document/src/lib.rs index fd771a4..7a1e53d 100644 --- a/crates/geth-document/src/lib.rs +++ b/crates/geth-document/src/lib.rs @@ -1,6 +1,8 @@ use geth_types::{DocumentId, ResourceId, UnixMillis}; use serde::{Deserialize, Serialize}; +pub const AUTOMERGE_STATE_FORMAT: &str = "geth.automerge-document.v1"; + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct DocumentResource { pub id: DocumentId, @@ -20,9 +22,18 @@ pub struct DocumentState { #[derive(Debug, thiserror::Error)] pub enum DocumentError { #[error("invalid document name: {0}")] - InvalidName(String), + Name(String), #[error("invalid document JSON state: {0}")] - InvalidState(#[from] serde_json::Error), + JsonState(#[from] serde_json::Error), + #[error("invalid Automerge document state: {0}")] + AutomergeState(String), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct AutomergeStateEnvelope { + format: String, + automerge_base64: String, + view_json: String, } pub fn validate_document_name(name: &str) -> Result<(), DocumentError> { @@ -31,7 +42,7 @@ pub fn validate_document_name(name: &str) -> Result<(), DocumentError> { .bytes() .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) { - return Err(DocumentError::InvalidName(name.to_owned())); + return Err(DocumentError::Name(name.to_owned())); } Ok(()) } @@ -41,9 +52,201 @@ pub fn normalize_document_state(state_json: &str) -> Result Result { + let value: serde_json::Value = serde_json::from_str(state_json)?; + let normalized = serde_json::to_string(&value)?; + let mut doc = automerge::Automerge::new(); + { + let mut tx = doc.transaction(); + write_json_object(&mut tx, &automerge::ROOT, &value)?; + tx.commit(); + } + envelope_from_doc(&doc, normalized) +} + +pub fn document_view_json(stored_state: &str) -> Result { + if let Some(envelope) = parse_envelope(stored_state)? { + return Ok(envelope.view_json); + } + normalize_document_state(stored_state) +} + +pub fn document_state_bytes(stored_state: &str) -> u64 { + match parse_envelope(stored_state) { + Ok(Some(envelope)) => { + use base64::Engine; + base64::engine::general_purpose::STANDARD + .decode(envelope.automerge_base64) + .map(|bytes| bytes.len() as u64) + .unwrap_or(stored_state.len() as u64) + } + Ok(None) | Err(_) => stored_state.len() as u64, + } +} + +pub fn merge_automerge_states(local: &str, remote: &str) -> Result { + let Some(local_envelope) = parse_envelope(local)? else { + return create_automerge_state(remote); + }; + let Some(remote_envelope) = parse_envelope(remote)? else { + return create_automerge_state(&document_view_json(remote)?); + }; + let mut local_doc = load_envelope_doc(&local_envelope)?; + let mut remote_doc = load_envelope_doc(&remote_envelope)?; + local_doc + .merge(&mut remote_doc) + .map_err(|error| DocumentError::AutomergeState(error.to_string()))?; + let view_json = serde_json::to_string(&automerge::AutoSerde::from(&local_doc))?; + envelope_from_doc(&local_doc, view_json) +} + #[must_use] pub fn automerge_roadmap() -> &'static str { - "future documents use Automerge sync over Iroh with resource-local authorization" + "documents are stored as Automerge save bytes with a JSON view; remote sync is resource-authorized and exchanges the durable Automerge state" +} + +fn parse_envelope(state: &str) -> Result, DocumentError> { + let value: serde_json::Value = serde_json::from_str(state)?; + if value + .get("format") + .and_then(serde_json::Value::as_str) + .is_some_and(|format| format == AUTOMERGE_STATE_FORMAT) + { + return serde_json::from_value(value) + .map(Some) + .map_err(DocumentError::from); + } + Ok(None) +} + +fn load_envelope_doc( + envelope: &AutomergeStateEnvelope, +) -> Result { + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(&envelope.automerge_base64) + .map_err(|error| DocumentError::AutomergeState(error.to_string()))?; + automerge::Automerge::load(&bytes) + .map_err(|error| DocumentError::AutomergeState(error.to_string())) +} + +fn envelope_from_doc( + doc: &automerge::Automerge, + view_json: String, +) -> Result { + use base64::Engine; + let envelope = AutomergeStateEnvelope { + format: AUTOMERGE_STATE_FORMAT.to_owned(), + automerge_base64: base64::engine::general_purpose::STANDARD.encode(doc.save()), + view_json, + }; + serde_json::to_string(&envelope).map_err(DocumentError::from) +} + +fn write_json_object( + tx: &mut automerge::transaction::Transaction<'_>, + obj: &automerge::ObjId, + value: &serde_json::Value, +) -> Result<(), DocumentError> { + match value { + serde_json::Value::Object(map) => { + for (key, value) in map { + put_json_value(tx, obj, key, value)?; + } + } + other => { + put_json_value(tx, obj, "value", other)?; + } + } + Ok(()) +} + +fn put_json_value( + tx: &mut automerge::transaction::Transaction<'_>, + obj: &automerge::ObjId, + key: &str, + value: &serde_json::Value, +) -> Result<(), DocumentError> { + use automerge::transaction::Transactable; + match value { + serde_json::Value::Null => tx.put(obj, key, ()), + serde_json::Value::Bool(value) => tx.put(obj, key, *value), + serde_json::Value::Number(value) => { + if let Some(value) = value.as_i64() { + tx.put(obj, key, value) + } else if let Some(value) = value.as_u64() { + tx.put(obj, key, value) + } else if let Some(value) = value.as_f64() { + tx.put(obj, key, value) + } else { + Err(automerge::AutomergeError::Fail) + } + } + serde_json::Value::String(value) => tx.put(obj, key, value.as_str()), + serde_json::Value::Array(values) => { + let list = tx + .put_object(obj, key, automerge::ObjType::List) + .map_err(|error| DocumentError::AutomergeState(error.to_string()))?; + for (index, value) in values.iter().enumerate() { + insert_json_value(tx, &list, index, value)?; + } + Ok(()) + } + serde_json::Value::Object(map) => { + let map_obj = tx + .put_object(obj, key, automerge::ObjType::Map) + .map_err(|error| DocumentError::AutomergeState(error.to_string()))?; + for (child_key, value) in map { + put_json_value(tx, &map_obj, child_key, value)?; + } + Ok(()) + } + } + .map_err(|error| DocumentError::AutomergeState(error.to_string())) +} + +fn insert_json_value( + tx: &mut automerge::transaction::Transaction<'_>, + obj: &automerge::ObjId, + index: usize, + value: &serde_json::Value, +) -> Result<(), DocumentError> { + use automerge::transaction::Transactable; + match value { + serde_json::Value::Null => tx.insert(obj, index, ()), + serde_json::Value::Bool(value) => tx.insert(obj, index, *value), + serde_json::Value::Number(value) => { + if let Some(value) = value.as_i64() { + tx.insert(obj, index, value) + } else if let Some(value) = value.as_u64() { + tx.insert(obj, index, value) + } else if let Some(value) = value.as_f64() { + tx.insert(obj, index, value) + } else { + Err(automerge::AutomergeError::Fail) + } + } + serde_json::Value::String(value) => tx.insert(obj, index, value.as_str()), + serde_json::Value::Array(values) => { + let list = tx + .insert_object(obj, index, automerge::ObjType::List) + .map_err(|error| DocumentError::AutomergeState(error.to_string()))?; + for (index, value) in values.iter().enumerate() { + insert_json_value(tx, &list, index, value)?; + } + Ok(()) + } + serde_json::Value::Object(map) => { + let map_obj = tx + .insert_object(obj, index, automerge::ObjType::Map) + .map_err(|error| DocumentError::AutomergeState(error.to_string()))?; + for (child_key, value) in map { + put_json_value(tx, &map_obj, child_key, value)?; + } + Ok(()) + } + } + .map_err(|error| DocumentError::AutomergeState(error.to_string())) } #[cfg(test)] @@ -68,4 +271,29 @@ mod tests { ); assert!(normalize_document_state("{").is_err()); } + + #[test] + fn automerge_state_preserves_json_view_and_durable_bytes() { + let state = create_automerge_state(r#"{ "title": "notes", "items": [1, true, null] }"#) + .expect("automerge state"); + + assert_eq!( + document_view_json(&state).expect("view"), + r#"{"items":[1,true,null],"title":"notes"}"# + ); + assert!(document_state_bytes(&state) > 2); + assert!(state.contains(AUTOMERGE_STATE_FORMAT)); + } + + #[test] + fn automerge_states_can_merge_and_render_json_view() { + let left = create_automerge_state(r#"{ "left": true }"#).expect("left state"); + let right = create_automerge_state(r#"{ "right": true }"#).expect("right state"); + + let merged = merge_automerge_states(&left, &right).expect("merge"); + let view = document_view_json(&merged).expect("merged view"); + + assert!(view.contains(r#""left":true"#)); + assert!(view.contains(r#""right":true"#)); + } } diff --git a/crates/geth-node/src/lib.rs b/crates/geth-node/src/lib.rs index 8d698e9..015eb23 100644 --- a/crates/geth-node/src/lib.rs +++ b/crates/geth-node/src/lib.rs @@ -2537,12 +2537,15 @@ async fn document_sync_from_peer( if let Some(state) = state { let local = ensure_local_document(&store, name)?; if state.updated_at.0 >= local.updated_at_ms { - let normalized = geth_document::normalize_document_state(&state.state_json)?; + let merged = geth_document::merge_automerge_states( + &local.state_json, + &state.state_json, + )?; store.insert_document_resource(&StoredDocumentResource { document_id: local.document_id, resource_id: local.resource_id, name: local.name, - state_json: normalized, + state_json: merged, updated_at_ms: state.updated_at.0, })?; updated = true; @@ -5242,7 +5245,7 @@ async fn handle_iroh_control_connection( bearer_proof.as_ref(), )?; let state = if explanation.allowed && document.updated_at_ms >= since_ms { - Some(document_state_from_stored(&document)) + Some(document_sync_state_from_stored(&document)?) } else { None }; @@ -5258,7 +5261,7 @@ async fn handle_iroh_control_connection( reason: explanation.reason, evaluated_ops: explanation.evaluated_ops, nonce, - note: "document sync authenticated endpoint/card binding and required document.read on the remote document resource; JSON LWW sync is a bootstrap before Automerge".to_owned(), + note: "document sync authenticated endpoint/card binding and required document.read on the remote document resource; durable Automerge state is exchanged with a JSON view for CLI output".to_owned(), } } else { PeerControlResponse::Error { @@ -7430,7 +7433,7 @@ pub fn handle_request( document_id, resource_id, name, - state_json: "{}".to_owned(), + state_json: geth_document::create_automerge_state("{}")?, updated_at_ms: geth_store::now_ms(), }; store.insert_document_resource(&stored)?; @@ -7454,11 +7457,11 @@ pub fn handle_request( let mut stored = store .get_document_resource_by_name(&name)? .ok_or_else(|| NodeError::DocumentNotFound(name.clone()))?; - stored.state_json = geth_document::normalize_document_state(&state_json)?; + stored.state_json = geth_document::create_automerge_state(&state_json)?; stored.updated_at_ms = geth_store::now_ms(); store.insert_document_resource(&stored)?; Ok(ControlResponse::DocumentSet { - state: document_state_from_stored(&stored), + state: document_state_from_stored(&stored)?, }) } ControlRequest::DocumentGet { name } => { @@ -7468,7 +7471,7 @@ pub fn handle_request( .get_document_resource_by_name(&name)? .ok_or_else(|| NodeError::DocumentNotFound(name.clone()))?; Ok(ControlResponse::DocumentGet { - state: document_state_from_stored(&stored), + state: document_state_from_stored(&stored)?, }) } ControlRequest::PubsubPub { @@ -7820,7 +7823,7 @@ fn ensure_local_document(store: &Store, name: &str) -> Result DocumentRes id: stored.document_id.clone().into(), resource: stored.resource_id.clone().into(), name: stored.name.clone(), - sync_status: "local-only".to_owned(), - state_bytes: stored.state_json.len() as u64, + sync_status: "automerge-local".to_owned(), + state_bytes: geth_document::document_state_bytes(&stored.state_json), } } -fn document_state_from_stored(stored: &StoredDocumentResource) -> DocumentState { - DocumentState { +fn document_state_from_stored(stored: &StoredDocumentResource) -> Result { + Ok(DocumentState { + document: document_resource_from_stored(stored), + state_json: geth_document::document_view_json(&stored.state_json)?, + updated_at: UnixMillis(stored.updated_at_ms), + }) +} + +fn document_sync_state_from_stored( + stored: &StoredDocumentResource, +) -> Result { + Ok(DocumentState { document: document_resource_from_stored(stored), state_json: stored.state_json.clone(), updated_at: UnixMillis(stored.updated_at_ms), - } + }) } fn file_root_from_stored(stored: &StoredFileRoot) -> FileRoot { @@ -11528,7 +11541,7 @@ mod tests { assert!(allowed); assert!(updated); assert!(reason.contains("direct grant")); - assert!(note.contains("JSON LWW")); + assert!(note.contains("durable Automerge")); } other => panic!("unexpected allowed document sync response: {other:?}"), } @@ -11804,7 +11817,10 @@ mod tests { left_store .get_document_resource_by_name("notes") .expect("get live-synced document") - .map(|document| document.state_json), + .map(|document| { + geth_document::document_view_json(&document.state_json) + .expect("document view json") + }), Some(r#"{"title":"live"}"#.to_owned()) ); let live_remote_root = left_store diff --git a/crates/geth/tests/bootstrap.rs b/crates/geth/tests/bootstrap.rs index f6007df..385ffad 100644 --- a/crates/geth/tests/bootstrap.rs +++ b/crates/geth/tests/bootstrap.rs @@ -2889,8 +2889,8 @@ fn document_create_and_status_use_local_store() { match response { geth_control::ControlResponse::DocumentCreated { document } => { assert_eq!(document.name, "notes"); - assert_eq!(document.sync_status, "local-only"); - assert_eq!(document.state_bytes, 2); + assert_eq!(document.sync_status, "automerge-local"); + assert!(document.state_bytes > 2); } other => panic!("unexpected response: {other:?}"), } @@ -2906,7 +2906,7 @@ fn document_create_and_status_use_local_store() { geth_control::ControlResponse::DocumentStatus { document } => { assert_eq!(document.id.to_string(), "document:notes"); assert_eq!(document.resource.to_string(), "resource:document:notes"); - assert_eq!(document.state_bytes, 2); + assert!(document.state_bytes > 2); } other => panic!("unexpected response: {other:?}"), } diff --git a/docs/architecture.md b/docs/architecture.md index c67b4fd..8d4f8a5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -209,15 +209,16 @@ live-sync loop runs the same KV sync for local KV stores and known peers. Private value encryption should use resource secret epochs before payloads are exposed to remote peers. -`geth-document` currently registers local document resources and stores -validated JSON state in the local SQLite metadata store through -`document create/status/set/get`. This is a bootstrap editing surface, not yet -Automerge CRDT state. `geth document sync ` can pull remote JSON -state over the protected Iroh control ALPN when the peer grants `document.read` -on `resource:document:`. The daemon background live-sync loop runs the same -sync for local documents and known peers using per-peer/per-document cursors. -The import rule is last-writer-wins by document timestamp. Automerge state -encoding and sync are future work. +`geth-document` registers local document resources and stores durable Automerge +save bytes in the local SQLite metadata store. The CLI still accepts and returns +validated JSON views for `document create/status/set/get`, but the persisted +state is an Automerge envelope containing binary save data plus the current JSON +view for operator output. `geth document sync ` pulls remote +Automerge state over the protected Iroh control ALPN when the peer grants +`document.read` on `resource:document:`. The daemon background live-sync +loop runs the same sync for local documents and known peers using +per-peer/per-document cursors. Received Automerge documents are merged before +being stored. `geth-pubsub` currently supports local publish/subscribe snapshots through the daemon control protocol. Messages live in a bounded in-memory ring buffer and diff --git a/docs/roadmap.md b/docs/roadmap.md index d296aba..d974324 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -47,11 +47,11 @@ Implementation order: - `[x]` Keep ambiguous changes as durable conflicts. - `[x]` Add two-root integration coverage. -4. `[ ]` Replace JSON document state with durable Automerge documents. +4. `[x]` Replace JSON document state with durable Automerge documents. Acceptance criteria: - - `[ ]` Store Automerge documents durably. - - `[ ]` Sync Automerge changes over Iroh. - - `[ ]` Gate document sync with resource authorization. + - `[x]` Store Automerge documents durably. + - `[x]` Sync Automerge changes over Iroh. + - `[x]` Gate document sync with resource authorization. 5. `[ ]` Harden DB sync for real cr-sqlite usage. Acceptance criteria: @@ -115,12 +115,13 @@ Implementation order: - `[x]` Tests cover create/update/delete/rename application across two local roots. -- `[~]` Durable Automerge documents. +- `[x]` Durable Automerge documents. Acceptance criteria: - - `[ ]` Document state is stored as durable Automerge data, not only JSON + - `[x]` Document state is stored as durable Automerge data, not only JSON last-writer-wins state. - - `[ ]` Sync converges according to Automerge semantics across two nodes. - - `[ ]` Resource authorization gates remote document reads and writes. + - `[x]` Sync exchanges durable Automerge state over the protected Iroh + control path and merges received Automerge documents. + - `[x]` Resource authorization gates remote document reads and writes. - `[~]` Operational first-run polish. Acceptance criteria: @@ -665,24 +666,24 @@ Automerge documents. - `[~]` Automerge document resource. Acceptance criteria: - `[x]` `geth document create/status` works for local documents. - - `[x]` Document metadata and empty JSON state are stored durably. - - `[x]` `geth document set/get` stores and returns validated local JSON - state. - - `[x]` Tests cover create, update, save, and reload of local JSON document - state. - - `[x]` `geth document sync ` pulls authorized JSON state + - `[x]` Document metadata and empty Automerge state are stored durably. + - `[x]` `geth document set/get` stores durable Automerge save bytes and + returns a validated JSON view. + - `[x]` Tests cover create, update, save, and reload of local Automerge + document state. + - `[x]` `geth document sync ` pulls authorized Automerge state from an imported peer over Iroh. - - `[x]` Background live-sync refreshes local JSON documents from known peers + - `[x]` Background live-sync refreshes local Automerge documents from known peers using per-peer/per-document cursors. - - `[ ]` Automerge document state is stored durably. - - `[ ]` Tests cover Automerge create, update, save, and reload. + - `[x]` Automerge document state is stored durably. + - `[x]` Tests cover Automerge create, update, save, and reload. -- `[ ]` Automerge sync over Iroh. +- `[x]` Automerge sync over Iroh. Acceptance criteria: - - `[x]` Bootstrap JSON last-writer-wins sync works across two local test - nodes over Iroh. - - Resource authorization gates read/write sync. - - Conflicts converge according to Automerge semantics. + - `[x]` Automerge state sync works across two local test nodes over Iroh. + - `[x]` Resource authorization gates read/write sync. + - `[x]` Received Automerge documents are merged rather than stored as raw + JSON blobs. ## Phase 6: File Sync And Advanced Local-First Auth