Store documents as Automerge state
This commit is contained in:
parent
606641dd4a
commit
b01da7be4b
9 changed files with 490 additions and 73 deletions
191
Cargo.lock
generated
191
Cargo.lock
generated
|
|
@ -11,7 +11,7 @@ dependencies = [
|
|||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"rustc_version",
|
||||
"smol_str",
|
||||
"smol_str 0.1.24",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
|
@ -38,7 +38,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"crypto-common",
|
||||
"crypto-common 0.1.7",
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
|
|
@ -214,6 +214,30 @@ version = "1.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
|
||||
[[package]]
|
||||
name = "automerge"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49330107ffb66056ccb16b57549bfd145d696c85f3f0ad7fa1b2329c1c421a1a"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"flate2",
|
||||
"getrandom 0.3.4",
|
||||
"hex",
|
||||
"hexane",
|
||||
"itertools",
|
||||
"leb128",
|
||||
"rand 0.9.4",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
"sha2 0.11.0",
|
||||
"smol_str 0.3.2",
|
||||
"thiserror 2.0.18",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backon"
|
||||
version = "1.6.0"
|
||||
|
|
@ -299,6 +323,25 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be"
|
||||
dependencies = [
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"cfg_aliases",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bounded-integer"
|
||||
version = "0.5.8"
|
||||
|
|
@ -380,7 +423,7 @@ version = "0.4.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"crypto-common 0.1.7",
|
||||
"inout",
|
||||
"zeroize",
|
||||
]
|
||||
|
|
@ -457,6 +500,12 @@ version = "0.9.6"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.4.2"
|
||||
|
|
@ -522,6 +571,15 @@ version = "2.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "critical-section"
|
||||
version = "1.2.0"
|
||||
|
|
@ -563,6 +621,15 @@ dependencies = [
|
|||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
|
||||
dependencies = [
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto_box"
|
||||
version = "0.9.1"
|
||||
|
|
@ -604,7 +671,7 @@ dependencies = [
|
|||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"curve25519-dalek-derive",
|
||||
"digest",
|
||||
"digest 0.10.7",
|
||||
"fiat-crypto",
|
||||
"rand_core 0.6.4",
|
||||
"rustc_version",
|
||||
|
|
@ -636,7 +703,7 @@ version = "0.7.10"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
|
||||
dependencies = [
|
||||
"const-oid",
|
||||
"const-oid 0.9.6",
|
||||
"der_derive",
|
||||
"pem-rfc7468",
|
||||
"zeroize",
|
||||
|
|
@ -696,11 +763,22 @@ version = "0.10.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
"block-buffer 0.10.4",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
||||
dependencies = [
|
||||
"block-buffer 0.12.0",
|
||||
"const-oid 0.10.2",
|
||||
"crypto-common 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "directories"
|
||||
version = "5.0.1"
|
||||
|
|
@ -780,11 +858,17 @@ dependencies = [
|
|||
"ed25519",
|
||||
"rand_core 0.6.4",
|
||||
"serde",
|
||||
"sha2",
|
||||
"sha2 0.10.9",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
|
||||
|
||||
[[package]]
|
||||
name = "embedded-io"
|
||||
version = "0.4.0"
|
||||
|
|
@ -875,6 +959,16 @@ version = "0.1.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
|
|
@ -1190,7 +1284,10 @@ dependencies = [
|
|||
name = "geth-document"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"automerge",
|
||||
"base64",
|
||||
"geth-types",
|
||||
"hexane",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
|
|
@ -1506,6 +1603,16 @@ version = "0.4.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hexane"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f117759ffbdfa82096cd01c7adcc8465486df830eeafb5033502074f8df3f0c7"
|
||||
dependencies = [
|
||||
"leb128",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hickory-proto"
|
||||
version = "0.25.2"
|
||||
|
|
@ -1558,7 +1665,7 @@ version = "0.12.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest",
|
||||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1628,6 +1735,15 @@ version = "1.0.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "hybrid-array"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "1.9.0"
|
||||
|
|
@ -2108,6 +2224,15 @@ version = "1.70.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
|
|
@ -2132,6 +2257,12 @@ version = "1.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "leb128"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545"
|
||||
|
||||
[[package]]
|
||||
name = "leb128fmt"
|
||||
version = "0.1.0"
|
||||
|
|
@ -2253,6 +2384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2665,7 +2797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220"
|
||||
dependencies = [
|
||||
"pest",
|
||||
"sha2",
|
||||
"sha2 0.10.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3421,7 +3553,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
|
|||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"digest",
|
||||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3438,7 +3570,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
|||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"digest",
|
||||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.0",
|
||||
"digest 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3475,6 +3618,12 @@ dependencies = [
|
|||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
|
|
@ -3508,6 +3657,16 @@ version = "0.1.24"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9"
|
||||
|
||||
[[package]]
|
||||
name = "smol_str"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9676b89cd56310a87b93dec47b11af744f34d5fc9f367b829474eec0a891350d"
|
||||
dependencies = [
|
||||
"borsh",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snafu"
|
||||
version = "0.8.9"
|
||||
|
|
@ -4137,6 +4296,12 @@ dependencies = [
|
|||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.6"
|
||||
|
|
@ -4149,7 +4314,7 @@ version = "0.5.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ rust-version = "1.85"
|
|||
[workspace.dependencies]
|
||||
anyhow = "1"
|
||||
async-trait = "0.1"
|
||||
automerge = "0.7"
|
||||
base64 = "0.22"
|
||||
blake3 = "1"
|
||||
bytes = "1"
|
||||
|
|
@ -44,6 +45,7 @@ directories = "5"
|
|||
ed25519-dalek = { version = "2", features = ["rand_core"] }
|
||||
futures = "0.3"
|
||||
hex = "0.4"
|
||||
hexane = "=0.1.5"
|
||||
iroh = { version = "0.90.0", features = ["discovery-local-network"] }
|
||||
n0-watcher = "0.2"
|
||||
postcard = { version = "1", features = ["alloc"] }
|
||||
|
|
|
|||
13
README.md
13
README.md
|
|
@ -155,9 +155,10 @@ The bootstrap implementation provides:
|
|||
`--subject <principal>` to exercise local capability checks for non-local
|
||||
callers; `geth kv sync <node-id> <name> [--bearer-secret <secret>]` pulls
|
||||
authorized remote updates
|
||||
- local JSON document commands: `geth document create/status/set/get`; `geth
|
||||
document sync <node-id> <name> [--bearer-secret <secret>]` pulls authorized
|
||||
remote JSON state
|
||||
- local Automerge document commands: `geth document create/status/set/get`;
|
||||
CLI input and output are JSON views, while the store keeps durable Automerge
|
||||
save bytes. `geth document sync <node-id> <name> [--bearer-secret <secret>]`
|
||||
pulls authorized remote Automerge state.
|
||||
- local daemon-lifetime pubsub snapshots: `geth pubsub pub/sub`; `geth pubsub
|
||||
pub <topic> <message> --node <node-id>` publishes to an authorized peer;
|
||||
`geth pubsub sub <topic> --node <node-id>` reads an authorized peer snapshot
|
||||
|
|
@ -281,10 +282,10 @@ byte stream; SSH is not a geth transport backend.
|
|||
over the protected Iroh control path. It requires `ssh_proxy.admin_shell` on the
|
||||
same resource and supports only built-in commands (`help`, `status`, `node-id`);
|
||||
it does not execute host shell commands.
|
||||
Document sync is a bootstrap JSON last-writer-wins path before Automerge:
|
||||
Document sync exchanges durable Automerge state with a JSON view for CLI output:
|
||||
manual `geth document sync <node-id> <name>` and background live-sync require
|
||||
`document.read` on `resource:document:<name>` and import only state that is not
|
||||
older than the local document timestamp.
|
||||
`document.read` on `resource:document:<name>` and merge authorized remote state
|
||||
when the peer advertises a state timestamp at or after the local document.
|
||||
DB sync is a staged cr-sqlite path: manual `geth db sync <node-id> <name>` and
|
||||
background live-sync require `db.sync` on `resource:db:<name>`, exchange typed
|
||||
`crsql_changes` batches over the protected Iroh control path, and check remote
|
||||
|
|
|
|||
|
|
@ -6,6 +6,9 @@ rust-version.workspace = true
|
|||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
automerge.workspace = true
|
||||
base64.workspace = true
|
||||
hexane.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
use geth_types::{DocumentId, ResourceId, UnixMillis};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const AUTOMERGE_STATE_FORMAT: &str = "geth.automerge-document.v1";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct DocumentResource {
|
||||
pub id: DocumentId,
|
||||
|
|
@ -20,9 +22,18 @@ pub struct DocumentState {
|
|||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum DocumentError {
|
||||
#[error("invalid document name: {0}")]
|
||||
InvalidName(String),
|
||||
Name(String),
|
||||
#[error("invalid document JSON state: {0}")]
|
||||
InvalidState(#[from] serde_json::Error),
|
||||
JsonState(#[from] serde_json::Error),
|
||||
#[error("invalid Automerge document state: {0}")]
|
||||
AutomergeState(String),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
struct AutomergeStateEnvelope {
|
||||
format: String,
|
||||
automerge_base64: String,
|
||||
view_json: String,
|
||||
}
|
||||
|
||||
pub fn validate_document_name(name: &str) -> Result<(), DocumentError> {
|
||||
|
|
@ -31,7 +42,7 @@ pub fn validate_document_name(name: &str) -> Result<(), DocumentError> {
|
|||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
|
||||
{
|
||||
return Err(DocumentError::InvalidName(name.to_owned()));
|
||||
return Err(DocumentError::Name(name.to_owned()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -41,9 +52,201 @@ pub fn normalize_document_state(state_json: &str) -> Result<String, DocumentErro
|
|||
Ok(serde_json::to_string(&value)?)
|
||||
}
|
||||
|
||||
pub fn create_automerge_state(state_json: &str) -> Result<String, DocumentError> {
|
||||
let value: serde_json::Value = serde_json::from_str(state_json)?;
|
||||
let normalized = serde_json::to_string(&value)?;
|
||||
let mut doc = automerge::Automerge::new();
|
||||
{
|
||||
let mut tx = doc.transaction();
|
||||
write_json_object(&mut tx, &automerge::ROOT, &value)?;
|
||||
tx.commit();
|
||||
}
|
||||
envelope_from_doc(&doc, normalized)
|
||||
}
|
||||
|
||||
pub fn document_view_json(stored_state: &str) -> Result<String, DocumentError> {
|
||||
if let Some(envelope) = parse_envelope(stored_state)? {
|
||||
return Ok(envelope.view_json);
|
||||
}
|
||||
normalize_document_state(stored_state)
|
||||
}
|
||||
|
||||
pub fn document_state_bytes(stored_state: &str) -> u64 {
|
||||
match parse_envelope(stored_state) {
|
||||
Ok(Some(envelope)) => {
|
||||
use base64::Engine;
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(envelope.automerge_base64)
|
||||
.map(|bytes| bytes.len() as u64)
|
||||
.unwrap_or(stored_state.len() as u64)
|
||||
}
|
||||
Ok(None) | Err(_) => stored_state.len() as u64,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge_automerge_states(local: &str, remote: &str) -> Result<String, DocumentError> {
|
||||
let Some(local_envelope) = parse_envelope(local)? else {
|
||||
return create_automerge_state(remote);
|
||||
};
|
||||
let Some(remote_envelope) = parse_envelope(remote)? else {
|
||||
return create_automerge_state(&document_view_json(remote)?);
|
||||
};
|
||||
let mut local_doc = load_envelope_doc(&local_envelope)?;
|
||||
let mut remote_doc = load_envelope_doc(&remote_envelope)?;
|
||||
local_doc
|
||||
.merge(&mut remote_doc)
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))?;
|
||||
let view_json = serde_json::to_string(&automerge::AutoSerde::from(&local_doc))?;
|
||||
envelope_from_doc(&local_doc, view_json)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn automerge_roadmap() -> &'static str {
|
||||
"future documents use Automerge sync over Iroh with resource-local authorization"
|
||||
"documents are stored as Automerge save bytes with a JSON view; remote sync is resource-authorized and exchanges the durable Automerge state"
|
||||
}
|
||||
|
||||
fn parse_envelope(state: &str) -> Result<Option<AutomergeStateEnvelope>, DocumentError> {
|
||||
let value: serde_json::Value = serde_json::from_str(state)?;
|
||||
if value
|
||||
.get("format")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|format| format == AUTOMERGE_STATE_FORMAT)
|
||||
{
|
||||
return serde_json::from_value(value)
|
||||
.map(Some)
|
||||
.map_err(DocumentError::from);
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn load_envelope_doc(
|
||||
envelope: &AutomergeStateEnvelope,
|
||||
) -> Result<automerge::Automerge, DocumentError> {
|
||||
use base64::Engine;
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(&envelope.automerge_base64)
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))?;
|
||||
automerge::Automerge::load(&bytes)
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))
|
||||
}
|
||||
|
||||
fn envelope_from_doc(
|
||||
doc: &automerge::Automerge,
|
||||
view_json: String,
|
||||
) -> Result<String, DocumentError> {
|
||||
use base64::Engine;
|
||||
let envelope = AutomergeStateEnvelope {
|
||||
format: AUTOMERGE_STATE_FORMAT.to_owned(),
|
||||
automerge_base64: base64::engine::general_purpose::STANDARD.encode(doc.save()),
|
||||
view_json,
|
||||
};
|
||||
serde_json::to_string(&envelope).map_err(DocumentError::from)
|
||||
}
|
||||
|
||||
fn write_json_object(
|
||||
tx: &mut automerge::transaction::Transaction<'_>,
|
||||
obj: &automerge::ObjId,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<(), DocumentError> {
|
||||
match value {
|
||||
serde_json::Value::Object(map) => {
|
||||
for (key, value) in map {
|
||||
put_json_value(tx, obj, key, value)?;
|
||||
}
|
||||
}
|
||||
other => {
|
||||
put_json_value(tx, obj, "value", other)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn put_json_value(
|
||||
tx: &mut automerge::transaction::Transaction<'_>,
|
||||
obj: &automerge::ObjId,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<(), DocumentError> {
|
||||
use automerge::transaction::Transactable;
|
||||
match value {
|
||||
serde_json::Value::Null => tx.put(obj, key, ()),
|
||||
serde_json::Value::Bool(value) => tx.put(obj, key, *value),
|
||||
serde_json::Value::Number(value) => {
|
||||
if let Some(value) = value.as_i64() {
|
||||
tx.put(obj, key, value)
|
||||
} else if let Some(value) = value.as_u64() {
|
||||
tx.put(obj, key, value)
|
||||
} else if let Some(value) = value.as_f64() {
|
||||
tx.put(obj, key, value)
|
||||
} else {
|
||||
Err(automerge::AutomergeError::Fail)
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(value) => tx.put(obj, key, value.as_str()),
|
||||
serde_json::Value::Array(values) => {
|
||||
let list = tx
|
||||
.put_object(obj, key, automerge::ObjType::List)
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))?;
|
||||
for (index, value) in values.iter().enumerate() {
|
||||
insert_json_value(tx, &list, index, value)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
serde_json::Value::Object(map) => {
|
||||
let map_obj = tx
|
||||
.put_object(obj, key, automerge::ObjType::Map)
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))?;
|
||||
for (child_key, value) in map {
|
||||
put_json_value(tx, &map_obj, child_key, value)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))
|
||||
}
|
||||
|
||||
fn insert_json_value(
|
||||
tx: &mut automerge::transaction::Transaction<'_>,
|
||||
obj: &automerge::ObjId,
|
||||
index: usize,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<(), DocumentError> {
|
||||
use automerge::transaction::Transactable;
|
||||
match value {
|
||||
serde_json::Value::Null => tx.insert(obj, index, ()),
|
||||
serde_json::Value::Bool(value) => tx.insert(obj, index, *value),
|
||||
serde_json::Value::Number(value) => {
|
||||
if let Some(value) = value.as_i64() {
|
||||
tx.insert(obj, index, value)
|
||||
} else if let Some(value) = value.as_u64() {
|
||||
tx.insert(obj, index, value)
|
||||
} else if let Some(value) = value.as_f64() {
|
||||
tx.insert(obj, index, value)
|
||||
} else {
|
||||
Err(automerge::AutomergeError::Fail)
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(value) => tx.insert(obj, index, value.as_str()),
|
||||
serde_json::Value::Array(values) => {
|
||||
let list = tx
|
||||
.insert_object(obj, index, automerge::ObjType::List)
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))?;
|
||||
for (index, value) in values.iter().enumerate() {
|
||||
insert_json_value(tx, &list, index, value)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
serde_json::Value::Object(map) => {
|
||||
let map_obj = tx
|
||||
.insert_object(obj, index, automerge::ObjType::Map)
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))?;
|
||||
for (child_key, value) in map {
|
||||
put_json_value(tx, &map_obj, child_key, value)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
.map_err(|error| DocumentError::AutomergeState(error.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -68,4 +271,29 @@ mod tests {
|
|||
);
|
||||
assert!(normalize_document_state("{").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automerge_state_preserves_json_view_and_durable_bytes() {
|
||||
let state = create_automerge_state(r#"{ "title": "notes", "items": [1, true, null] }"#)
|
||||
.expect("automerge state");
|
||||
|
||||
assert_eq!(
|
||||
document_view_json(&state).expect("view"),
|
||||
r#"{"items":[1,true,null],"title":"notes"}"#
|
||||
);
|
||||
assert!(document_state_bytes(&state) > 2);
|
||||
assert!(state.contains(AUTOMERGE_STATE_FORMAT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automerge_states_can_merge_and_render_json_view() {
|
||||
let left = create_automerge_state(r#"{ "left": true }"#).expect("left state");
|
||||
let right = create_automerge_state(r#"{ "right": true }"#).expect("right state");
|
||||
|
||||
let merged = merge_automerge_states(&left, &right).expect("merge");
|
||||
let view = document_view_json(&merged).expect("merged view");
|
||||
|
||||
assert!(view.contains(r#""left":true"#));
|
||||
assert!(view.contains(r#""right":true"#));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2537,12 +2537,15 @@ async fn document_sync_from_peer(
|
|||
if let Some(state) = state {
|
||||
let local = ensure_local_document(&store, name)?;
|
||||
if state.updated_at.0 >= local.updated_at_ms {
|
||||
let normalized = geth_document::normalize_document_state(&state.state_json)?;
|
||||
let merged = geth_document::merge_automerge_states(
|
||||
&local.state_json,
|
||||
&state.state_json,
|
||||
)?;
|
||||
store.insert_document_resource(&StoredDocumentResource {
|
||||
document_id: local.document_id,
|
||||
resource_id: local.resource_id,
|
||||
name: local.name,
|
||||
state_json: normalized,
|
||||
state_json: merged,
|
||||
updated_at_ms: state.updated_at.0,
|
||||
})?;
|
||||
updated = true;
|
||||
|
|
@ -5242,7 +5245,7 @@ async fn handle_iroh_control_connection(
|
|||
bearer_proof.as_ref(),
|
||||
)?;
|
||||
let state = if explanation.allowed && document.updated_at_ms >= since_ms {
|
||||
Some(document_state_from_stored(&document))
|
||||
Some(document_sync_state_from_stored(&document)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
|
@ -5258,7 +5261,7 @@ async fn handle_iroh_control_connection(
|
|||
reason: explanation.reason,
|
||||
evaluated_ops: explanation.evaluated_ops,
|
||||
nonce,
|
||||
note: "document sync authenticated endpoint/card binding and required document.read on the remote document resource; JSON LWW sync is a bootstrap before Automerge".to_owned(),
|
||||
note: "document sync authenticated endpoint/card binding and required document.read on the remote document resource; durable Automerge state is exchanged with a JSON view for CLI output".to_owned(),
|
||||
}
|
||||
} else {
|
||||
PeerControlResponse::Error {
|
||||
|
|
@ -7430,7 +7433,7 @@ pub fn handle_request(
|
|||
document_id,
|
||||
resource_id,
|
||||
name,
|
||||
state_json: "{}".to_owned(),
|
||||
state_json: geth_document::create_automerge_state("{}")?,
|
||||
updated_at_ms: geth_store::now_ms(),
|
||||
};
|
||||
store.insert_document_resource(&stored)?;
|
||||
|
|
@ -7454,11 +7457,11 @@ pub fn handle_request(
|
|||
let mut stored = store
|
||||
.get_document_resource_by_name(&name)?
|
||||
.ok_or_else(|| NodeError::DocumentNotFound(name.clone()))?;
|
||||
stored.state_json = geth_document::normalize_document_state(&state_json)?;
|
||||
stored.state_json = geth_document::create_automerge_state(&state_json)?;
|
||||
stored.updated_at_ms = geth_store::now_ms();
|
||||
store.insert_document_resource(&stored)?;
|
||||
Ok(ControlResponse::DocumentSet {
|
||||
state: document_state_from_stored(&stored),
|
||||
state: document_state_from_stored(&stored)?,
|
||||
})
|
||||
}
|
||||
ControlRequest::DocumentGet { name } => {
|
||||
|
|
@ -7468,7 +7471,7 @@ pub fn handle_request(
|
|||
.get_document_resource_by_name(&name)?
|
||||
.ok_or_else(|| NodeError::DocumentNotFound(name.clone()))?;
|
||||
Ok(ControlResponse::DocumentGet {
|
||||
state: document_state_from_stored(&stored),
|
||||
state: document_state_from_stored(&stored)?,
|
||||
})
|
||||
}
|
||||
ControlRequest::PubsubPub {
|
||||
|
|
@ -7820,7 +7823,7 @@ fn ensure_local_document(store: &Store, name: &str) -> Result<StoredDocumentReso
|
|||
document_id: format!("document:{name}"),
|
||||
resource_id: format!("resource:document:{name}"),
|
||||
name: name.to_owned(),
|
||||
state_json: "{}".to_owned(),
|
||||
state_json: geth_document::create_automerge_state("{}")?,
|
||||
updated_at_ms: 0,
|
||||
};
|
||||
store.insert_document_resource(&stored)?;
|
||||
|
|
@ -7832,17 +7835,27 @@ fn document_resource_from_stored(stored: &StoredDocumentResource) -> DocumentRes
|
|||
id: stored.document_id.clone().into(),
|
||||
resource: stored.resource_id.clone().into(),
|
||||
name: stored.name.clone(),
|
||||
sync_status: "local-only".to_owned(),
|
||||
state_bytes: stored.state_json.len() as u64,
|
||||
sync_status: "automerge-local".to_owned(),
|
||||
state_bytes: geth_document::document_state_bytes(&stored.state_json),
|
||||
}
|
||||
}
|
||||
|
||||
fn document_state_from_stored(stored: &StoredDocumentResource) -> DocumentState {
|
||||
DocumentState {
|
||||
fn document_state_from_stored(stored: &StoredDocumentResource) -> Result<DocumentState, NodeError> {
|
||||
Ok(DocumentState {
|
||||
document: document_resource_from_stored(stored),
|
||||
state_json: geth_document::document_view_json(&stored.state_json)?,
|
||||
updated_at: UnixMillis(stored.updated_at_ms),
|
||||
})
|
||||
}
|
||||
|
||||
fn document_sync_state_from_stored(
|
||||
stored: &StoredDocumentResource,
|
||||
) -> Result<DocumentState, NodeError> {
|
||||
Ok(DocumentState {
|
||||
document: document_resource_from_stored(stored),
|
||||
state_json: stored.state_json.clone(),
|
||||
updated_at: UnixMillis(stored.updated_at_ms),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn file_root_from_stored(stored: &StoredFileRoot) -> FileRoot {
|
||||
|
|
@ -11528,7 +11541,7 @@ mod tests {
|
|||
assert!(allowed);
|
||||
assert!(updated);
|
||||
assert!(reason.contains("direct grant"));
|
||||
assert!(note.contains("JSON LWW"));
|
||||
assert!(note.contains("durable Automerge"));
|
||||
}
|
||||
other => panic!("unexpected allowed document sync response: {other:?}"),
|
||||
}
|
||||
|
|
@ -11804,7 +11817,10 @@ mod tests {
|
|||
left_store
|
||||
.get_document_resource_by_name("notes")
|
||||
.expect("get live-synced document")
|
||||
.map(|document| document.state_json),
|
||||
.map(|document| {
|
||||
geth_document::document_view_json(&document.state_json)
|
||||
.expect("document view json")
|
||||
}),
|
||||
Some(r#"{"title":"live"}"#.to_owned())
|
||||
);
|
||||
let live_remote_root = left_store
|
||||
|
|
|
|||
|
|
@ -2889,8 +2889,8 @@ fn document_create_and_status_use_local_store() {
|
|||
match response {
|
||||
geth_control::ControlResponse::DocumentCreated { document } => {
|
||||
assert_eq!(document.name, "notes");
|
||||
assert_eq!(document.sync_status, "local-only");
|
||||
assert_eq!(document.state_bytes, 2);
|
||||
assert_eq!(document.sync_status, "automerge-local");
|
||||
assert!(document.state_bytes > 2);
|
||||
}
|
||||
other => panic!("unexpected response: {other:?}"),
|
||||
}
|
||||
|
|
@ -2906,7 +2906,7 @@ fn document_create_and_status_use_local_store() {
|
|||
geth_control::ControlResponse::DocumentStatus { document } => {
|
||||
assert_eq!(document.id.to_string(), "document:notes");
|
||||
assert_eq!(document.resource.to_string(), "resource:document:notes");
|
||||
assert_eq!(document.state_bytes, 2);
|
||||
assert!(document.state_bytes > 2);
|
||||
}
|
||||
other => panic!("unexpected response: {other:?}"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -209,15 +209,16 @@ live-sync loop runs the same KV sync for local KV stores and known peers.
|
|||
Private value encryption should use resource secret epochs before payloads are
|
||||
exposed to remote peers.
|
||||
|
||||
`geth-document` currently registers local document resources and stores
|
||||
validated JSON state in the local SQLite metadata store through
|
||||
`document create/status/set/get`. This is a bootstrap editing surface, not yet
|
||||
Automerge CRDT state. `geth document sync <node-id> <name>` can pull remote JSON
|
||||
state over the protected Iroh control ALPN when the peer grants `document.read`
|
||||
on `resource:document:<name>`. The daemon background live-sync loop runs the same
|
||||
sync for local documents and known peers using per-peer/per-document cursors.
|
||||
The import rule is last-writer-wins by document timestamp. Automerge state
|
||||
encoding and sync are future work.
|
||||
`geth-document` registers local document resources and stores durable Automerge
|
||||
save bytes in the local SQLite metadata store. The CLI still accepts and returns
|
||||
validated JSON views for `document create/status/set/get`, but the persisted
|
||||
state is an Automerge envelope containing binary save data plus the current JSON
|
||||
view for operator output. `geth document sync <node-id> <name>` pulls remote
|
||||
Automerge state over the protected Iroh control ALPN when the peer grants
|
||||
`document.read` on `resource:document:<name>`. The daemon background live-sync
|
||||
loop runs the same sync for local documents and known peers using
|
||||
per-peer/per-document cursors. Received Automerge documents are merged before
|
||||
being stored.
|
||||
|
||||
`geth-pubsub` currently supports local publish/subscribe snapshots through the
|
||||
daemon control protocol. Messages live in a bounded in-memory ring buffer and
|
||||
|
|
|
|||
|
|
@ -47,11 +47,11 @@ Implementation order:
|
|||
- `[x]` Keep ambiguous changes as durable conflicts.
|
||||
- `[x]` Add two-root integration coverage.
|
||||
|
||||
4. `[ ]` Replace JSON document state with durable Automerge documents.
|
||||
4. `[x]` Replace JSON document state with durable Automerge documents.
|
||||
Acceptance criteria:
|
||||
- `[ ]` Store Automerge documents durably.
|
||||
- `[ ]` Sync Automerge changes over Iroh.
|
||||
- `[ ]` Gate document sync with resource authorization.
|
||||
- `[x]` Store Automerge documents durably.
|
||||
- `[x]` Sync Automerge changes over Iroh.
|
||||
- `[x]` Gate document sync with resource authorization.
|
||||
|
||||
5. `[ ]` Harden DB sync for real cr-sqlite usage.
|
||||
Acceptance criteria:
|
||||
|
|
@ -115,12 +115,13 @@ Implementation order:
|
|||
- `[x]` Tests cover create/update/delete/rename application across two local
|
||||
roots.
|
||||
|
||||
- `[~]` Durable Automerge documents.
|
||||
- `[x]` Durable Automerge documents.
|
||||
Acceptance criteria:
|
||||
- `[ ]` Document state is stored as durable Automerge data, not only JSON
|
||||
- `[x]` Document state is stored as durable Automerge data, not only JSON
|
||||
last-writer-wins state.
|
||||
- `[ ]` Sync converges according to Automerge semantics across two nodes.
|
||||
- `[ ]` Resource authorization gates remote document reads and writes.
|
||||
- `[x]` Sync exchanges durable Automerge state over the protected Iroh
|
||||
control path and merges received Automerge documents.
|
||||
- `[x]` Resource authorization gates remote document reads and writes.
|
||||
|
||||
- `[~]` Operational first-run polish.
|
||||
Acceptance criteria:
|
||||
|
|
@ -665,24 +666,24 @@ Automerge documents.
|
|||
- `[~]` Automerge document resource.
|
||||
Acceptance criteria:
|
||||
- `[x]` `geth document create/status` works for local documents.
|
||||
- `[x]` Document metadata and empty JSON state are stored durably.
|
||||
- `[x]` `geth document set/get` stores and returns validated local JSON
|
||||
state.
|
||||
- `[x]` Tests cover create, update, save, and reload of local JSON document
|
||||
state.
|
||||
- `[x]` `geth document sync <node-id> <name>` pulls authorized JSON state
|
||||
- `[x]` Document metadata and empty Automerge state are stored durably.
|
||||
- `[x]` `geth document set/get` stores durable Automerge save bytes and
|
||||
returns a validated JSON view.
|
||||
- `[x]` Tests cover create, update, save, and reload of local Automerge
|
||||
document state.
|
||||
- `[x]` `geth document sync <node-id> <name>` pulls authorized Automerge state
|
||||
from an imported peer over Iroh.
|
||||
- `[x]` Background live-sync refreshes local JSON documents from known peers
|
||||
- `[x]` Background live-sync refreshes local Automerge documents from known peers
|
||||
using per-peer/per-document cursors.
|
||||
- `[ ]` Automerge document state is stored durably.
|
||||
- `[ ]` Tests cover Automerge create, update, save, and reload.
|
||||
- `[x]` Automerge document state is stored durably.
|
||||
- `[x]` Tests cover Automerge create, update, save, and reload.
|
||||
|
||||
- `[ ]` Automerge sync over Iroh.
|
||||
- `[x]` Automerge sync over Iroh.
|
||||
Acceptance criteria:
|
||||
- `[x]` Bootstrap JSON last-writer-wins sync works across two local test
|
||||
nodes over Iroh.
|
||||
- Resource authorization gates read/write sync.
|
||||
- Conflicts converge according to Automerge semantics.
|
||||
- `[x]` Automerge state sync works across two local test nodes over Iroh.
|
||||
- `[x]` Resource authorization gates read/write sync.
|
||||
- `[x]` Received Automerge documents are merged rather than stored as raw
|
||||
JSON blobs.
|
||||
|
||||
## Phase 6: File Sync And Advanced Local-First Auth
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue