Add SQLite schema metadata for DB resources
This commit is contained in:
parent
1a0f1e7671
commit
527090fd4c
8 changed files with 119 additions and 14 deletions
|
|
@ -118,8 +118,9 @@ Roadmap items should be actionable and checkable:
|
||||||
`keychain status`. SSH signature capture/verification is still roadmap work.
|
`keychain status`. SSH signature capture/verification is still roadmap work.
|
||||||
- Local CAS supports pin/unpin metadata, surfaced through `cas list`, and
|
- Local CAS supports pin/unpin metadata, surfaced through `cas list`, and
|
||||||
`cas cleanup` evicts unpinned blobs while retaining pinned blobs.
|
`cas cleanup` evicts unpinned blobs while retaining pinned blobs.
|
||||||
- DB resources can be registered locally and report local-only status. cr-sqlite
|
- DB resources can be registered locally and report local-only status plus a
|
||||||
loading, change extraction, and sync are still roadmap work.
|
read-only SQLite schema summary/hash. cr-sqlite loading, change extraction,
|
||||||
|
and sync are still roadmap work.
|
||||||
- KV stores support local SQLite-backed create/set/get. Iroh Documents
|
- KV stores support local SQLite-backed create/set/get. Iroh Documents
|
||||||
replication and prefix-capability enforcement are still roadmap work.
|
replication and prefix-capability enforcement are still roadmap work.
|
||||||
- Signed peer-card LAN discovery payloads, peer auth over Iroh, cr-sqlite,
|
- Signed peer-card LAN discovery payloads, peer auth over Iroh, cr-sqlite,
|
||||||
|
|
|
||||||
4
Cargo.lock
generated
4
Cargo.lock
generated
|
|
@ -1047,6 +1047,7 @@ dependencies = [
|
||||||
"geth-control",
|
"geth-control",
|
||||||
"geth-node",
|
"geth-node",
|
||||||
"geth-store",
|
"geth-store",
|
||||||
|
"rusqlite",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
|
@ -1145,8 +1146,11 @@ dependencies = [
|
||||||
name = "geth-db"
|
name = "geth-db"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"blake3",
|
||||||
"geth-types",
|
"geth-types",
|
||||||
|
"rusqlite",
|
||||||
"serde",
|
"serde",
|
||||||
|
"tempfile",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,11 @@ rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
blake3.workspace = true
|
||||||
|
rusqlite.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
geth-types = { path = "../geth-types" }
|
geth-types = { path = "../geth-types" }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tempfile.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
use geth_types::{DbId, ResourceId};
|
use geth_types::{DbId, ResourceId};
|
||||||
|
use rusqlite::{Connection, OpenFlags};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct DbResource {
|
pub struct DbResource {
|
||||||
|
|
@ -17,6 +19,8 @@ pub struct DbResource {
|
||||||
pub enum DbError {
|
pub enum DbError {
|
||||||
#[error("invalid db resource name: {0}")]
|
#[error("invalid db resource name: {0}")]
|
||||||
InvalidName(String),
|
InvalidName(String),
|
||||||
|
#[error("sqlite error: {0}")]
|
||||||
|
Sqlite(#[from] rusqlite::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn validate_db_name(name: &str) -> Result<(), DbError> {
|
pub fn validate_db_name(name: &str) -> Result<(), DbError> {
|
||||||
|
|
@ -30,6 +34,50 @@ pub fn validate_db_name(name: &str) -> Result<(), DbError> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn schema_metadata(path: &Path) -> Result<String, DbError> {
|
||||||
|
let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
r#"SELECT type, name, COALESCE(sql, '')
|
||||||
|
FROM sqlite_master
|
||||||
|
WHERE name NOT LIKE 'sqlite_%'
|
||||||
|
ORDER BY type, name"#,
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map([], |row| {
|
||||||
|
Ok((
|
||||||
|
row.get::<_, String>(0)?,
|
||||||
|
row.get::<_, String>(1)?,
|
||||||
|
row.get::<_, String>(2)?,
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut canonical = String::new();
|
||||||
|
let mut table_count = 0_usize;
|
||||||
|
let mut index_count = 0_usize;
|
||||||
|
let mut view_count = 0_usize;
|
||||||
|
let mut trigger_count = 0_usize;
|
||||||
|
for row in rows {
|
||||||
|
let (kind, name, sql) = row?;
|
||||||
|
match kind.as_str() {
|
||||||
|
"table" => table_count += 1,
|
||||||
|
"index" => index_count += 1,
|
||||||
|
"view" => view_count += 1,
|
||||||
|
"trigger" => trigger_count += 1,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
canonical.push_str(&kind);
|
||||||
|
canonical.push('\0');
|
||||||
|
canonical.push_str(&name);
|
||||||
|
canonical.push('\0');
|
||||||
|
canonical.push_str(&sql);
|
||||||
|
canonical.push('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(format!(
|
||||||
|
"tables={table_count} indexes={index_count} views={view_count} triggers={trigger_count} schema_hash={}",
|
||||||
|
blake3::hash(canonical.as_bytes())
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn crsqlite_sync_roadmap() -> &'static str {
|
pub fn crsqlite_sync_roadmap() -> &'static str {
|
||||||
"future db sync reads crsql_changes, exchanges changes over Iroh, and applies through crsql_changes"
|
"future db sync reads crsql_changes, exchanges changes over Iroh, and applies through crsql_changes"
|
||||||
|
|
@ -48,4 +96,27 @@ mod tests {
|
||||||
assert!(validate_db_name("notes/main").is_err());
|
assert!(validate_db_name("notes/main").is_err());
|
||||||
assert!(validate_db_name("notes main").is_err());
|
assert!(validate_db_name("notes main").is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_metadata_reports_counts_and_stable_hash() {
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let path = dir.path().join("notes.sqlite");
|
||||||
|
let conn = Connection::open(&path).expect("open sqlite");
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE notes(id INTEGER PRIMARY KEY, body TEXT NOT NULL)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.expect("create table");
|
||||||
|
conn.execute("CREATE INDEX notes_body ON notes(body)", [])
|
||||||
|
.expect("create index");
|
||||||
|
drop(conn);
|
||||||
|
|
||||||
|
let first = schema_metadata(&path).expect("schema metadata");
|
||||||
|
let second = schema_metadata(&path).expect("schema metadata again");
|
||||||
|
|
||||||
|
assert_eq!(first, second);
|
||||||
|
assert!(first.contains("tables=1"));
|
||||||
|
assert!(first.contains("indexes=1"));
|
||||||
|
assert!(first.contains("schema_hash="));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -40,6 +40,8 @@ pub enum NodeError {
|
||||||
Store(#[from] geth_store::StoreError),
|
Store(#[from] geth_store::StoreError),
|
||||||
#[error("cas error: {0}")]
|
#[error("cas error: {0}")]
|
||||||
Cas(#[from] geth_cas::CasError),
|
Cas(#[from] geth_cas::CasError),
|
||||||
|
#[error("db error: {0}")]
|
||||||
|
Db(#[from] geth_db::DbError),
|
||||||
#[error("control error: {0}")]
|
#[error("control error: {0}")]
|
||||||
Control(#[from] geth_control::ControlError),
|
Control(#[from] geth_control::ControlError),
|
||||||
#[error("json error: {0}")]
|
#[error("json error: {0}")]
|
||||||
|
|
@ -570,6 +572,7 @@ pub fn handle_request(
|
||||||
return Err(NodeError::InvalidDbPath(path.display().to_string()));
|
return Err(NodeError::InvalidDbPath(path.display().to_string()));
|
||||||
}
|
}
|
||||||
let path = std::fs::canonicalize(path)?;
|
let path = std::fs::canonicalize(path)?;
|
||||||
|
let schema_metadata = geth_db::schema_metadata(&path)?;
|
||||||
let resource_id = format!("resource:db:{name}");
|
let resource_id = format!("resource:db:{name}");
|
||||||
let db_id = format!("db:{name}");
|
let db_id = format!("db:{name}");
|
||||||
let resource = StoredResource {
|
let resource = StoredResource {
|
||||||
|
|
@ -587,7 +590,7 @@ pub fn handle_request(
|
||||||
};
|
};
|
||||||
store.insert_db_resource(&stored)?;
|
store.insert_db_resource(&stored)?;
|
||||||
Ok(ControlResponse::DbAdded {
|
Ok(ControlResponse::DbAdded {
|
||||||
db: db_resource_from_stored(&stored)?,
|
db: db_resource_from_stored_with_schema(&stored, schema_metadata),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
ControlRequest::DbStatus { name } => {
|
ControlRequest::DbStatus { name } => {
|
||||||
|
|
@ -669,16 +672,30 @@ fn stored_resource_to_descriptor(stored: StoredResource) -> Result<ResourceDescr
|
||||||
fn db_resource_from_stored(stored: &StoredDbResource) -> Result<DbResource, NodeError> {
|
fn db_resource_from_stored(stored: &StoredDbResource) -> Result<DbResource, NodeError> {
|
||||||
let path = Path::new(&stored.path);
|
let path = Path::new(&stored.path);
|
||||||
let metadata = path.metadata().ok();
|
let metadata = path.metadata().ok();
|
||||||
Ok(DbResource {
|
let schema_metadata = if metadata.as_ref().is_some_and(std::fs::Metadata::is_file) {
|
||||||
|
geth_db::schema_metadata(path).unwrap_or_else(|error| format!("unavailable:{error}"))
|
||||||
|
} else {
|
||||||
|
"unavailable:path-missing".to_owned()
|
||||||
|
};
|
||||||
|
Ok(db_resource_from_stored_with_schema(stored, schema_metadata))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn db_resource_from_stored_with_schema(
|
||||||
|
stored: &StoredDbResource,
|
||||||
|
schema_metadata: String,
|
||||||
|
) -> DbResource {
|
||||||
|
let path = Path::new(&stored.path);
|
||||||
|
let metadata = path.metadata().ok();
|
||||||
|
DbResource {
|
||||||
id: stored.db_id.clone().into(),
|
id: stored.db_id.clone().into(),
|
||||||
resource: stored.resource_id.clone().into(),
|
resource: stored.resource_id.clone().into(),
|
||||||
name: stored.name.clone(),
|
name: stored.name.clone(),
|
||||||
path: stored.path.clone(),
|
path: stored.path.clone(),
|
||||||
path_exists: metadata.as_ref().is_some_and(std::fs::Metadata::is_file),
|
path_exists: metadata.as_ref().is_some_and(std::fs::Metadata::is_file),
|
||||||
size_bytes: metadata.map(|metadata| metadata.len()),
|
size_bytes: metadata.map(|metadata| metadata.len()),
|
||||||
schema_metadata: "not-inspected-until-crsqlite-integration".to_owned(),
|
schema_metadata,
|
||||||
sync_status: "local-only".to_owned(),
|
sync_status: "local-only".to_owned(),
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn kv_resource_from_stored(stored: &StoredKvStore) -> KvResource {
|
fn kv_resource_from_stored(stored: &StoredKvStore) -> KvResource {
|
||||||
|
|
|
||||||
|
|
@ -21,4 +21,5 @@ geth-config = { path = "../geth-config" }
|
||||||
geth-control = { path = "../geth-control" }
|
geth-control = { path = "../geth-control" }
|
||||||
geth-node = { path = "../geth-node" }
|
geth-node = { path = "../geth-node" }
|
||||||
geth-store = { path = "../geth-store" }
|
geth-store = { path = "../geth-store" }
|
||||||
|
rusqlite.workspace = true
|
||||||
tempfile.workspace = true
|
tempfile.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -393,7 +393,13 @@ fn db_add_and_status_register_local_db_metadata() {
|
||||||
let paths = geth_config::GethPaths::from_home(home.path());
|
let paths = geth_config::GethPaths::from_home(home.path());
|
||||||
let node = geth_node::init_node(&paths).expect("init node");
|
let node = geth_node::init_node(&paths).expect("init node");
|
||||||
let db_path = home.path().join("notes.sqlite");
|
let db_path = home.path().join("notes.sqlite");
|
||||||
std::fs::write(&db_path, b"sqlite placeholder").expect("write db");
|
let conn = rusqlite::Connection::open(&db_path).expect("open sqlite");
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE notes(id INTEGER PRIMARY KEY, body TEXT NOT NULL)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.expect("create notes table");
|
||||||
|
drop(conn);
|
||||||
|
|
||||||
let response = geth_node::handle_request(
|
let response = geth_node::handle_request(
|
||||||
&node,
|
&node,
|
||||||
|
|
@ -408,7 +414,9 @@ fn db_add_and_status_register_local_db_metadata() {
|
||||||
assert_eq!(db.name, "notes");
|
assert_eq!(db.name, "notes");
|
||||||
assert_eq!(db.sync_status, "local-only");
|
assert_eq!(db.sync_status, "local-only");
|
||||||
assert!(db.path_exists);
|
assert!(db.path_exists);
|
||||||
assert_eq!(db.size_bytes, Some(18));
|
assert!(db.size_bytes.unwrap_or_default() > 0);
|
||||||
|
assert!(db.schema_metadata.contains("tables=1"));
|
||||||
|
assert!(db.schema_metadata.contains("schema_hash="));
|
||||||
}
|
}
|
||||||
other => panic!("unexpected response: {other:?}"),
|
other => panic!("unexpected response: {other:?}"),
|
||||||
}
|
}
|
||||||
|
|
@ -424,10 +432,8 @@ fn db_add_and_status_register_local_db_metadata() {
|
||||||
geth_control::ControlResponse::DbStatus { db } => {
|
geth_control::ControlResponse::DbStatus { db } => {
|
||||||
assert_eq!(db.name, "notes");
|
assert_eq!(db.name, "notes");
|
||||||
assert!(db.path.ends_with("notes.sqlite"));
|
assert!(db.path.ends_with("notes.sqlite"));
|
||||||
assert_eq!(
|
assert!(db.schema_metadata.contains("tables=1"));
|
||||||
db.schema_metadata,
|
assert!(db.schema_metadata.contains("schema_hash="));
|
||||||
"not-inspected-until-crsqlite-integration"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
other => panic!("unexpected response: {other:?}"),
|
other => panic!("unexpected response: {other:?}"),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -94,8 +94,8 @@ blobs. Iroh-blobs, providers, encrypted blobs, richer cache policies, manifests,
|
||||||
and file sync trees are future work.
|
and file sync trees are future work.
|
||||||
|
|
||||||
`geth-db` currently registers local SQLite paths as DB resources and reports
|
`geth-db` currently registers local SQLite paths as DB resources and reports
|
||||||
local-only sync status plus placeholder schema metadata. cr-sqlite loading,
|
local-only sync status plus a read-only SQLite schema summary/hash. cr-sqlite
|
||||||
change extraction, and DB sync are future work.
|
loading, change extraction, and DB sync are future work.
|
||||||
|
|
||||||
`geth-kv` currently provides a SQLite-backed local fallback for named KV stores
|
`geth-kv` currently provides a SQLite-backed local fallback for named KV stores
|
||||||
through `kv create/set/get`. Iroh Documents namespaces, prefix authorization
|
through `kv create/set/get`. Iroh Documents namespaces, prefix authorization
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue