From 96d2412370824e2b77b6139b815477fa6bd46391 Mon Sep 17 00:00:00 2001 From: randogoth Date: Tue, 29 Sep 2026 00:16:54 +0300 Subject: [PATCH] feat: schema v2 with the kept column and contact history, migrated at open --- README.md | 9 +-- core/src/client.rs | 7 +- core/src/store.rs | 161 +++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 161 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ee40e36..0442195 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,9 @@ The intended call graph: `Store::open` once, `Account::new(master, rotations)` p ## The store schema -The schema is a contract, not an implementation detail: `SCHEMA_VERSION` is 1, written to SQLite's `PRAGMA user_version` at creation and checked at open, so a foreign importer — a migration from another client, a sync tool — can build rows directly instead of reading `store.rs`. `Store::open_in_memory()` gives an importer a scratch database with schema and version in place. Within a version the layout below is stable; a version bump comes with a documented migration. +The schema is a contract, not an implementation detail: `SCHEMA_VERSION` is 2, written to SQLite's `PRAGMA user_version` at creation and checked at open, so a foreign importer — a migration from another client, a sync tool — can build rows directly instead of reading `store.rs`. `Store::open_in_memory()` gives an importer a scratch database with schema and version in place. Within a version the layout below is stable; a version bump comes with a migration run automatically at open (version 1 stores gain the `kept` column and the `history` table in place), and only stores newer than the build are refused. -Every mail column holds a sealed envelope, never plaintext. Timestamps are Unix seconds; keys and ids are 32-byte blobs. +Every mail column holds a sealed envelope, never plaintext. Timestamps are Unix seconds, except `history.until`, which is milliseconds; keys and ids are 32-byte blobs. | Table | Columns | Meaning and invariants | |---|---|---| @@ -97,10 +97,11 @@ Every mail column holds a sealed envelope, never plaintext. Timestamps are Unix | `accepted` | `address`, `identity`, `active`, `added_at` | Main-tier admission (sec 5.8). `identity` is frozen at acceptance because the token the correspondent holds is derived from it; `active = 0` is a block and keeps the row | | `tokens` | `address`, `token`, `seen_at` | Accept tokens correspondents issued us, filed under their address, which outlives the keys behind it (sec 5.8) | | `seen` | `id`, `at` | Every message id ever fetched; a replay is not re-stored even after a local delete (sec 10) | -| `inbox` | `id`, `envelope`, `received_at`, `tier` | Sealed mail; `tier` is 0 main, 1 requests (arrived without a matching accept token) | +| `inbox` | `id`, `envelope`, `received_at`, `tier`, `kept` | Sealed mail; `tier` is 0 main, 1 requests (arrived without a matching accept token); `kept` records whether a server copy existed when the message was stored, so a local delete can offer to remove it there too — it is written once at first store and does not track later server-side deletes | +| `history` | `address`, `identity`, `until` | Keys a contact used before its current one, with when each stopped being current (epoch ms): the local record that a rotation happened, written when a signed chain displaces the key held (sec 7, sec 8) | | `sent` | `id`, `recipient`, `envelope`, `sent_at` | Sealed self-copies (sec 5.6), with the recipient for the listing | -An importer should set `user_version` to 1, keep the single `state` row, and respect the frozen `accepted.identity` and `sync_ok` semantics above — those are the two rows where a wrong guess loses data (an erased server-side token set, or a contact's token silently changing). +An importer should set `user_version` to 2 (a v1 store is migrated on open instead), keep the single `state` row, and respect the frozen `accepted.identity` and `sync_ok` semantics above — those are the two rows where a wrong guess loses data (an erased server-side token set, or a contact's token silently changing). ## Modules diff --git a/core/src/client.rs b/core/src/client.rs index f8a18c8..7931026 100644 --- a/core/src/client.rs +++ b/core/src/client.rs @@ -213,6 +213,11 @@ pub fn trust_key( Some((old, _)) if ct_eq(&old, &identity) => Ok((identity, TrustChange::None)), Some((old, verified)) => { if walk_chain(&addr.user, &old, &identity, &chain) { + // The displaced key is the only local record that this + // contact rotated, so it is kept (sec 8) before the current + // one moves on. `until` is milliseconds, the unit the export + // format carries. + store.save_history(&addr.short(), &old, crate::store::now() * 1000)?; store.save_contact(&addr.short(), &identity, verified)?; Ok((identity, TrustChange::Rotated)) } else { @@ -667,7 +672,7 @@ pub fn fetch_with( // if we deleted it locally in the meantime (sec 10). if !store.seen(&mid)? { let requests_tier = flags & FLAG_REQUESTS != 0; - store.store_inbox(&mid, &envelope, received_at, requests_tier)?; + store.store_inbox(&mid, &envelope, received_at, requests_tier, opts.keep)?; learn_token(store, &opened)?; summary.stored += 1; } diff --git a/core/src/store.rs b/core/src/store.rs index 595e345..1b91a06 100644 --- a/core/src/store.rs +++ b/core/src/store.rs @@ -55,10 +55,17 @@ CREATE TABLE IF NOT EXISTS tokens ( CREATE TABLE IF NOT EXISTS seen (id BLOB PRIMARY KEY, at INTEGER NOT NULL); CREATE TABLE IF NOT EXISTS inbox ( id BLOB PRIMARY KEY, envelope BLOB NOT NULL, - received_at INTEGER NOT NULL, tier INTEGER NOT NULL); + received_at INTEGER NOT NULL, tier INTEGER NOT NULL, + kept INTEGER NOT NULL DEFAULT 0); -- a server copy still exists (fetch --keep) CREATE TABLE IF NOT EXISTS sent ( id BLOB PRIMARY KEY, recipient TEXT NOT NULL, envelope BLOB NOT NULL, sent_at INTEGER NOT NULL); +-- Keys a contact used before its current one, with when each stopped being +-- current, in epoch milliseconds: the local record that a rotation +-- happened, so a later import can restore it (sec 8). +CREATE TABLE IF NOT EXISTS history ( + address TEXT NOT NULL, identity BLOB NOT NULL, until INTEGER NOT NULL, + PRIMARY KEY (address, identity)); "; /// One sealed message in a folder, still encrypted. @@ -68,6 +75,8 @@ pub struct Stored { pub at: i64, /// Only sent copies carry a recipient. pub recipient: Option, + /// A server copy still exists, so a local delete must also remove it. + pub kept: bool, } pub struct Store { @@ -81,7 +90,17 @@ pub type ContactRow = (String, [u8; KEY_LEN], bool, Option); /// The store's schema version, in SQLite's `user_version`. A store written /// by a newer fumi is refused rather than misread; within a version the /// schema is stable, and a bump comes with a documented migration. -pub const SCHEMA_VERSION: i32 = 1; +pub const SCHEMA_VERSION: i32 = 2; + +/// The migration from schema version 1: both changes are additive, so an +/// existing store is upgraded in place at open; only stores newer than this +/// build are refused. +const MIGRATE_FROM_1: &str = " +ALTER TABLE inbox ADD COLUMN kept INTEGER NOT NULL DEFAULT 0; +CREATE TABLE IF NOT EXISTS history ( + address TEXT NOT NULL, identity BLOB NOT NULL, until INTEGER NOT NULL, + PRIMARY KEY (address, identity)); +"; impl Store { /// Opens the store at `path`. SQLite's own `:memory:` path works too; @@ -112,6 +131,9 @@ impl Store { if version == 0 { db.execute_batch(SCHEMA)?; db.pragma_update(None, "user_version", SCHEMA_VERSION)?; + } else if version == 1 { + db.execute_batch(MIGRATE_FROM_1)?; + db.pragma_update(None, "user_version", SCHEMA_VERSION)?; } else if version != SCHEMA_VERSION { return Err(Error::SchemaVersion { found: version, @@ -447,6 +469,7 @@ impl Store { envelope: &[u8], received_at: i64, requests_tier: bool, + kept: bool, ) -> Result<(), Error> { let tier = if requests_tier { TIER_REQUESTS @@ -455,8 +478,8 @@ impl Store { }; self.db() .execute( - "INSERT OR IGNORE INTO inbox (id, envelope, received_at, tier) VALUES (?1, ?2, ?3, ?4)", - params![id, envelope, received_at, tier], + "INSERT OR IGNORE INTO inbox (id, envelope, received_at, tier, kept) VALUES (?1, ?2, ?3, ?4, ?5)", + params![id, envelope, received_at, tier, kept as i64], ) .map(|_| ())?; self.db() @@ -468,6 +491,40 @@ impl Store { Ok(()) } + /// A key a contact no longer uses, with when it stopped being current + /// (epoch ms). Written when a rotation displaces the key we held. + pub fn save_history( + &self, + address: &str, + identity: &[u8; KEY_LEN], + until: i64, + ) -> Result<(), Error> { + self.db() + .execute( + "INSERT OR IGNORE INTO history (address, identity, until) VALUES (?1, ?2, ?3)", + params![address, identity, until], + ) + .map(|_| ())?; + Ok(()) + } + + /// The keys a contact used before its current one, oldest displacement + /// first: the local record that rotations happened (sec 8). + pub fn history(&self, address: &str) -> Result, Error> { + let db = self.db(); + let mut stmt = + db.prepare("SELECT identity, until FROM history WHERE address = ?1 ORDER BY until")?; + let rows = stmt + .query_map(params![address], |row| { + Ok((row.get::<_, Vec>(0)?, row.get::<_, i64>(1)?)) + })? + .collect::, _>>()?; + Ok(rows + .into_iter() + .filter_map(|(identity, until)| Some((identity.try_into().ok()?, until))) + .collect()) + } + pub fn store_sent( &self, id: &[u8; ID_LEN], @@ -489,20 +546,20 @@ impl Store { pub fn mail(&self, folder: &str) -> Result, Error> { let (sql, tier): (&str, Option) = match folder { "sent" => ( - "SELECT id, envelope, sent_at, recipient FROM sent ORDER BY sent_at, id", + "SELECT id, envelope, sent_at, recipient, 0 FROM sent ORDER BY sent_at, id", None, ), "all" => ( - "SELECT id, envelope, received_at, NULL FROM inbox ORDER BY received_at, id", + "SELECT id, envelope, received_at, NULL, kept FROM inbox ORDER BY received_at, id", None, ), "requests" => ( - "SELECT id, envelope, received_at, NULL FROM inbox \ + "SELECT id, envelope, received_at, NULL, kept FROM inbox \ WHERE tier = ?1 ORDER BY received_at, id", Some(TIER_REQUESTS as i8), ), _ => ( - "SELECT id, envelope, received_at, NULL FROM inbox \ + "SELECT id, envelope, received_at, NULL, kept FROM inbox \ WHERE tier = ?1 ORDER BY received_at, id", Some(TIER_MAIN as i8), ), @@ -527,6 +584,7 @@ fn row_stored(row: &rusqlite::Row<'_>) -> rusqlite::Result { envelope: row.get(1)?, at: row.get(2)?, recipient: row.get(3)?, + kept: row.get::<_, i64>(4)? != 0, }) } @@ -650,9 +708,9 @@ mod tests { fn inbox_and_sent_store_sealed_and_deduplicate() { let store = temp_store("mail"); let id = [3u8; 32]; - store.store_inbox(&id, b"envelope", 100, false).unwrap(); + store.store_inbox(&id, b"envelope", 100, false, false).unwrap(); // A replayed id is not re-stored (sec 10). - store.store_inbox(&id, b"envelope2", 100, true).unwrap(); + store.store_inbox(&id, b"envelope2", 100, true, false).unwrap(); assert!(store.seen(&id).unwrap()); let inbox = store.mail("inbox").unwrap(); assert_eq!(inbox.len(), 1); @@ -723,6 +781,87 @@ mod tests { } } + #[test] + fn v1_store_migrates_at_open() { + let path = std::env::temp_dir().join(format!( + "fumi-store-v1-{}-{}.db", + std::process::id(), + now() + )); + // A schema-version-1 store: the inbox has no `kept` column and the + // history table does not exist yet. + let db = Connection::open(&path).unwrap(); + db.execute_batch( + "CREATE TABLE state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + username TEXT, host TEXT, port INTEGER, scheme TEXT, + rotations INTEGER NOT NULL DEFAULT 0, + after_time INTEGER NOT NULL DEFAULT 0, + after_id BLOB NOT NULL DEFAULT x'', + sync_ok INTEGER NOT NULL DEFAULT 1); + INSERT INTO state (id) VALUES (1); + CREATE TABLE servers ( + host TEXT PRIMARY KEY, static BLOB NOT NULL, pinned_at INTEGER NOT NULL); + CREATE TABLE contacts ( + address TEXT PRIMARY KEY, identity BLOB NOT NULL, + verified INTEGER NOT NULL, seen_at INTEGER NOT NULL); + CREATE TABLE accepted ( + address TEXT PRIMARY KEY, identity BLOB NOT NULL, + active INTEGER NOT NULL, added_at INTEGER NOT NULL); + CREATE TABLE tokens ( + address TEXT PRIMARY KEY, token BLOB NOT NULL, seen_at INTEGER NOT NULL); + CREATE TABLE seen (id BLOB PRIMARY KEY, at INTEGER NOT NULL); + CREATE TABLE inbox ( + id BLOB PRIMARY KEY, envelope BLOB NOT NULL, + received_at INTEGER NOT NULL, tier INTEGER NOT NULL); + CREATE TABLE sent ( + id BLOB PRIMARY KEY, recipient TEXT NOT NULL, + envelope BLOB NOT NULL, sent_at INTEGER NOT NULL); + INSERT INTO inbox (id, envelope, received_at, tier) + VALUES (x'0101010101010101010101010101010101010101010101010101010101010101', + x'02', 7, 0); + PRAGMA user_version = 1;", + ) + .unwrap(); + drop(db); + + let store = Store::open(&path).expect("v1 store migrates"); + let db = Connection::open(&path).unwrap(); + let version: i32 = db.query_row("PRAGMA user_version", [], |row| row.get(0)).unwrap(); + drop(db); + assert_eq!(version, SCHEMA_VERSION); + // Existing rows survive with the column's default. + let mail = store.mail("all").unwrap(); + assert_eq!(mail.len(), 1); + assert!(!mail[0].kept); + // The history table exists and round-trips. + store.save_history("alice@example.org", &[9u8; 32], 42).unwrap(); + assert_eq!(store.history("alice@example.org").unwrap(), [([9u8; 32], 42)]); + } + + #[test] + fn kept_survives_the_folder_listing() { + let store = temp_store("kept"); + store.store_inbox(&[1u8; 32], b"a", 1, false, true).unwrap(); + store.store_inbox(&[2u8; 32], b"b", 2, false, false).unwrap(); + let all = store.mail("all").unwrap(); + assert!(all[0].kept && !all[1].kept); + } + + #[test] + fn history_orders_by_displacement() { + let store = temp_store("history"); + store.save_history("a@example.org", &[1u8; 32], 20).unwrap(); + store.save_history("a@example.org", &[2u8; 32], 10).unwrap(); + // A re-save of the same key is a no-op, like a non-rotation. + store.save_history("a@example.org", &[2u8; 32], 30).unwrap(); + assert_eq!( + store.history("a@example.org").unwrap(), + [([2u8; 32], 10), ([1u8; 32], 20)] + ); + assert!(store.history("b@example.org").unwrap().is_empty()); + } + #[test] fn in_memory_store_round_trips() { let store = Store::open_in_memory().expect("memory store opens"); @@ -732,7 +871,7 @@ mod tests { store.account().unwrap().unwrap().short(), "alice@example.org" ); - store.store_inbox(&[1u8; 32], b"envelope", 5, false).unwrap(); + store.store_inbox(&[1u8; 32], b"envelope", 5, false, false).unwrap(); assert_eq!(store.mail("inbox").unwrap().len(), 1); // The same handle through the path spelling. let path = std::path::Path::new(":memory:");