diff --git a/README.md b/README.md index 0442195..c8f7322 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ fumi send
[--subject S] [--body TEXT | --file F | -] [--header K:V] [- fumi fetch [--keep] [--reset] retrieve, verify, store, acknowledge fumi delete ... remove from the server explicitly fumi list [--sent] [--requests] + +# backups +fumi export [FILE] write a gsmol backup: sealed mail, contacts, pins — never the master +fumi import-backup restore one; merges, never overwrites a trust binding fumi read [--sent] # RNS carrier (built with --features rns / nix build .#rns); addresses select @@ -116,6 +120,7 @@ An importer should set `user_version` to 2 (a v1 store is migrated on open inste | `core/src/rns/` | the Reticulum carrier behind the `rns` feature: FFI over `core/shim/`, path discovery, one link per session | | `core/src/client.rs` | the six operations, AUTH, chain walking, token sync, fetch pipeline with cancellation; store-taking items behind the `store` feature | | `core/src/store.rs` | SQLite behind the `store` feature: state, contacts, accepted, tokens, seen ids, inbox, sent; `Send + Sync`, WAL, versioned schema | +| `core/src/export.rs` | the gsmol backup format behind the `store` feature: sealed payload of mail, contacts and pins, interchangeable with gsmol exports | | `core/src/error.rs` | the `Error` enum embedders match on, the status-code mapping | | `core/vectors.json` | the committed reference vectors, the tests' source of truth | | `cli/src/main.rs` | the command line: keyfile and `--db` conventions, hints, output | diff --git a/cli/src/main.rs b/cli/src/main.rs index fe3c456..807c8eb 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -86,6 +86,14 @@ enum Command { Import { uri: String }, /// List known keys and how each was learned Contacts, + /// Write a gsmol backup: sealed mail, contacts and pins, never the master + Export { + /// Output file + #[arg(default_value = "smol-backup.json")] + file: PathBuf, + }, + /// Restore a gsmol backup file; merges, never overwrites a trust binding + ImportBackup { file: PathBuf }, /// Issue an accept token, admit to the main tier, sync the set Accept { address: String }, /// Withdraw a contact's accept token, sync the set @@ -192,6 +200,8 @@ fn run(cli: Cli) -> Result<()> { Command::Resolve { address } => resolve(&store, address, cli.timeout), Command::Import { uri } => import(&store, uri), Command::Contacts => contacts(&store), + Command::Export { file } => export_backup(&cli.key, &store, file), + Command::ImportBackup { file } => import_backup(&cli.key, &store, file), Command::Accept { address } => accept(&cli.key, &store, address, cli.timeout), Command::Block { address } => block(&cli.key, &store, address, cli.timeout), Command::Send { .. } => send(&cli, &store), @@ -448,6 +458,23 @@ fn contacts(store: &Store) -> Result<()> { Ok(()) } +fn export_backup(key: &PathBuf, store: &Store, file: &PathBuf) -> Result<()> { + let master = load_master(key)?; + let text = fumi::export::export(store, &master)?; + std::fs::write(file, text)?; + println!("wrote {} (only the master can open it)", file.display()); + Ok(()) +} + +fn import_backup(key: &PathBuf, store: &Store, file: &PathBuf) -> Result<()> { + let master = load_master(key)?; + let text = std::fs::read_to_string(file) + .with_context(|| format!("reading {}", file.display()))?; + let summary = fumi::export::import(store, &master, &text)?; + println!("{summary}"); + Ok(()) +} + /// An address plus the key we hold for it, for commands naming a contact. fn target_contact(store: &Store, text: &str) -> Result<(String, [u8; KEY_LEN])> { let addr = Address::parse(text)?; diff --git a/core/Cargo.toml b/core/Cargo.toml index d025ca5..0b7f1bf 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -11,7 +11,7 @@ default = ["store", "bundled-sqlite"] # one. Off, the crate keeps identities, addresses, seal/open and the raw # transport operations for hosts with their own database and thin FFI # consumers that need only the crypto and envelope layer. -store = ["dep:rusqlite"] +store = ["dep:rusqlite", "dep:serde", "dep:serde_json"] # Bundle libsqlite3 so the store needs no system SQLite; embedders with their # own SQLite (Android's NDK case) build with default-features = false. bundled-sqlite = ["rusqlite/bundled"] @@ -29,6 +29,8 @@ hmac = "0.12" hkdf = "0.12" rand_core = { version = "0.6", features = ["getrandom"] } rusqlite = { version = "0.32", optional = true } +serde = { version = "1", features = ["derive"], optional = true } +serde_json = { version = "1", optional = true } data-encoding = "2" [dev-dependencies] diff --git a/core/src/export.rs b/core/src/export.rs new file mode 100644 index 0000000..645acac --- /dev/null +++ b/core/src/export.rs @@ -0,0 +1,448 @@ +//! The gsmol export format: a sealed bundle of mail, contacts and pins a +//! gsmol or fumi client can restore. The payload is sealed to a key derived +//! from the master, so a backup file is only readable where the identity +//! already lives; the master itself is never in the file. Field names, +//! encodings and units match gsmol's `store.js` exactly, so exports are +//! interchangeable between the clients. + +use chacha20poly1305::aead::{Aead, KeyInit, Payload}; +use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce}; +use rand_core::{OsRng, RngCore}; +use serde::{Deserialize, Serialize}; + +use crate::crypto::{b32, hkdf_sha256, unb32}; +use crate::error::Error; +use crate::store::Store; +use crate::transport::{ID_LEN, KEY_LEN, TIER_MAIN, TIER_REQUESTS}; + +/// The HKDF label gsmol wrote originally; the format's version lives in the +/// container's `gsmolExport` field, not here. +const EXPORT_LABEL: &[u8] = b"gsmol/1 export"; + +fn export_key(master: &[u8; KEY_LEN]) -> [u8; KEY_LEN] { + hkdf_sha256(master, b"", EXPORT_LABEL, KEY_LEN) + .try_into() + .unwrap() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn unhex(text: &str) -> Option> { + if text.len() % 2 != 0 { + return None; + } + (0..text.len() / 2) + .map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).ok()) + .collect() +} + +fn seal(key: &[u8; KEY_LEN], plaintext: &[u8]) -> ([u8; 12], Vec) { + // The key is the same on every export, so a fresh nonce per seal is what + // rules out reuse — the one place a random nonce is mandatory. + let mut nonce = [0u8; 12]; + OsRng.fill_bytes(&mut nonce); + let cipher = ChaCha20Poly1305::new(Key::from_slice(key)); + let sealed = cipher + .encrypt( + &Nonce::from(nonce), + Payload { + msg: plaintext, + aad: b"", + }, + ) + .expect("encryption failed"); + (nonce, sealed) +} + +fn open(key: &[u8; KEY_LEN], nonce: &[u8], sealed: &[u8]) -> Option> { + if nonce.len() != 12 { + return None; + } + ChaCha20Poly1305::new(Key::from_slice(key)) + .decrypt( + Nonce::from_slice(nonce), + Payload { + msg: sealed, + aad: b"", + }, + ) + .ok() +} + +#[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct HistoryEntry { + key: String, + /// When the key stopped being current, in epoch milliseconds. + until: i64, +} + +#[derive(Serialize, Deserialize)] +struct ContactExport { + key: String, + verified: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + history: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct InboxRow { + id: String, + received_at: i64, + envelope: String, + tier: u8, + kept_on_server: bool, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SentRow { + id: String, + recipient: String, + sent_at: i64, + envelope: String, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Bundle { + servers: std::collections::BTreeMap, + contacts: std::collections::BTreeMap, + inbox: Vec, + sent: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Container { + gsmol_export: u8, + exported_at: i64, + nonce: String, + ciphertext: String, +} + +/// What an import did, in the shape gsmol's importer reports. A conflicting +/// pin or contact is counted, not applied: an existing trust binding changes +/// only by explicit user action, never silently. +#[derive(Debug, Default)] +pub struct ImportSummary { + pub pins_added: usize, + pub pins_conflicted: usize, + pub contacts_added: usize, + pub contacts_conflicted: usize, + pub mail_added: usize, + pub malformed: usize, +} + +impl std::fmt::Display for ImportSummary { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} messages, {} contacts ({} conflicted), {} server keys ({} conflicted), {} malformed", + self.mail_added, + self.contacts_added, + self.contacts_conflicted, + self.pins_added, + self.pins_conflicted, + self.malformed + ) + } +} + +/// Exports the store as a gsmol v2 container: pins, contacts with their key +/// history, and sealed mail, ChaCha20-Poly1305-sealed to the master-derived +/// key. Mail stays sealed; the master never enters the file. +pub fn export(store: &Store, master: &[u8; KEY_LEN]) -> Result { + let mut servers = std::collections::BTreeMap::new(); + for (host, key) in store.pins()? { + servers.insert(host, b32(&key)); + } + + let mut contacts = std::collections::BTreeMap::new(); + for (address, key, verified, _) in store.contact_rows()? { + let history = store + .history(&address)? + .into_iter() + .map(|(key, until)| HistoryEntry { + key: b32(&key), + until, + }) + .collect(); + contacts.insert(address, ContactExport { key: b32(&key), verified, history }); + } + + // Each folder listing carries its own tier, so rows round-trip into the + // tier they were fetched into. + let inbox = store + .mail("inbox")? + .into_iter() + .map(|row| inbox_row(row, TIER_MAIN)) + .chain(store.mail("requests")?.into_iter().map(|row| inbox_row(row, TIER_REQUESTS))) + .collect::>(); + let sent = store + .mail("sent")? + .into_iter() + .map(|row| SentRow { + id: hex(&row.id), + recipient: row.recipient.unwrap_or_default(), + sent_at: row.at, + envelope: data_encoding::BASE64.encode(&row.envelope), + }) + .collect(); + + let payload = serde_json::to_vec(&Bundle { + servers, + contacts, + inbox, + sent, + }) + .map_err(|e| Error::Other(format!("export serialization failed: {e}")))?; + let (nonce, sealed) = seal(&export_key(master), &payload); + let container = Container { + gsmol_export: 2, + // Milliseconds, the unit gsmol stamps the container with; the mail + // and history rows inside keep their own units. + exported_at: crate::store::now() * 1000, + nonce: data_encoding::BASE64.encode(&nonce), + ciphertext: data_encoding::BASE64.encode(&sealed), + }; + serde_json::to_string_pretty(&container) + .map_err(|e| Error::Other(format!("export serialization failed: {e}"))) +} + +fn inbox_row(row: crate::store::Stored, tier: u8) -> InboxRow { + InboxRow { + id: hex(&row.id), + received_at: row.at, + envelope: data_encoding::BASE64.encode(&row.envelope), + tier, + kept_on_server: row.kept, + } +} + +/// Imports a gsmol export. Mail is stored sealed and marked seen, so a later +/// fetch does not re-store it; pins and contacts merge without clobbering; +/// one malformed record is skipped and counted, never fatal. +pub fn import(store: &Store, master: &[u8; KEY_LEN], text: &str) -> Result { + let container: Container = serde_json::from_str(text) + .map_err(|_| Error::Other("not a gsmol export file".into()))?; + if container.gsmol_export != 2 { + return Err(Error::Other("not a gsmol export file".into())); + } + let nonce = data_encoding::BASE64 + .decode(container.nonce.as_bytes()) + .map_err(|_| Error::Other("not a gsmol export file".into()))?; + let sealed = data_encoding::BASE64 + .decode(container.ciphertext.as_bytes()) + .map_err(|_| Error::Other("not a gsmol export file".into()))?; + let plaintext = open(&export_key(master), &nonce, &sealed) + .ok_or_else(|| { + Error::Other("couldn't decrypt — exported by a different identity, or the file is corrupted".into()) + })?; + let payload: Bundle = serde_json::from_slice(&plaintext) + .map_err(|_| Error::Other("not a gsmol export file".into()))?; + + let mut summary = ImportSummary::default(); + + for (host, key) in &payload.servers { + let key: Option<[u8; KEY_LEN]> = unb32(key).ok().and_then(|k| k.try_into().ok()); + match key { + None => summary.malformed += 1, + Some(key) => match store.server_pin(host)? { + None => { + store.pin_server(host, &key)?; + summary.pins_added += 1; + } + Some(existing) if existing != key => summary.pins_conflicted += 1, + Some(_) => {} + }, + } + } + + for (address, contact) in &payload.contacts { + let key: Option<[u8; KEY_LEN]> = + unb32(&contact.key).ok().and_then(|k| k.try_into().ok()); + match key { + None => summary.malformed += 1, + Some(key) => match store.contact(address)? { + None => { + store.save_contact(address, &key, contact.verified)?; + summary.contacts_added += 1; + } + Some((existing, _)) if existing != key => summary.contacts_conflicted += 1, + Some(_) => {} + }, + } + // History is a record, not a trust binding: entries are additive and + // never clobber anything, so they merge in both directions. + for entry in &contact.history { + if let Some(old) = unb32(&entry.key).ok().and_then(|k| k.try_into().ok()) { + store.save_history(address, &old, entry.until)?; + } else { + summary.malformed += 1; + } + } + } + + for row in &payload.inbox { + let id: Option<[u8; ID_LEN]> = unhex(&row.id).and_then(|k| k.try_into().ok()); + let envelope = data_encoding::BASE64 + .decode(row.envelope.as_bytes()) + .ok() + .filter(|e| !e.is_empty()); + match (id, envelope) { + (Some(id), Some(envelope)) if !store.stored("inbox", &id)? => { + // store_inbox marks the message seen, so a later fetch of a + // still-kept server copy cannot store it twice. + store.store_inbox( + &id, + &envelope, + row.received_at, + row.tier == TIER_REQUESTS, + row.kept_on_server, + )?; + summary.mail_added += 1; + } + (Some(_), Some(_)) => {} + _ => summary.malformed += 1, + } + } + + for row in &payload.sent { + let id: Option<[u8; ID_LEN]> = unhex(&row.id).and_then(|k| k.try_into().ok()); + let envelope = data_encoding::BASE64 + .decode(row.envelope.as_bytes()) + .ok() + .filter(|e| !e.is_empty()); + match (id, envelope) { + (Some(id), Some(envelope)) if !store.stored("sent", &id)? => { + store.store_sent(&id, &row.recipient, &envelope, row.sent_at)?; + summary.mail_added += 1; + } + (Some(_), Some(_)) => {} + _ => summary.malformed += 1, + } + } + + Ok(summary) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn seeded_store() -> (Store, [u8; KEY_LEN]) { + let store = Store::open_in_memory().unwrap(); + let master = [7u8; KEY_LEN]; + store.pin_server("example.org", &[1u8; KEY_LEN]).unwrap(); + store + .save_contact("alice@example.org", &[2u8; KEY_LEN], true) + .unwrap(); + store + .save_history("alice@example.org", &[9u8; KEY_LEN], 500) + .unwrap(); + store + .store_inbox(&[3u8; ID_LEN], b"envelope", 42, false, true) + .unwrap(); + store + .store_inbox(&[4u8; ID_LEN], b"request", 43, true, false) + .unwrap(); + store.store_sent(&[5u8; ID_LEN], "bob@example.org", b"sent", 44).unwrap(); + (store, master) + } + + #[test] + fn export_import_round_trip() { + let (store, master) = seeded_store(); + let file = export(&store, &master).unwrap(); + + let fresh = Store::open_in_memory().unwrap(); + let summary = import(&fresh, &master, &file).unwrap(); + assert_eq!(summary.mail_added, 3); + assert_eq!(summary.pins_added, 1); + assert_eq!(summary.contacts_added, 1); + assert_eq!(summary.malformed, 0); + + assert_eq!(fresh.server_pin("example.org").unwrap(), Some([1u8; KEY_LEN])); + let (key, verified) = fresh.contact("alice@example.org").unwrap().unwrap(); + assert_eq!(key, [2u8; KEY_LEN]); + assert!(verified); + assert_eq!(fresh.history("alice@example.org").unwrap(), [([9u8; KEY_LEN], 500)]); + assert_eq!(fresh.mail("inbox").unwrap().len(), 1); + assert_eq!(fresh.mail("requests").unwrap().len(), 1); + assert_eq!(fresh.mail("inbox").unwrap()[0].kept, true); + assert_eq!(fresh.mail("sent").unwrap().len(), 1); + // Importing marks mail seen, so a later fetch cannot re-store it. + assert!(fresh.seen(&[3u8; ID_LEN]).unwrap()); + } + + #[test] + fn a_different_identity_cannot_open_it() { + let (store, master) = seeded_store(); + let file = export(&store, &master).unwrap(); + let fresh = Store::open_in_memory().unwrap(); + let err = import(&fresh, &[8u8; KEY_LEN], &file).unwrap_err(); + assert!(err.to_string().contains("couldn't decrypt")); + assert_eq!(fresh.mail("all").unwrap().len(), 0); + } + + #[test] + fn import_never_clobbers_a_trust_binding() { + let (store, master) = seeded_store(); + let file = export(&store, &master).unwrap(); + + let mine = Store::open_in_memory().unwrap(); + mine.pin_server("example.org", &[6u8; KEY_LEN]).unwrap(); + mine.save_contact("alice@example.org", &[7u8; KEY_LEN], false) + .unwrap(); + let summary = import(&mine, &master, &file).unwrap(); + assert_eq!(summary.pins_conflicted, 1); + assert_eq!(summary.contacts_conflicted, 1); + assert_eq!(mine.server_pin("example.org").unwrap(), Some([6u8; KEY_LEN])); + let (key, _) = mine.contact("alice@example.org").unwrap().unwrap(); + assert_eq!(key, [7u8; KEY_LEN]); + } + + #[test] + fn malformed_records_are_skipped_not_fatal() { + let (store, master) = seeded_store(); + let file = export(&store, &master).unwrap(); + // Corrupt one inbox row's id inside the payload by re-sealing a + // hand-edited payload with the same key. + let container: Container = serde_json::from_str(&file).unwrap(); + let nonce = data_encoding::BASE64.decode(container.nonce.as_bytes()).unwrap(); + let sealed = data_encoding::BASE64.decode(container.ciphertext.as_bytes()).unwrap(); + let plaintext = open(&export_key(&master), &nonce, &sealed).unwrap(); + let mut payload: Bundle = serde_json::from_slice(&plaintext).unwrap(); + payload.inbox[0].id = "zz".to_string(); + payload.servers.insert("bad".to_string(), "not-base32!".to_string()); + let (nonce, sealed) = seal(&export_key(&master), &serde_json::to_vec(&payload).unwrap()); + let file = serde_json::to_string(&Container { + gsmol_export: 2, + exported_at: 0, + nonce: data_encoding::BASE64.encode(&nonce), + ciphertext: data_encoding::BASE64.encode(&sealed), + }) + .unwrap(); + + let fresh = Store::open_in_memory().unwrap(); + let summary = import(&fresh, &master, &file).unwrap(); + assert_eq!(summary.malformed, 2); + assert_eq!(summary.mail_added, 2); + assert_eq!(summary.pins_added, 1); + } + + #[test] + fn not_a_gsmol_file_is_refused() { + let fresh = Store::open_in_memory().unwrap(); + assert!(import(&fresh, &[7u8; KEY_LEN], "{}").is_err()); + assert!(import(&fresh, &[7u8; KEY_LEN], "[]").is_err()); + // A v1 file is the pre-encryption shape gsmol still accepts; this + // importer takes the sealed v2 container only. + let v1 = r#"{"gsmolExport": 1, "servers": {}}"#; + assert!(import(&fresh, &[7u8; KEY_LEN], v1).is_err()); + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index cac0109..0e86989 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -11,6 +11,10 @@ pub mod address; pub mod client; pub mod crypto; pub mod error; +// The gsmol export format is sealed mail plus the store's trust rows, so it +// exists only alongside the store. +#[cfg(feature = "store")] +pub mod export; pub mod message; #[cfg(feature = "rns")] pub mod rns; diff --git a/core/src/store.rs b/core/src/store.rs index 1b91a06..1deada1 100644 --- a/core/src/store.rs +++ b/core/src/store.rs @@ -283,6 +283,21 @@ impl Store { .and_then(|k| k.try_into().ok())) } + /// Every pin, host and key, ordered by host — the export's server map. + pub fn pins(&self) -> Result, Error> { + let db = self.db(); + let mut stmt = db.prepare("SELECT host, static FROM servers ORDER BY host")?; + let rows = stmt + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Vec>(1)?)) + })? + .collect::, _>>()?; + Ok(rows + .into_iter() + .filter_map(|(host, key)| Some((host, key.try_into().ok()?))) + .collect()) + } + pub fn pin_server(&self, host: &str, key: &[u8; KEY_LEN]) -> Result<(), Error> { self.db() .execute( @@ -541,6 +556,21 @@ impl Store { Ok(()) } + /// Whether a message id is already stored in `folder` ("inbox" or "sent"), + /// so an import can skip rather than re-store. + pub fn stored(&self, folder: &str, id: &[u8; ID_LEN]) -> Result { + let table = match folder { + "sent" => "sent", + _ => "inbox", + }; + Ok(self + .one::( + &format!("SELECT 1 FROM {table} WHERE id = ?1"), + &[id], + )? + .is_some()) + } + /// One folder, ordered by arrival or sending time. `folder` is "inbox", /// "requests", "sent" or "all". pub fn mail(&self, folder: &str) -> Result, Error> { @@ -597,13 +627,16 @@ mod tests { core::array::from_fn(|i| i as u8) } + /// A fresh scratch path per call: a monotonic counter, so two tests or + /// two runs cannot collide the way pid-and-second can. + fn scratch_path(tag: &str) -> std::path::PathBuf { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + std::env::temp_dir().join(format!("fumi-store-{tag}-{n}-{}.db", std::process::id())) + } + fn temp_store(tag: &str) -> Store { - let path = std::env::temp_dir().join(format!( - "fumi-store-{tag}-{}-{}.db", - std::process::id(), - now() - )); - Store::open(&path).expect("open store") + Store::open(&scratch_path(tag)).expect("open store") } #[test] @@ -761,11 +794,7 @@ mod tests { #[test] fn schema_is_versioned_and_refuses_a_newer_store() { - let path = std::env::temp_dir().join(format!( - "fumi-store-schema-{}-{}.db", - std::process::id(), - now() - )); + let path = scratch_path("schema"); drop(Store::open(&path).expect("open store")); let db = Connection::open(&path).unwrap(); let version: i32 = db.query_row("PRAGMA user_version", [], |row| row.get(0)).unwrap(); @@ -783,11 +812,7 @@ mod tests { #[test] fn v1_store_migrates_at_open() { - let path = std::env::temp_dir().join(format!( - "fumi-store-v1-{}-{}.db", - std::process::id(), - now() - )); + let path = scratch_path("v1"); // A schema-version-1 store: the inbox has no `kept` column and the // history table does not exist yet. let db = Connection::open(&path).unwrap();