From 71f5f628be773eaae6545f821993b52c831aafd7 Mon Sep 17 00:00:00 2001 From: randogoth Date: Sun, 27 Sep 2026 08:27:08 +0300 Subject: [PATCH] feat: revise to protocol 1.1 with accept tokens, fetch cursors and 32-byte ids --- README.md | 28 ++- SPEC.md | 166 ++++++++++++------ smolmail.py | 477 +++++++++++++++++++++++++++++++++++++++------------ smolmaild.py | 261 ++++++++++++++++++++++------ 4 files changed, 712 insertions(+), 220 deletions(-) diff --git a/README.md b/README.md index 4b9a7c7..b509162 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A minimalist, decentralized, end-to-end encrypted mail protocol. Email should be simple again. Inspired by [Spartan](spartan://spartan.mozz.us/), [Misfin](gemini://misfin.org/), and [LXMF](https://github.com/markqvist/lxmf). Smol Mail reduces electronic correspondence to its essentials: an address, a key and a message. One keypair per user, five operations, one binary message format. No sprawling infrastructure, no proprietary accounts, no advertising, no tracking. Anyone can run a server, anyone can write a client, and only the intended recipient can read a message. -A server learns which mailbox a message is for, how big it is and when it arrived. Nothing else — not the sender, not the subject, not the content. +A server learns which mailbox a message is for, how big it is and when it arrived. Not the sender, not the subject, not the content. - **Minimal** — five operations, four primitives, no extensibility mechanisms. - **Private** — messages are sealed and signed; senders are absent from the wire format. @@ -13,10 +13,12 @@ A server learns which mailbox a message is for, how big it is and when it arrive ## Identity -Identity is one Ed25519 keypair. The 32-byte public key *is* the identity; the 32-byte seed is the only secret to back up. Encryption keys are derived from the same keypair, so there is exactly one thing to hold, move between servers, print on paper or scan from a screen. +The only secret is a 32-byte master. Back that up and nothing else: the Ed25519 signing key, the encryption key derived from it, and the key that issues accept tokens all come out of it. One thing to hold, move between servers, print on paper or scan from a screen. Signatures prove authorship. A throwaway key per message means a stolen identity key cannot decrypt anything already sent. +Rotating a key advances an index rather than inventing an unrelated key, so the master alone can re-derive every key you have ever used — which is what makes mail addressed to a superseded key readable years later, with nothing archived. `restore` rebuilds a client from the master and a username. + ## Addressing ``` @@ -26,10 +28,18 @@ smol://alice@example.org/mfrggzdfmztwq2lknnwg23tpo… self-certifying, carries The short form is typeable. The long form carries the key itself, so an address shared by QR code, contact file or link needs no trust in any server at all. Either way, changing servers never changes an identity. -Keys learned from a server are pinned on first use. Later changes need a rotation certificate signed by the previous key, or explicit confirmation. +Keys learned from a server are pinned on first use. Later changes need a rotation certificate signed by both the old and the new key, or explicit confirmation. Recipients learn only the sender's key, never a name: a first contact arrives as a bare fingerprint until its address is known. A client may carry its full `smol://` address in a signed `Reply-To` field so first contacts can be answered; receivers bind it only when its key matches the message's signer (SPEC.md §5.7). +## A mailbox nobody else can fill + +A server cannot see senders, so it cannot tell wanted mail from a flood, and without help anyone who knows an address could fill a mailbox to its quota. + +An **accept token** fixes that without telling the server anything. It is a 32-byte secret you derive per correspondent, hand to them inside a sealed reply, and upload to your own server. Their messages carry a MAC over it; your server matches the MAC and puts those messages in your mailbox proper. Everything else — a first contact, a stranger, a flood — goes to a small **requests** tier with a short retention, which your client shows separately. Accepting someone is one command, or just replying to them; withdrawing the token puts them back in requests. + +Your server learns from this how many correspondents you have accepted and which token a message matched, so a token is a stable pseudonym. It still never learns who is behind one (SPEC.md §5.8, §9). + ## Cryptography Ed25519 · X25519 · ChaCha20-Poly1305 · SHA-256. Four established primitives, no novel cryptography, nothing else anywhere in the protocol. Transport is TCP with a Noise handshake — no certificates, no CA, no expiry. @@ -40,7 +50,7 @@ Traffic analysis, delivery timing, mailbox size, and whether a user has an accou There is no forward secrecy on the recipient side: a seized recipient key decrypts ciphertext recorded while it was valid. Messages are signed, so they carry non-repudiation rather than deniability. -Version 1 excludes attachments, group messaging, federation, anonymous routing, multi-device synchronisation and key revocation. +Version 1.1 excludes attachments, group messaging, federation, anonymous routing, multi-device synchronisation and key revocation. ## Reference server @@ -53,7 +63,7 @@ uv run smolmaild.py serve `keygen` writes the server's static key to `server.key` and prints its public key in base32. Publish that public key through a trusted channel — clients pin it, and a mismatch aborts the handshake. -`serve` listens on `127.0.0.1:1961` by default and stores mail in `mail.db`. Use `--host 0.0.0.0` to accept remote connections, and `--invite-token` to close registration. `--help` lists the size, quota, retention and rate limits. +`serve` listens on `127.0.0.1:1961` by default and stores mail in `mail.db`. Use `--host 0.0.0.0` to accept remote connections, and `--invite-token` to close registration. `--help` lists the size, quota, retention and rate limits, including the separate `--requests-quota` for mail that arrives without an accept token. ## Reference client @@ -72,10 +82,12 @@ uv run smolmail.py read Sent mail carries the sender's full `smol://` address in a signed `Reply-To` field (SPEC.md §5.7), so a first-time recipient can answer without an out-of-band channel; pass `--anonymous` to omit it. -`import` adds a contact from a `smol://` address without trusting any server, `contacts` lists known keys and how each was learned, and `rotate` replaces the identity with a signed certificate your contacts accept automatically. +`accept` admits a contact to your mailbox proper and pushes the token to your server at once; `block` withdraws it; `list --requests` shows what arrived without one. `import` adds a contact from a `smol://` address without trusting any server, `contacts` lists known keys and how each was learned, and `rotate` advances the identity with a certificate signed by both keys, which your contacts accept automatically. -Mail is stored sealed and opened on demand, so the local database holds no plaintext. Superseded keys are retained after a rotation, because mail already addressed to them is readable with nothing else. +`fetch` deletes what it has verified and stored. `fetch --keep` leaves mail on the server and remembers where it stopped, and `--reset` pages through it again. + +Mail is stored sealed and opened on demand, so the local database holds no plaintext. ## Specification -[SPEC.md](SPEC.md) — wire format, operations, trust model and conformance. +[SPEC.md](SPEC.md) — wire format, operations and trust model. diff --git a/SPEC.md b/SPEC.md index aa9dc48..3678668 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,4 +1,4 @@ -# Smol Mail Protocol, version 1 +# Smol Mail Protocol, version 1.1 A minimalist, decentralized, end-to-end encrypted mail protocol. @@ -9,18 +9,31 @@ Electronic correspondence reduced to an address, a key and a message. One keypai - **Decentralized** — independent servers, no federation, no directories. - **Portable** — identity is a keypair, not an account. +Version 1.1 is not wire-compatible with version 1.0 and does not negotiate: identifiers, certificates and three operation bodies changed shape. + ## 1. Primitives -Ed25519 (RFC 8032) · X25519 (RFC 7748) · ChaCha20-Poly1305 (RFC 8439) · SHA-256 / HKDF-SHA256 (RFC 5869). No other cryptographic primitive appears anywhere in this protocol. +Ed25519 (RFC 8032) · X25519 (RFC 7748) · ChaCha20-Poly1305 (RFC 8439) · SHA-256, with HMAC-SHA256 (RFC 2104) and HKDF-SHA256 (RFC 5869). No other cryptographic primitive appears anywhere in this protocol; the Noise pattern of §4 is a construction over these four, not a fifth. -Integers are big-endian. Text is UTF-8. Every hash and signature is domain-separated by a label from §11. +Integers are big-endian. Text is UTF-8. Every hash, signature and MAC is domain-separated by a label from §11. ## 2. Identity -Identity is one **Ed25519 keypair**. The 32-byte public key *is* the identity; the 32-byte seed is the only secret a user must back up. X25519 keys for key agreement are derived from the same keypair: +The only secret a user holds is a **32-byte master**. It is the only thing to back up, and it never changes. Everything else is derived from it: ``` -x_priv = clamp(SHA-512(seed)[0..32]) # [0] &= 248; [31] &= 127; [31] |= 64 +seed_n = HKDF-SHA256(master, salt = "", info = "smolmail/1 identity" || n, len = 32) # n: u32 +accept = HKDF-SHA256(master, salt = "", info = "smolmail/1 accept", len = 32) +``` + +`n` is the **rotation index**, counting from 0. `seed_n` is an Ed25519 seed, and its 32-byte public key *is* the identity at that index. Rotation (§7) advances `n`; the accept key of §5.8 does not depend on it, so it survives every rotation. HKDF is one-way, so a compromised `seed_n` does not expose the master or any other index. + +A client restoring from the master alone recovers the rest: it resolves its own username, derives `seed_0`, `seed_1`, … and stops at the index whose public key the server has bound, within the chain limit of §7. Every superseded signing key is therefore re-derivable and none has to be archived. + +X25519 keys for key agreement come from the same keypair: + +``` +x_priv = clamp(SHA-512(seed_n)[0..32]) # [0] &= 248; [31] &= 127; [31] |= 64 x_pub = (1 + y) / (1 - y) mod 2^255-19 # y from the Edwards point; sign bit discarded ``` @@ -35,11 +48,11 @@ alice@example.org[:1961] short form, resolved via t smol://alice@example.org/mfrggzdfmztwq2lknnwg23tpo… self-certifying, carries its own key ``` -Usernames are 1–63 bytes of `[a-z0-9._-]`, must not begin or end with a separator, and are compared case-insensitively; clients normalise to lowercase before sending. The default TCP port is **1961**. +Usernames are 1–63 bytes of `[a-z0-9._-]`. The separators are `.`, `_` and `-`: a username MUST begin and end with `[a-z0-9]` and MUST NOT contain two adjacent separators, so `a.b_c` is valid while `.a`, `a-` and `a..b` are not. Usernames are compared case-insensitively; clients normalise to lowercase before sending. The default TCP port is **1961**. The `smol://` path component is the 32-byte identity key in RFC 4648 base32, lowercase and unpadded (52 characters). Because it carries the key, an address shared this way requires no trust in any server; it is the form used for QR codes, contact files and links. -Fingerprints for spoken or visual verification are the first 20 characters of that base32 string in groups of four — a truncation of the identity, not a separate encoding. +Fingerprints for spoken or visual verification are the first 20 characters of that base32 string in groups of four — a truncation of the identity, not a separate encoding. Twenty characters carry 100 bits, so producing a second key with a given fingerprint costs about 2^100 trials. ## 4. Transport @@ -58,11 +71,14 @@ A client MUST NOT `REGISTER`, `FETCH` or `DELETE` against a server whose static ``` username_len u8 || username || identity 32 || signature 64 + || sync u8 || count u16 || token 32 × count signature = Ed25519 over "smolmail/1 auth" || h # h = Noise handshake hash ``` The server verifies the signature and that `identity` is the key bound to `username`. Because `h` incorporates the server's fresh ephemeral, this requires no challenge round trip and cannot be replayed across sessions or servers. +The trailing block carries the mailbox's accept tokens (§5.8). `sync = 0` leaves the stored set untouched and `count` MUST then be 0; `sync = 1` replaces the stored set with exactly the tokens that follow, which is how a token is both added and removed. A client MUST send `sync = 0` unless it knows its set is complete — in particular a client restored from the master alone, whose empty set would otherwise erase the server's. A set larger than the server's limit is refused with status 7 and leaves the session unauthenticated. + ## 5. Message format ### 5.1 Envelope — cleartext, what the server stores @@ -78,6 +94,8 @@ tag 16 Poly1305 ChaCha20-Poly1305 with `nonce = 0^12` and `aad = magic || version || to || epk`. The all-zero nonce is correct because the key is used exactly once (§5.2); there is no nonce field, so nonce reuse is not a failure mode an implementation can have. The envelope is also the on-disk and export format, hence the magic and version. +**Sizes.** An envelope is 69 bytes of header plus ciphertext plus a 16-byte tag, and is at most 768 KiB (§10). The payload it seals is 109 bytes plus the body. A message identifier is 32 bytes, a rotation certificate 200, an accept token 32 and its MAC 32. + ### 5.2 Sealing The sender generates a throwaway X25519 keypair per message: @@ -94,7 +112,9 @@ The recipient recovers the same key with `X25519(own_x_priv, epk)`. The sender's long-term key signs only and never participates in key agreement. Therefore the sealing key is unique per message; compromising a sender's identity key reveals nothing about messages already sent; and the sender does not appear in the envelope, so a server cannot observe who corresponds with whom. -A seized *recipient* key still decrypts ciphertext stored while it was valid. Closing that requires a ratchet and per-peer session state, which §13 places out of scope. +A seized *recipient* key still decrypts ciphertext stored while it was valid. + +An envelope has exactly one recipient. Addressing one message to several people means sealing one envelope per recipient, with no shared state between them. ### 5.3 Sealed payload @@ -110,15 +130,19 @@ padding 0..n ignored The signature covers `"smolmail/1 msg" || to || epk || version || sender || time || body_len || body`. Covering `to` prevents a recorded message being re-addressed to a third party and still verifying; covering `epk` binds the signature to this exact sealing. A receiver MUST verify the signature against `sender` and MUST check that `to` equals its own identity key. +A receiver MUST reject a payload whose `time` is more than 86400 seconds ahead of its own clock, and MAY mark a larger backwards skew. + +This `version` is not the envelope's: the envelope version governs the sealed-box construction of §5.1 and §5.2, this one governs the plaintext layout, and either can change while the other does not. + Senders MAY pad the plaintext to a multiple of 1024 bytes to blur message length; receivers MUST ignore trailing bytes. `body_len` makes padding unambiguous, and the AEAD tag protects it from alteration. ### 5.4 Message identifier ``` -id = SHA-256("smolmail/1 id" || envelope)[0..16] +id = SHA-256("smolmail/1 id" || envelope) ``` -The identifier is derived rather than asserted: a sender cannot choose it, two distinct messages cannot collide on it, and both servers and clients can deduplicate without trusting anyone. +The identifier is derived rather than asserted: a sender cannot choose it, and both servers and clients can deduplicate without trusting anyone. It is used whole; nothing truncates it. ### 5.5 Body frontmatter @@ -127,15 +151,15 @@ If the body begins with the exact bytes `---\n`, an optional frontmatter block r ``` --- Subject: Re: the thing -In-Reply-To: 4f2a1c9e8b7d6a5f3e2d1c0b9a8f7e6d +In-Reply-To: 4f2a1c9e8b7d6a5f3e2d1c0b9a8f7e6d5c4b3a291807f6e5d4c3b2a1908f7e6d5 X-Mood: cautiously optimistic --- Body text starts here. ``` -One `Key: value` per line. Key is 1–64 bytes of `[A-Za-z0-9-]`; value is the remainder of the line after `:`, trimmed of surrounding spaces. No nesting, arrays, multi-line values, quoting, comments or type coercion. Where a key repeats, the first occurrence wins. Maximum 4 KiB and 64 keys. A malformed line invalidates the whole block, which is then displayed as ordinary body text — frontmatter fails closed toward display, never toward silent discard. +One `Key: value` per line. Key is 1–64 bytes of `[A-Za-z0-9-]`; value is the remainder of the line after `:`, trimmed of surrounding spaces. No nesting, arrays, multi-line values, quoting, comments or type coercion. Keys are compared case-insensitively, so `Subject` and `subject` are the same key; where a key repeats, the first occurrence wins. Maximum 4 KiB and 64 keys. A malformed line invalidates the whole block, which is then displayed as ordinary body text — frontmatter fails closed toward display, never toward silent discard. -`Subject`, `In-Reply-To` (a message id as 32 lowercase hex characters) and `Reply-To` (§5.7) are reserved. Unknown keys MUST be preserved, MAY be displayed, and MUST NOT alter client behaviour. A body whose text genuinely begins with `---` is escaped by emitting an empty block first. +`Subject`, `In-Reply-To` (a message id as 64 lowercase hex characters), `Reply-To` (§5.7) and `Accept` (§5.8) are reserved. Unknown keys MUST be preserved, MAY be displayed, and MUST NOT alter client behaviour. A body whose text genuinely begins with `---` is escaped by emitting an empty block first. **This grammar is not YAML; do not use a YAML parser.** It is flat so that a correct parser is twenty dependency-free lines, in a code path that handles attacker-controlled input. @@ -149,58 +173,98 @@ Nothing on the wire names a sender. A receiver learns only the signing key of § Like the sent copies of §5.6 this is a client convention and involves no server: a sender MAY include a `Reply-To` field holding its own full `smol://` address. The value sits inside the sealed, signed payload, so the signature binds the claim. A receiver that acts on it MUST check that the key carried in the URI equals the message's `sender`, MUST treat anything else as ordinary text, and MUST NOT let it replace a key already bound to that address (§8) — it is a first-contact aid, not a trust upgrade. +### 5.8 Accept tokens + +A server cannot see senders, so it cannot tell wanted mail from a flood. An **accept token** is the recipient's own 32-byte secret, given to one correspondent and to her server, which lets the server separate the two without learning who anyone is. + +``` +t = HMAC-SHA256(accept, correspondent_identity) # accept: §2 +mac = HMAC-SHA256(t, "smolmail/1 mac" || id) # id: §5.4 +``` + +The accept key never leaves the owner's device. `t` is derived rather than random so that it is a function of the master and the correspondent's identity alone, and a MUST: one distinct token per correspondent, or they can neither be told apart nor removed individually. The owner uploads her tokens with `AUTH` (§4) and delivers each one to its correspondent as a reserved `Accept` frontmatter field, base32, inside a sealed and signed payload. A receiver MUST ignore an `Accept` value unless the payload's signature verifies, and MUST attribute the token to the signer. Because a token is issued by a mailbox rather than by one of its keys, a receiver SHOULD file it under the address it knows that signer by — one already bound to the key (§8), or the `Reply-To` the same payload signs for itself (§5.7) — so that the token keeps working across the issuer's rotations and a rotation it never observed cannot orphan it. + +A sender holding a token attaches `mac` to `SEND` (§6.1). The server recomputes `HMAC(t_i, …)` for each token `t_i` stored for the recipient's mailbox and compares in constant time: + +- **A token matches** — the envelope enters the mailbox's main quota and retention. +- **No token matches, the MAC is malformed, or none is attached** — the envelope enters the **requests** tier, a much smaller quota with a short retention (§10). + +`FETCH` marks which tier a message arrived in, so a client can present unsolicited mail separately from correspondence. A failed match is never reported to the sender: `SEND` succeeds either way, and cannot be used to test whether a token is still accepted. + +Removing a token from the set — an `AUTH` with `sync = 1` that omits it — demotes that correspondent to the requests tier from the next message on. The tier is a quota boundary, not a filter: refusing someone's mail outright remains the client's decision, after decryption. + +Because the accept key is independent of the rotation index (§2), tokens survive the owner's key rotation. A token is likewise derived from the correspondent's identity as it stood when they were accepted, so their later rotation does not invalidate the token they hold. + ## 6. Operations | Type | Op | Auth | Request → response | | --- | --- | --- | --- | | 0x01 | `RESOLVE` | no | username → identity key + rotation chain, possibly empty | -| 0x02 | `SEND` | no | one envelope → id. Redelivery is a no-op; the id already exists | -| 0x03 | `FETCH` | yes | → count, then `id \|\| received_at \|\| envelope` records, oldest first, up to a server byte budget | -| 0x04 | `DELETE` | yes | count + 16-byte ids → number removed. Unknown ids are not an error | -| 0x05 | `REGISTER` | no | username + identity key + optional invite token + optional rotation certificate | +| 0x02 | `SEND` | no | one envelope, optionally with an accept MAC → id. Redelivery is a no-op | +| 0x03 | `FETCH` | yes | cursor → count, then `id \|\| received_at \|\| flags \|\| envelope` records, oldest first, up to a server byte budget | +| 0x04 | `DELETE` | yes | count + 32-byte ids → number removed. Unknown ids are not an error | +| 0x05 | `REGISTER` | no | username + identity key + proof of possession + optional invite token + optional rotation certificate | `AUTH` (§4) is session setup, not an operation. `SEND` requires no account on the recipient's server. Clients connect directly to the server named in the address; there is no relaying and no federation, so an outbound queue lives in the sending client. -`FETCH` has no cursors, flags or server-side read state; a client loops until it returns nothing. `REGISTER` without a certificate binds a free username, or returns code 9; with one, it rebinds an existing username when the chain validates against the currently bound key. +`FETCH` carries a cursor and has no flags or server-side read state; a client pages forward until a response is empty, and deleting is a separate decision. `REGISTER` without a certificate binds a free username, or returns code 9; with one, it rebinds an existing username when the chain validates against the currently bound key. ### 6.1 Request and response bodies -A request body is the operation's payload. A response body begins with `status (u8)` from §12; on `0` the operation's payload follows, on anything else an optional UTF-8 reason string. A response reuses the request's type byte. `AUTH` is type `0x00` and its response carries a status and no payload. +A request body is the operation's payload. A response body begins with `status (u8)` from §12; on `0` the operation's payload follows, on anything else an optional UTF-8 reason string. A response reuses the request's type byte. `AUTH` is type `0x00` and its response carries a status and, on success, the number of accept tokens the server now holds. ``` +AUTH → username_len u8 || username || identity 32 || signature 64 + || sync u8 || count u16 || token 32 × count + ← accepted u16 + RESOLVE → username_len u8 || username - ← identity 32 || chain_len u8 || cert 136 × chain_len + ← identity 32 || chain_len u8 || cert 200 × chain_len -SEND → envelope - ← id 16 +SEND → mac_len u8 || mac || envelope + ← id 32 -FETCH → (empty) - ← count u16 || ( id 16 || received_at i64 || env_len u32 || envelope ) × count +FETCH → after_received_at i64 || after_id 32 + ← count u16 || ( id 32 || received_at i64 || flags u8 + || env_len u32 || envelope ) × count -DELETE → count u16 || id 16 × count +DELETE → count u16 || id 32 × count ← removed u16 -REGISTER → username_len u8 || username || identity 32 +REGISTER → username_len u8 || username || identity 32 || signature 64 || token_len u8 || token || cert_len u8 || cert ← (empty) ``` -A zero length means the field is absent. `FETCH` returns records oldest first, up to a server byte budget which MUST be smaller than the maximum application frame; 512 KiB against the default 1 MiB frame is a reasonable choice. +A zero length means the field is absent. `SEND`'s `mac_len` is 0 or 32 (§5.8). + +`REGISTER`'s signature is proof of possession, required on every registration and rotation: + +``` +signature = Ed25519 by identity over + "smolmail/1 register" || server_static || username || identity +``` + +`server_static` is the key the server sent during the handshake, so the attestation cannot be replayed to another server. Without it any party could bind any public key to any username. + +`FETCH` returns records ordered by `(received_at, id)`, strictly greater than the cursor; an all-zero cursor starts at the beginning, and a client's next cursor is the last record it received. A server fills a response up to a byte budget which MUST be smaller than the maximum application frame — 512 KiB against the default 1 MiB frame is a reasonable choice — and MUST return at least one record even when it alone exceeds the budget, so no message can wedge a mailbox shut. In `flags`, bit 0 is set when the message arrived without a matching accept token (§5.8); the remaining bits are zero. ## 7. Key rotation ``` -old_pub 32 || new_pub 32 || time 8 || signature 64 -signature = Ed25519 by old_pub over "smolmail/1 rotate" || old_pub || new_pub || time +old_pub 32 || new_pub 32 || time 8 || sig_old 64 || sig_new 64 +both signatures over "smolmail/1 rotate" || username || old_pub || new_pub || time ``` -A server keeps each username's ordered chain of certificates and returns it with every `RESOLVE`. A client holding a stale pinned key walks the chain forward from that key, verifying each link, and accepts the new key silently if the chain terminates at the key `RESOLVE` returned. A broken, absent or over-long chain — maximum 16 links — requires explicit user confirmation. Senders SHOULD also push their certificate to contacts as an ordinary message, so rotation propagates without depending on the old server. +`sig_old` is by `old_pub` and `sig_new` by `new_pub`. Both are required: the old key alone could otherwise hand a username to a key nobody controls, or to someone else's. The username is covered but not carried, which keeps the certificate fixed-size and stops it being a portable object replayable against another username bound to the same key; a verifier always knows which username it is checking. + +A server keeps each username's ordered chain of certificates and returns it with every `RESOLVE`. A client holding a stale pinned key walks the chain forward from that key, verifying both signatures on each link, and accepts the new key silently if the chain terminates at the key `RESOLVE` returned. A broken, absent or over-long chain — maximum 16 links — requires explicit user confirmation. Senders SHOULD also push their certificate to contacts as an ordinary message, so rotation propagates without depending on the old server. A server MUST retain every key ever bound to a username, MUST accept `SEND` addressed to any of them, and MUST return messages addressed to any of them on `FETCH`. Without this, mail from a contact who has not yet seen a rotation is addressed to a superseded key and becomes unreachable. -A client MUST likewise retain every private key it has rotated away from. Sealing is to a specific key (§5.2), so mail addressed to a superseded key is only ever readable with that key's private half. +A client MUST likewise be able to produce every private key it has rotated away from, since sealing is to a specific key (§5.2) and mail addressed to a superseded key is readable with nothing else. Advancing the rotation index rather than generating an unrelated key makes those keys a derivation from the master (§2) rather than an archive. -**Rotation is not revocation.** An attacker holding a stolen identity key can rotate to their own key and the chain will validate. Clients MUST surface rotations in the interface rather than applying them invisibly; out-of-band re-verification is the only defence against a compromised identity key. There is no revocation mechanism in version 1. +**Rotation is not revocation.** An attacker holding a stolen identity key can rotate to their own key and the chain will validate. Clients MUST surface rotations in the interface rather than applying them invisibly; out-of-band re-verification is the only defence against a compromised identity key. There is no revocation mechanism. ## 8. Trust model @@ -218,7 +282,9 @@ A client MUST likewise retain every private key it has rotated away from. Sealin **A server learns** which mailbox each envelope is for, its size, and when it arrived, was fetched and was deleted, plus the connecting IP address. It does not learn the sender, the subject, the content, or any relationship between messages. -**A network observer learns** that a client contacted a host on this port, and the volume and timing of traffic. Everything after the first round trip is encrypted, including usernames and recipient keys. +Accept tokens (§5.8) add to that: the server learns how many correspondents a user has accepted, whether a given message came from one, and which token it matched. A token is therefore a stable pseudonym, and the server can group a correspondent's messages under it without learning the identity behind it. That is the price of a mailbox that cannot be filled by anyone who knows an address, and an unfillable mailbox is a precondition for the rest of this document mattering. + +**A network observer learns** that a client contacted a host on this port, and the volume and timing of traffic. Everything after the first round trip is encrypted, including usernames, recipient keys and tokens. **Not protected:** traffic analysis, delivery timing correlation, mailbox size, and whether a given user has an account on a given server. Operators wanting metadata resistance should run the server as a Tor onion service; the transport is plain TCP and needs no changes. @@ -226,18 +292,28 @@ There is no recipient-side forward secrecy. Messages are signed, which provides ## 10. Servers -Independent store-and-forward mailboxes that never communicate with one another. Each holds its own username bindings, rotation chains and envelope queues. Envelopes persist until deleted or expired. Defaults, all configurable: 1 MiB per envelope, 64 MiB per mailbox, 30-day retention, and per-IP connection and `SEND` rate limits. +Independent store-and-forward mailboxes that never communicate with one another. Each holds its own username bindings, rotation chains, accept tokens and envelope queues. Envelopes persist until deleted or expired. -There are no blocklists or allowlists at any server, because a server cannot see senders. Abuse control is quotas, size caps and rate limits; sender and content filtering belong in the client, after decryption. Operators SHOULD NOT log recipient keys alongside IP addresses and timestamps, which would reconstruct much of the metadata the message format withholds. +Every mailbox has two tiers (§5.8). Defaults, all configurable: -A conforming server is a single binary over an embedded key-value store. +``` +768 KiB per envelope 2 MiB requests-tier quota + 64 MiB main-tier quota 7 days requests-tier retention + 30 days main-tier retention 1024 accept tokens per mailbox +``` + +A server also keeps the identifier of every envelope it has accepted for the length of its retention window, and answers a resend with the same id and no new message, so an envelope captured and replayed after deletion does not reappear. Clients keep their own seen-id set for the same reason. + +Abuse control is quotas, size caps and rate limits: per-IP connection and `SEND` limits, and a `SEND` limit per accept token, so one accepted correspondent cannot fill the main tier either. The accept-token set is an allow-list the mailbox owner controls and the server cannot interpret; there are no server-side denials, because a server still cannot see senders, and content filtering belongs in the client, after decryption. Operators SHOULD NOT log recipient keys alongside IP addresses and timestamps, which would reconstruct much of the metadata the message format withholds. ## 11. Domain separation labels ``` -"smolmail/1" Noise prologue "smolmail/1 msg" message signature -"smolmail/1 auth" session auth "smolmail/1 id" message identifier -"smolmail/1 seal" HKDF info "smolmail/1 rotate" rotation certificate +"smolmail/1" Noise prologue "smolmail/1 msg" message signature +"smolmail/1 identity" signing seed "smolmail/1 id" message identifier +"smolmail/1 accept" accept key "smolmail/1 mac" accept token MAC +"smolmail/1 auth" session auth "smolmail/1 rotate" rotation certificate +"smolmail/1 seal" HKDF info "smolmail/1 register" proof of possession ``` ## 12. Status codes @@ -249,15 +325,3 @@ A conforming server is a single binary over an embedded key-value store. ``` A status is the first byte of every response body (§6.1), which MAY be followed by a UTF-8 reason string. Clients MUST NOT parse the reason. - -## 13. Out of scope for version 1 - -Attachments. Group messaging — multiple recipients means multiple envelopes with no shared state. Federation and relaying. Anonymous routing. Multi-device synchronisation. Recipient-side forward secrecy. Key revocation. Metadata privacy against a network observer. Server push and notification. - -## 14. Conformance - -An implementation is conforming when it agrees byte for byte with the published test vectors: a fixed seed, its Ed25519 public key, its derived X25519 pair, an envelope sealed to a known recipient with a fixed ephemeral, and that envelope's message identifier. Those vectors are the interoperability test and SHOULD be the first artifact of any implementation. - -Sizes follow from §5: an envelope is 69 bytes of header plus ciphertext plus a 16-byte tag; a payload is 109 bytes plus the body. - -Every field in §5 has exactly one stated reason to exist. Any field proposed for a future version should be held to the same test. diff --git a/smolmail.py b/smolmail.py index a7dce6f..23957c4 100755 --- a/smolmail.py +++ b/smolmail.py @@ -5,9 +5,9 @@ # /// """Smol Mail reference client. -Implements SPEC.md version 1 from both ends: one Ed25519 identity, sealed and +Implements SPEC.md version 1.1 from both ends: one master secret, sealed and signed messages, server key pinning, trust on first use with rotation chains, -and the five operations. +accept tokens, and the five operations. Mail is stored sealed and opened on demand, so a stolen database is no more readable than a stolen mailbox on a server. @@ -42,6 +42,8 @@ NOISE_PROTOCOL = b"Noise_NX_25519_ChaChaPoly_SHA256" PROLOGUE = b"smolmail/1" LABEL_AUTH, LABEL_SEAL = b"smolmail/1 auth", b"smolmail/1 seal" LABEL_MSG, LABEL_ID, LABEL_ROTATE = b"smolmail/1 msg", b"smolmail/1 id", b"smolmail/1 rotate" +LABEL_IDENTITY, LABEL_ACCEPT = b"smolmail/1 identity", b"smolmail/1 accept" +LABEL_MAC, LABEL_REGISTER = b"smolmail/1 mac", b"smolmail/1 register" OP_AUTH, OP_RESOLVE, OP_SEND, OP_FETCH, OP_DELETE, OP_REGISTER = range(6) STATUS = {0: "ok", 1: "malformed", 2: "bad version", 3: "unknown user", @@ -49,12 +51,15 @@ STATUS = {0: "ok", 1: "malformed", 2: "bad version", 3: "unknown user", 8: "rate limited", 9: "not permitted", 10: "internal error"} MAGIC, VERSION = b"SMOL", 1 -KEY_LEN, ID_LEN, SIG_LEN, CERT_LEN = 32, 16, 64, 136 +KEY_LEN, ID_LEN, SIG_LEN, CERT_LEN, TOKEN_LEN = 32, 32, 64, 200, 32 PAYLOAD_HEADER = 45 # version 1 + sender 32 + time 8 + body_len 4 ENVELOPE_HEADER = 69 # magic 4 + version 1 + to 32 + epk 32 MAX_CHAIN, PAD_TO = 16, 1024 DEFAULT_PORT, MAX_FRAME, NOISE_PAYLOAD = 1961, 1 << 20, 65535 - 16 FRONTMATTER_MAX, FRONTMATTER_KEYS = 4096, 64 +SEPARATORS = "._-" +MAX_SKEW = 86400 # §5.3, how far ahead of our clock a payload may be dated +TIER_MAIN, TIER_REQUESTS, FLAG_REQUESTS = 0, 1, 0x01 class SmolError(Exception): @@ -111,6 +116,13 @@ class Reader: ADDRESS = re.compile(r"^(?P[a-z0-9._-]{1,63})@(?P[^/:]+)(?::(?P\d+))?$") +def valid_username(name: str) -> bool: + """§3: alphanumeric at both ends, never two separators in a row.""" + if name[0] in SEPARATORS or name[-1] in SEPARATORS: + return False + return not any(a in SEPARATORS and b in SEPARATORS for a, b in zip(name, name[1:])) + + class Address: """A short `user@host` address, or a self-certifying `smol://` one (§3).""" @@ -135,8 +147,9 @@ class Address: m = ADDRESS.match(text) if not m: raise SmolError(f"{text!r} is not a valid address") - if m["user"][0] in "._-" or m["user"][-1] in "._-": - raise SmolError(f"{m['user']!r} may not begin or end with a separator") + if not valid_username(m["user"]): + raise SmolError(f"{m['user']!r} must begin and end with a letter or digit " + f"and may not contain two separators in a row") return cls(m["user"], m["host"], int(m["port"] or DEFAULT_PORT), identity) @property @@ -160,6 +173,21 @@ def hkdf_sha256(ikm: bytes, salt: bytes, info: bytes, length: int = 32) -> bytes return out[:length] +def identity_seed(master: bytes, index: int) -> bytes: + """§2. The Ed25519 seed at a rotation index.""" + return hkdf_sha256(master, b"", LABEL_IDENTITY + struct.pack(">I", index)) + + +def accept_key(master: bytes) -> bytes: + """§2. Independent of the rotation index, so tokens survive a rotation.""" + return hkdf_sha256(master, b"", LABEL_ACCEPT) + + +def accept_mac(token: bytes, mid: bytes) -> bytes: + """§5.8. What a sender attaches to SEND to reach the main tier.""" + return hmac.new(token, LABEL_MAC + mid, hashlib.sha256).digest() + + def ed_to_x25519_pub(identity: bytes) -> bytes: """§2, via libsodium's own conversion as the spec prefers.""" try: @@ -184,6 +212,26 @@ class Identity: return self._signing.sign(message).signature +class Account: + """The master secret of §2 and everything derived from it. + + `keys` holds every index up to the current one: sealing is to a specific key, + so mail addressed to a superseded one is readable only with that key (§7). + Rotation advances the index, so none of them has to be archived. + """ + + def __init__(self, master: bytes, index: int) -> None: + if len(master) != KEY_LEN: + raise SmolError(f"master secret must be {KEY_LEN} bytes, got {len(master)}") + self.master, self.index = master, index + self.keys = [Identity(identity_seed(master, n)) for n in range(index + 1)] + self.me = self.keys[-1] + + def token_for(self, identity: bytes) -> bytes: + """§5.8. The accept token this account issues to one correspondent.""" + return hmac.new(accept_key(self.master), identity, hashlib.sha256).digest() + + def verify_sig(identity: bytes, signature: bytes, message: bytes) -> bool: try: VerifyKey(identity).verify(message, signature) @@ -193,7 +241,7 @@ def verify_sig(identity: bytes, signature: bytes, message: bytes) -> bool: def agree(x_priv: bytes, x_pub: bytes) -> bytes: - """X25519 with §3.2's checks; libsodium rejects low-order points itself.""" + """X25519 with §2's checks; libsodium rejects low-order points itself.""" try: shared = sodium.crypto_scalarmult(x_priv, x_pub) except (RuntimeError, nacl.exceptions.CryptoError) as exc: @@ -205,7 +253,7 @@ def agree(x_priv: bytes, x_pub: bytes) -> bytes: def message_id(envelope: bytes) -> bytes: """§5.4, derived from the envelope so no sender can choose it.""" - return hashlib.sha256(LABEL_ID + envelope).digest()[:ID_LEN] + return hashlib.sha256(LABEL_ID + envelope).digest() def seal(sender: Identity, recipient: bytes, body: bytes, when: int | None = None, @@ -228,11 +276,7 @@ def seal(sender: Identity, recipient: bytes, body: bytes, when: int | None = Non def unseal(identities: list[Identity], envelope: bytes) -> dict: - """Inverse of seal(); raises unless signature and recipient both check out. - - `identities` may include retired keys: sealing is to a specific key, so mail - addressed to a superseded one needs that key (§7). - """ + """Inverse of seal(); raises unless signature and recipient both check out.""" if len(envelope) < ENVELOPE_HEADER + 16: raise SmolError("envelope too short") if envelope[:4] != MAGIC: @@ -261,6 +305,8 @@ def unseal(identities: list[Identity], envelope: bytes) -> dict: if not verify_sig(sender, signature, LABEL_MSG + to + epk + plaintext[:PAYLOAD_HEADER] + body): raise SmolError("signature does not verify") + if when > int(time.time()) + MAX_SKEW: + raise SmolError("payload is dated in the future") return {"sender": sender, "time": when, "body": body, "id": message_id(envelope)} @@ -273,7 +319,8 @@ def parse_frontmatter(body: str) -> tuple[dict[str, str], str]: """§5.5. A flat `Key: value` block, deliberately not YAML. Any malformed line invalidates the whole block, which is then returned as - ordinary body text: frontmatter fails closed toward display. + ordinary body text: frontmatter fails closed toward display. Keys are + compared case-insensitively, so they are kept lowercased. """ if not body.startswith("---\n"): return {}, body @@ -292,7 +339,7 @@ def parse_frontmatter(body: str) -> tuple[dict[str, str], str]: key, sep, value = line.partition(":") if not sep or not FM_KEY.match(key): return {}, body - fields.setdefault(key, value.strip()) # first occurrence wins + fields.setdefault(key.lower(), value.strip()) # first occurrence wins return fields, rest @@ -307,21 +354,39 @@ def build_frontmatter(fields: list[tuple[str, str]], body: str) -> str: # --- local state ------------------------------------------------------------ SCHEMA = """ -CREATE TABLE IF NOT EXISTS account ( - id INTEGER PRIMARY KEY CHECK (id = 1), username TEXT, host TEXT, port INTEGER); +-- One row, always present, so every update is a plain UPDATE. +CREATE TABLE IF NOT EXISTS state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + username TEXT, host TEXT, port INTEGER, + rotations INTEGER NOT NULL DEFAULT 0, -- §2 rotation index + after_time INTEGER NOT NULL DEFAULT 0, -- §6.1 FETCH cursor + after_id BLOB NOT NULL DEFAULT x'', + sync_ok INTEGER NOT NULL DEFAULT 1); -- may we replace the server's set? +INSERT OR IGNORE INTO state (id) VALUES (1); CREATE TABLE IF NOT EXISTS servers ( host TEXT PRIMARY KEY, static BLOB NOT NULL, pinned_at INTEGER NOT NULL); CREATE TABLE IF NOT EXISTS contacts ( address TEXT PRIMARY KEY, identity BLOB NOT NULL, verified INTEGER NOT NULL, -- 1 when the key came from a smol:// address seen_at INTEGER NOT NULL); --- Private keys rotated away from. Mail sealed to a superseded key is readable --- only with that key, so these can never be discarded (§7). -CREATE TABLE IF NOT EXISTS retired ( - identity BLOB PRIMARY KEY, seed BLOB NOT NULL, retired_at INTEGER NOT NULL); +-- Correspondents admitted to this mailbox's main tier (§5.8). The identity is +-- frozen at acceptance because the token is derived from it: a contact's later +-- rotation must not change the token they already hold. +CREATE TABLE IF NOT EXISTS accepted ( + address TEXT PRIMARY KEY, identity BLOB NOT NULL, + active INTEGER NOT NULL, added_at INTEGER NOT NULL); +-- Accept tokens received from correspondents, filed under the address that +-- issued them: an address outlives the keys behind it, so a token keeps working +-- across the issuer's rotations (§5.8). +CREATE TABLE IF NOT EXISTS tokens ( + address TEXT PRIMARY KEY, token BLOB NOT NULL, seen_at INTEGER NOT NULL); +-- Every id ever fetched, so an envelope resent after we deleted it locally is +-- not stored again (§10). +CREATE TABLE IF NOT EXISTS seen (id BLOB PRIMARY KEY, at INTEGER NOT NULL); -- Envelopes are stored sealed and opened on demand; no plaintext at rest. CREATE TABLE IF NOT EXISTS inbox ( - id BLOB PRIMARY KEY, envelope BLOB NOT NULL, received_at INTEGER NOT NULL); + id BLOB PRIMARY KEY, envelope BLOB NOT NULL, + received_at INTEGER NOT NULL, tier INTEGER NOT NULL); CREATE TABLE IF NOT EXISTS sent ( id BLOB PRIMARY KEY, recipient TEXT NOT NULL, envelope BLOB NOT NULL, sent_at INTEGER NOT NULL); @@ -345,14 +410,24 @@ class Store: return self.db.execute(sql, args) def account(self) -> Address | None: - row = self.one("SELECT username, host, port FROM account WHERE id = 1") + row = self.one("SELECT username, host, port FROM state WHERE id = 1") return Address(row[0], row[1], row[2]) if row and row[0] else None def set_account(self, addr: Address) -> None: - self.run("INSERT INTO account (id, username, host, port) VALUES (1, ?1, ?2, ?3) " - "ON CONFLICT (id) DO UPDATE SET username = ?1, host = ?2, port = ?3", + self.run("UPDATE state SET username = ?, host = ?, port = ? WHERE id = 1", addr.user, addr.host, addr.port) + def rotations(self) -> int: + return self.one("SELECT rotations FROM state WHERE id = 1")[0] + + def cursor(self) -> tuple[int, bytes]: + after_time, after_id = self.one("SELECT after_time, after_id FROM state WHERE id = 1") + return after_time, after_id if len(after_id) == ID_LEN else bytes(ID_LEN) + + def set_cursor(self, after_time: int, after_id: bytes) -> None: + self.run("UPDATE state SET after_time = ?, after_id = ? WHERE id = 1", + after_time, after_id) + def contact(self, address: str) -> tuple[bytes, bool] | None: row = self.one("SELECT identity, verified FROM contacts WHERE address = ?", address) return (row[0], bool(row[1])) if row else None @@ -367,17 +442,63 @@ class Store: "ON CONFLICT (host) DO UPDATE SET static = ?2, pinned_at = ?3", host, static, int(time.time())) - def identities(self, me: Identity) -> list[Identity]: - """The current key plus every key rotated away from.""" - return [me] + [Identity(row[0]) for row in - self.all("SELECT seed FROM retired ORDER BY retired_at")] + def token_set(self, account: "Account") -> tuple[int, list[bytes]]: + """§4. The accept tokens to push with AUTH, and whether to push at all. + + A client that cannot vouch for its own set — one restored from the master + alone — must not replace the server's with an incomplete one. + """ + if not self.one("SELECT sync_ok FROM state WHERE id = 1")[0]: + return 0, [] + rows = self.all("SELECT identity FROM accepted WHERE active = 1 ORDER BY added_at") + return 1, [account.token_for(row[0]) for row in rows] + + def address_of(self, sender: bytes, reply_to: str | None) -> str | None: + """The address we know a signer by: a contact, or the Reply-To it signed + for itself. Naming a mailbox is not trusting a key, so nothing is + pinned here (§5.7, §8).""" + for address, identity in self.all("SELECT address, identity FROM contacts"): + if hmac.compare_digest(identity, sender): + return address + if reply_to: + try: + claimed = Address.parse(reply_to) + except SmolError: + return None + if claimed.identity and hmac.compare_digest(claimed.identity, sender): + return claimed.short + return None + + def learn_token(self, opened: dict) -> None: + """§5.8. An Accept field is bound to the signer of the message that + carried it, which unseal() has already verified.""" + fields, _ = parse_frontmatter(opened["body"].decode("utf-8", "replace")) + raw = fields.get("accept") + if not raw: + return + try: + token = unb32(raw) + except Exception: + return + if len(token) != TOKEN_LEN: + return + address = self.address_of(opened["sender"], fields.get("reply-to")) + if address is None: + return # no address to send to, so no use for a token + self.run("INSERT INTO tokens (address, token, seen_at) VALUES (?,?,?) " + "ON CONFLICT (address) DO UPDATE SET token = ?2, seen_at = ?3", + address, token, int(time.time())) def mail(self, box: str) -> list[tuple]: if box == "sent": return self.all("SELECT id, envelope, sent_at, recipient FROM sent " "ORDER BY sent_at, id") + if box == "all": + return self.all("SELECT id, envelope, received_at, NULL FROM inbox " + "ORDER BY received_at, id") + tier = TIER_REQUESTS if box == "requests" else TIER_MAIN return self.all("SELECT id, envelope, received_at, NULL FROM inbox " - "ORDER BY received_at, id") + "WHERE tier = ? ORDER BY received_at, id", tier) # --- transport (§4) --------------------------------------------------------- @@ -486,23 +607,25 @@ def do_resolve(conn: Conn, user: str) -> tuple[bytes, list[bytes]]: return identity, [r.take(CERT_LEN) for _ in range(r.u8())] -def walk_chain(pinned: bytes, current: bytes, chain: list[bytes]) -> bool: +def walk_chain(username: str, pinned: bytes, current: bytes, chain: list[bytes]) -> bool: """§7. Accept a key change only when a signed chain leads from the key we - hold to the one the server now returns.""" + hold to the one the server now returns. Both keys must sign each link.""" if hmac.compare_digest(pinned, current): return True if not chain or len(chain) > MAX_CHAIN: return False key, started = pinned, False for cert in chain: - old, new, when, sig = cert[:32], cert[32:64], cert[64:72], cert[72:] + old, new, when = cert[:32], cert[32:64], cert[64:72] + sig_old, sig_new = cert[72:136], cert[136:200] if not started: if old != key: continue # a link predating the key we hold started = True elif old != key: return False # the chain is not continuous - if not verify_sig(old, sig, LABEL_ROTATE + old + new + when): + signed = LABEL_ROTATE + username.encode() + old + new + when + if not verify_sig(old, sig_old, signed) or not verify_sig(new, sig_new, signed): return False key = new return started and hmac.compare_digest(key, current) @@ -520,7 +643,7 @@ def trust_key(store: Store, conn: Conn, addr: Address) -> bytes: old, verified = known if hmac.compare_digest(old, identity): return identity - if walk_chain(old, identity, chain): + if walk_chain(addr.user, old, identity, chain): store.save_contact(addr.short, identity, verified) warn(f"{addr.short} rotated its key; a signed chain confirms it") print(f" now {b32(identity)}") @@ -530,12 +653,33 @@ def trust_key(store: Store, conn: Conn, addr: Address) -> bytes: f"Verify out of band, then: smolmail.py import {addr.uri(identity)}") -def authenticate(conn: Conn, addr: Address, me: Identity) -> None: - """§4 session authentication: sign the handshake hash.""" +def register_signed(server_static: bytes, username: str, identity: bytes) -> bytes: + """§6.1 proof of possession, bound to the server that will store the binding.""" + return LABEL_REGISTER + server_static + username.encode() + identity + + +def authenticate(conn: Conn, addr: Address, account: Account, store: Store) -> int: + """§4 session authentication: sign the handshake hash and push the token set.""" name = addr.user.encode() - status, _ = conn.call(OP_AUTH, bytes([len(name)]) + name + me.pk - + me.sign(LABEL_AUTH + conn.handshake_hash)) + sync, tokens = store.token_set(account) + body = (bytes([len(name)]) + name + account.me.pk + + account.me.sign(LABEL_AUTH + conn.handshake_hash) + + bytes([sync]) + struct.pack(">H", len(tokens)) + b"".join(tokens)) + status, payload = conn.call(OP_AUTH, body) expect_ok(status, "authentication") + return struct.unpack(">H", payload[:2])[0] if len(payload) >= 2 else 0 + + +def push_tokens(store: Store, account: Account) -> None: + """§5.8. An accept or a block only takes effect once the server holds the + changed set, so it is pushed now rather than at the next fetch.""" + addr = store.account() + if addr is None: + warn("not registered; the set will be pushed with your first fetch") + return + with connect(store, addr, require_pin=True) as conn: + held = authenticate(conn, addr, account, store) + print(f"server now holds {held} accept token(s)") # --- helpers ---------------------------------------------------------------- @@ -545,19 +689,24 @@ def warn(message: str) -> None: print(f"warning: {message}", file=sys.stderr) -def load_identity(args: argparse.Namespace) -> Identity: +def load_master(args: argparse.Namespace) -> bytes: try: with open(args.key, "rb") as fh: - return Identity(fh.read()) + return fh.read() except FileNotFoundError: raise SmolError(f"no identity at {args.key}; run: smolmail.py keygen") from None -def write_seed(path: str, seed: bytes) -> None: - """0600 before any bytes land, so the seed is never briefly world-readable.""" +def load_account(args: argparse.Namespace, store: Store) -> Account: + """§2. The master from disk plus the rotation index from local state.""" + return Account(load_master(args), store.rotations()) + + +def write_secret(path: str, secret: bytes) -> None: + """0600 before any bytes land, so the secret is never briefly world-readable.""" fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, "wb") as fh: - fh.write(seed) + fh.write(secret) def describe(store: Store, envelope: bytes, identities: list[Identity]) -> dict: @@ -567,33 +716,48 @@ def describe(store: Store, envelope: bytes, identities: list[Identity]) -> dict: sender = opened["sender"] known = next((a for a, i, _ in store.all( "SELECT address, identity, verified FROM contacts") if hmac.compare_digest(i, sender)), None) - opened |= {"fields": fields, "text": text, "subject": fields.get("Subject", ""), + opened |= {"fields": fields, "text": text, "subject": fields.get("subject", ""), "from": known or f"<{b32(sender)[:20]}…>"} return opened +def target_contact(store: Store, text: str) -> tuple[str, bytes]: + """An address plus the key we hold for it, for commands naming a contact.""" + addr = Address.parse(text) + if addr.identity is not None: + return addr.short, addr.identity + known = store.contact(addr.short) + if known is None: + raise SmolError(f"no key for {addr.short}; run `resolve` or `import` first") + return addr.short, known[0] + + # --- commands --------------------------------------------------------------- def cmd_keygen(args: argparse.Namespace, store: Store) -> int: if os.path.exists(args.key) and not args.force: raise SmolError(f"{args.key} exists; refusing to overwrite (use --force)") - me = Identity(os.urandom(KEY_LEN)) - write_seed(args.key, me.seed) - print(f"identity: {args.key} (back this up; it is the only secret)") - print(f"public key: {b32(me.pk)}\nfingerprint: {fingerprint(me.pk)}") + master = os.urandom(KEY_LEN) + write_secret(args.key, master) + account = Account(master, 0) + print(f"master: {args.key} (back this up; it is the only secret)") + print(f"public key: {b32(account.me.pk)}\nfingerprint: {fingerprint(account.me.pk)}") return 0 def cmd_whoami(args: argparse.Namespace, store: Store) -> int: - me = load_identity(args) - print(f"public key: {b32(me.pk)}\nfingerprint: {fingerprint(me.pk)}") + account = load_account(args, store) + print(f"public key: {b32(account.me.pk)}\nfingerprint: {fingerprint(account.me.pk)}") addr = store.account() - print(f"address: {addr.short}\nuri: {addr.uri(me.pk)}" if addr + print(f"address: {addr.short}\nuri: {addr.uri(account.me.pk)}" if addr else "address: (not registered)") - retired = len(store.all("SELECT 1 FROM retired")) - if retired: - print(f"retired: {retired} superseded key(s), kept to read old mail") + if account.index: + print(f"rotations: {account.index} (earlier keys derived on demand)") + live = store.one("SELECT COUNT(*) FROM accepted WHERE active = 1")[0] + sync = store.one("SELECT sync_ok FROM state WHERE id = 1")[0] + print(f"accepted: {live} correspondent(s)" + + ("" if sync else ", not pushed to the server until rebuilt")) return 0 @@ -614,10 +778,13 @@ def cmd_trust(args: argparse.Namespace, store: Store) -> int: def cmd_register(args: argparse.Namespace, store: Store) -> int: - me, addr = load_identity(args), Address.parse(args.address) + account, addr = load_account(args, store), Address.parse(args.address) + me = account.me name, token = addr.user.encode(), (args.token or "").encode() - body = (bytes([len(name)]) + name + me.pk + bytes([len(token)]) + token + b"\0") with connect(store, addr, require_pin=True) as conn: + body = (bytes([len(name)]) + name + me.pk + + me.sign(register_signed(conn.server_static, addr.user, me.pk)) + + bytes([len(token)]) + token + b"\0") status, _ = conn.call(OP_REGISTER, body) expect_ok(status, f"registering {addr.short}") store.set_account(addr) @@ -645,18 +812,49 @@ def cmd_import(args: argparse.Namespace, store: Store) -> int: def cmd_contacts(args: argparse.Namespace, store: Store) -> int: - rows = store.all("SELECT address, identity, verified FROM contacts ORDER BY address") + rows = store.all("SELECT c.address, c.identity, c.verified, a.active FROM contacts c " + "LEFT JOIN accepted a ON a.address = c.address ORDER BY c.address") if not rows: print("no contacts") return 0 - width = max(len(a) for a, _, _ in rows) - for address, identity, verified in rows: - print(f"{address:<{width}} {b32(identity)} {'verified' if verified else 'tofu'}") + width = max(len(a) for a, _, _, _ in rows) + for address, identity, verified, active in rows: + state = "accepted" if active else ("blocked" if active == 0 else "") + print(f"{address:<{width}} {b32(identity)} " + f"{'verified' if verified else 'tofu':<8} {state}".rstrip()) + return 0 + + +def cmd_accept(args: argparse.Namespace, store: Store) -> int: + account = load_account(args, store) + address, identity = target_contact(store, args.address) + if not store.one("SELECT sync_ok FROM state WHERE id = 1")[0]: + warn("this client's accepted set was not restored; from now on it replaces " + "the server's, so re-accept everyone you still correspond with") + # ON CONFLICT leaves `identity` alone: the token stays the one the + # correspondent already holds, even after they rotate (§5.8). + store.run("INSERT INTO accepted (address, identity, active, added_at) VALUES (?,?,1,?) " + "ON CONFLICT (address) DO UPDATE SET active = 1", address, identity, + int(time.time())) + store.run("UPDATE state SET sync_ok = 1 WHERE id = 1") + print(f"accepted {address}; its token travels in your next message to them") + push_tokens(store, account) + return 0 + + +def cmd_block(args: argparse.Namespace, store: Store) -> int: + account = load_account(args, store) + address, _ = target_contact(store, args.address) + if not store.run("UPDATE accepted SET active = 0 WHERE address = ?", address).rowcount: + raise SmolError(f"{address} was never accepted") + print(f"blocked {address}; their mail lands in requests from their next message on") + push_tokens(store, account) return 0 def cmd_send(args: argparse.Namespace, store: Store) -> int: - me, addr = load_identity(args), Address.parse(args.address) + account, addr = load_account(args, store), Address.parse(args.address) + me = account.me if args.body is not None: text = args.body elif sys.stdin.isatty(): @@ -681,36 +879,48 @@ def cmd_send(args: argparse.Namespace, store: Store) -> int: raise SmolError(f"{raw!r} is not a valid `Key: value` header") fields.append((key.strip(), value.strip())) # §5.7: a signed reply address lets a first-time recipient answer us. - if (account := store.account()) is not None and not args.anonymous: - fields.append(("Reply-To", account.uri(me.pk))) + if (account_addr := store.account()) is not None and not args.anonymous: + fields.append(("Reply-To", account_addr.uri(me.pk))) + # §5.8: hand an accepted correspondent the token for our own mailbox. + if (row := store.one("SELECT identity FROM accepted WHERE address = ? AND active = 1", + addr.short)) is not None: + fields.append(("Accept", b32(account.token_for(row[0])))) body = build_frontmatter(fields, text).encode() envelope = seal(me, recipient, body, pad=not args.no_pad) - with connect(store, addr, require_pin=False) as conn: - status, payload = conn.call(OP_SEND, envelope) - expect_ok(status, f"sending to {addr.short}") mid = message_id(envelope) + # §5.8: our token for their mailbox, if they have given us one. + held = store.one("SELECT token FROM tokens WHERE address = ?", addr.short) + mac = accept_mac(held[0], mid) if held else b"" + with connect(store, addr, require_pin=False) as conn: + status, payload = conn.call(OP_SEND, bytes([len(mac)]) + mac + envelope) + expect_ok(status, f"sending to {addr.short}") if payload and payload != mid: warn("server returned an id we did not derive; it is not authoritative") # §5.6: the ephemeral is gone, so keep a copy sealed to ourselves. store.run("INSERT OR IGNORE INTO sent (id, recipient, envelope, sent_at) VALUES (?,?,?,?)", mid, addr.short, seal(me, me.pk, body, pad=not args.no_pad), int(time.time())) - print(f"sent {mid.hex()} to {addr.short} ({len(envelope)} bytes)") + print(f"sent {mid.hex()[:16]} to {addr.short} ({len(envelope)} bytes" + + (", accepted" if mac else "") + ")") return 0 def cmd_fetch(args: argparse.Namespace, store: Store) -> int: - me = load_identity(args) + account = load_account(args, store) addr = store.account() if addr is None: raise SmolError("not registered; run: smolmail.py register ") - identities = store.identities(me) + if args.reset: + store.set_cursor(0, bytes(ID_LEN)) + # Acknowledging deletes what it takes, so it always pages from the start and + # leaves the stored cursor at zero; --keep instead remembers its place. + after_time, after_id = store.cursor() if args.keep else (0, bytes(ID_LEN)) stored = rejected = total = 0 with connect(store, addr, require_pin=True) as conn: - authenticate(conn, addr, me) + authenticate(conn, addr, account, store) while True: - status, body = conn.call(OP_FETCH) + status, body = conn.call(OP_FETCH, struct.pack(">q", after_time) + after_id) expect_ok(status, "fetching") r = Reader(body) count = r.u16() @@ -719,42 +929,55 @@ def cmd_fetch(args: argparse.Namespace, store: Store) -> int: acked = [] for _ in range(count): total += 1 - mid, received_at = r.take(ID_LEN), r.i64() + mid, received_at, flags = r.take(ID_LEN), r.i64(), r.u8() envelope = r.take(r.u32()) + after_time, after_id = received_at, mid try: if message_id(envelope) != mid: raise SmolError("id does not match the envelope") - unseal(identities, envelope) + opened = unseal(account.keys, envelope) except SmolError as exc: # Left on the server rather than destroyed, so a client-side # bug cannot lose mail. - warn(f"{mid.hex()}: {exc}; left on server") + warn(f"{mid.hex()[:16]}: {exc}; left on server") rejected += 1 continue - if store.run("INSERT OR IGNORE INTO inbox (id, envelope, received_at) " - "VALUES (?,?,?)", mid, envelope, received_at).rowcount: + # A message we have already had once is not stored again, even + # if we deleted it locally in the meantime (§10). + if store.one("SELECT 1 FROM seen WHERE id = ?", mid) is None: + tier = TIER_REQUESTS if flags & FLAG_REQUESTS else TIER_MAIN + store.run("INSERT OR IGNORE INTO inbox " + "(id, envelope, received_at, tier) VALUES (?,?,?,?)", + mid, envelope, received_at, tier) + store.run("INSERT OR IGNORE INTO seen (id, at) VALUES (?,?)", + mid, received_at) + store.learn_token(opened) stored += 1 acked.append(mid) - if not acked or args.keep: - break - status, _ = conn.call(OP_DELETE, struct.pack(">H", len(acked)) + b"".join(acked)) - expect_ok(status, "acknowledging") + if args.keep: + store.set_cursor(after_time, after_id) + elif acked: + status, _ = conn.call(OP_DELETE, + struct.pack(">H", len(acked)) + b"".join(acked)) + expect_ok(status, "acknowledging") + if not args.keep: + store.set_cursor(0, bytes(ID_LEN)) print(f"{total} message(s): {stored} new, {rejected} rejected" + (" (left on the server)" if args.keep and total else "")) return 0 def cmd_list(args: argparse.Namespace, store: Store) -> int: - me = load_identity(args) - identities, box = store.identities(me), "sent" if args.sent else "inbox" + account = load_account(args, store) + box = "sent" if args.sent else "requests" if args.requests else "inbox" rows = store.mail(box) if not rows: print(f"{box} is empty") return 0 for mid, envelope, when, recipient in rows: try: - opened = describe(store, envelope, identities) + opened = describe(store, envelope, account.keys) who = recipient if box == "sent" else opened["from"] subject = opened["subject"] or "(no subject)" except SmolError as exc: @@ -765,56 +988,91 @@ def cmd_list(args: argparse.Namespace, store: Store) -> int: def cmd_read(args: argparse.Namespace, store: Store) -> int: - me = load_identity(args) - identities, box = store.identities(me), "sent" if args.sent else "inbox" + account = load_account(args, store) + box = "sent" if args.sent else "all" matches = [r for r in store.mail(box) if r[0].hex().startswith(args.id.lower())] if not matches: - raise SmolError(f"no message in {box} matching {args.id!r}") + raise SmolError(f"no {'sent ' if args.sent else ''}message matching {args.id!r}") if len(matches) > 1: raise SmolError(f"{args.id!r} matches {len(matches)} messages; be more specific") mid, envelope, _, recipient = matches[0] - opened = describe(store, envelope, identities) - print(f"id: {mid.hex()}") + opened = describe(store, envelope, account.keys) + def field(key: str, value: str) -> None: + print(f"{key + ':':<12} {value}") + + field("id", mid.hex()) if box == "sent": - print(f"to: {recipient}") - print(f"from: {opened['from']}\nkey: {b32(opened['sender'])}") - print("date: " + time.strftime("%Y-%m-%d %H:%M:%S %z", time.localtime(opened["time"]))) + field("to", recipient) + field("from", opened["from"]) + field("key", b32(opened["sender"])) + field("date", time.strftime("%Y-%m-%d %H:%M:%S %z", time.localtime(opened["time"]))) for key, value in opened["fields"].items(): - print(f"{key.lower() + ':':<9}{value}") + if key != "accept": # machinery, not content (§5.8) + field(key, value) print("signature verified\n") print(opened["text"], end="" if opened["text"].endswith("\n") else "\n") return 0 def cmd_rotate(args: argparse.Namespace, store: Store) -> int: - old = load_identity(args) + account = load_account(args, store) addr = store.account() if addr is None: raise SmolError("not registered; run: smolmail.py register ") - new = Identity(os.urandom(KEY_LEN)) + if account.index >= MAX_CHAIN: + raise SmolError(f"the rotation chain is full at {MAX_CHAIN} links") + old = account.me + new = Identity(identity_seed(account.master, account.index + 1)) when = struct.pack(">q", int(time.time())) - cert = old.pk + new.pk + when + old.sign(LABEL_ROTATE + old.pk + new.pk + when) + # §7: both keys sign, so the old key alone cannot hand the username to a + # key nobody controls. + signed = LABEL_ROTATE + addr.user.encode() + old.pk + new.pk + when + cert = old.pk + new.pk + when + old.sign(signed) + new.sign(signed) name = addr.user.encode() - body = bytes([len(name)]) + name + new.pk + b"\0" + bytes([len(cert)]) + cert with connect(store, addr, require_pin=True) as conn: + body = (bytes([len(name)]) + name + new.pk + + new.sign(register_signed(conn.server_static, addr.user, new.pk)) + + b"\0" + bytes([len(cert)]) + cert) status, _ = conn.call(OP_REGISTER, body) expect_ok(status, "rotating") - # Retain the old seed first: mail already sealed to it is otherwise unreadable. - store.run("INSERT OR IGNORE INTO retired (identity, seed, retired_at) VALUES (?,?,?)", - old.pk, old.seed, int(time.time())) - write_seed(args.key, new.seed) + # The master is untouched; only the index moves, and the superseded key stays + # derivable from it (§2). + store.run("UPDATE state SET rotations = ? WHERE id = 1", account.index + 1) print(f"rotated {addr.short}\nnew key: {b32(new.pk)}") print(f"fingerprint: {fingerprint(new.pk)}\nshare: {addr.uri(new.pk)}") print("\nTell your contacts; they will accept the change from the signed chain.") return 0 +def cmd_restore(args: argparse.Namespace, store: Store) -> int: + """§2. Recover the rotation index, and with it every superseded key, from + the master alone.""" + master = load_master(args) + addr = Address.parse(args.address) + with connect(store, addr, require_pin=False) as conn: + identity, _ = do_resolve(conn, addr.user) + for index in range(MAX_CHAIN + 1): + if Identity(identity_seed(master, index)).pk == identity: + break + else: + raise SmolError(f"the key bound to {addr.short} is not derived from this master " + f"within {MAX_CHAIN} rotations") + store.set_account(addr) + store.set_cursor(0, bytes(ID_LEN)) + # The accepted set is gone, and an empty one must not replace the server's (§4). + store.run("UPDATE state SET rotations = ?, sync_ok = 0 WHERE id = 1", index) + print(f"restored {addr.short} at rotation {index}\npublic key: {b32(identity)}") + print("Your accepted correspondents are still live on the server; `accept` them " + "again only when you are ready to replace that set.") + return 0 + + # --- CLI -------------------------------------------------------------------- COMMANDS = [ - ("keygen", cmd_keygen, "create an identity", + ("keygen", cmd_keygen, "create a master secret", [(("--force",), {"action": "store_true"})]), ("whoami", cmd_whoami, "show this identity", []), ("trust", cmd_trust, "pin a server's static key", @@ -822,9 +1080,13 @@ COMMANDS = [ (("--force",), {"action": "store_true"})]), ("register", cmd_register, "bind this identity to a username", [(("address",), {}), (("--token",), {"help": "invite token, if required"})]), + ("restore", cmd_restore, "recover local state from the master", + [(("address",), {})]), ("resolve", cmd_resolve, "look up and pin a contact's key", [(("address",), {})]), ("import", cmd_import, "add a contact from a smol:// address", [(("uri",), {})]), ("contacts", cmd_contacts, "list known keys", []), + ("accept", cmd_accept, "admit a contact to the main tier", [(("address",), {})]), + ("block", cmd_block, "withdraw a contact's accept token", [(("address",), {})]), ("send", cmd_send, "seal and deliver a message", [(("address",), {}), (("--subject",), {}), (("--reply-to",), {"metavar": "ID"}), @@ -835,17 +1097,22 @@ COMMANDS = [ "help": "omit the Reply-To field carrying this address (SPEC.md §5.7)"}), (("--no-pad",), {"action": "store_true", "help": "do not pad to 1 KiB"})]), ("fetch", cmd_fetch, "retrieve, verify and acknowledge mail", - [(("--keep",), {"action": "store_true", "help": "do not delete from the server"})]), - ("list", cmd_list, "list stored mail", [(("--sent",), {"action": "store_true"})]), + [(("--keep",), {"action": "store_true", + "help": "do not delete from the server; remember the cursor instead"}), + (("--reset",), {"action": "store_true", "help": "forget the cursor and page again"})]), + ("list", cmd_list, "list stored mail", + [(("--sent",), {"action": "store_true"}), + (("--requests",), {"action": "store_true", + "help": "mail that arrived without an accept token"})]), ("read", cmd_read, "show one message", [(("id",), {}), (("--sent",), {"action": "store_true"})]), - ("rotate", cmd_rotate, "replace this identity, signing the change", []), + ("rotate", cmd_rotate, "advance to the next identity, signing the change", []), ] def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--key", default="identity.key", help="identity seed file") + parser.add_argument("--key", default="identity.key", help="master secret file") parser.add_argument("--db", default="smolmail.db", help="local state and mail") sub = parser.add_subparsers(dest="command", required=True) for name, func, help_text, arguments in COMMANDS: diff --git a/smolmaild.py b/smolmaild.py index d66b4ae..aeaf581 100755 --- a/smolmaild.py +++ b/smolmaild.py @@ -5,7 +5,7 @@ # /// """Smol Mail reference server. -Implements SPEC.md version 1: a store-and-forward mailbox reachable over TCP +Implements SPEC.md version 1.1: a store-and-forward mailbox reachable over TCP with a Noise_NX handshake. The server never sees plaintext, sender identities, or any private key belonging to a user; it stores sealed envelopes addressed to a recipient public key and hands them back to whoever can sign for that key. @@ -19,6 +19,7 @@ from __future__ import annotations import argparse import base64 import hashlib +import hmac import logging import os import socket @@ -45,7 +46,9 @@ NOISE_PROTOCOL = b"Noise_NX_25519_ChaChaPoly_SHA256" PROLOGUE = b"smolmail/1" LABEL_AUTH = b"smolmail/1 auth" LABEL_ID = b"smolmail/1 id" +LABEL_MAC = b"smolmail/1 mac" LABEL_ROTATE = b"smolmail/1 rotate" +LABEL_REGISTER = b"smolmail/1 register" OP_AUTH = 0x00 OP_RESOLVE = 0x01 @@ -70,16 +73,24 @@ ENVELOPE_MAGIC = b"SMOL" ENVELOPE_VERSION = 1 ENVELOPE_HEADER = 69 # magic 4 + version 1 + to 32 + epk 32 ENVELOPE_MIN = ENVELOPE_HEADER + 16 # + Poly1305 tag -ID_LEN = 16 +ID_LEN = 32 KEY_LEN = 32 -CERT_LEN = 136 # old_pub 32 + new_pub 32 + time 8 + signature 64 +SIG_LEN = 64 +TOKEN_LEN = 32 # §5.8 accept token, and the MAC derived from it +CERT_LEN = 200 # old_pub 32 + new_pub 32 + time 8 + two signatures MAX_CHAIN = 16 +SEPARATORS = "._-" + +TIER_MAIN = 0 +TIER_REQUESTS = 1 +FLAG_REQUESTS = 0x01 # §6.1 record flags DEFAULT_PORT = 1961 MAX_FRAME = 1 << 20 # §4 application frame ceiling NOISE_MAX = 65535 # §4 Noise message ceiling NOISE_PAYLOAD = NOISE_MAX - 16 # minus the AEAD tag FETCH_BUDGET = 512 * 1024 # §6.1, must stay under MAX_FRAME +RECORD_OVERHEAD = ID_LEN + 8 + 1 + 4 # id + received_at + flags + env_len IDLE_TIMEOUT = 120.0 PURGE_INTERVAL = 60.0 @@ -137,17 +148,19 @@ class Reader: def valid_username(name: str) -> bool: - """§3: 1-63 bytes of [a-z0-9._-], not starting or ending with a separator.""" + """§3: 1-63 bytes of [a-z0-9._-], alphanumeric ends, no adjacent separators.""" if not 1 <= len(name) <= 63: return False - if not all(c.isdigit() or ("a" <= c <= "z") or c in "._-" for c in name): + if not all("0" <= c <= "9" or "a" <= c <= "z" or c in SEPARATORS for c in name): return False - return name[0] not in "._-" and name[-1] not in "._-" + if name[0] in SEPARATORS or name[-1] in SEPARATORS: + return False + return not any(a in SEPARATORS and b in SEPARATORS for a, b in zip(name, name[1:])) def message_id(envelope: bytes) -> bytes: """§5.4. Derived from the envelope, so a sender cannot choose it.""" - return hashlib.sha256(LABEL_ID + envelope).digest()[:ID_LEN] + return hashlib.sha256(LABEL_ID + envelope).digest() def verify(pubkey: bytes, signature: bytes, message: bytes) -> bool: @@ -179,14 +192,30 @@ CREATE TABLE IF NOT EXISTS rotations ( cert BLOB NOT NULL, PRIMARY KEY (username, seq) ); +-- Accept tokens (§5.8): opaque secrets the mailbox owner uploads with AUTH. +-- The server can match them against a message's MAC and nothing else; it never +-- learns which correspondent a token belongs to. +CREATE TABLE IF NOT EXISTS accepted ( + username TEXT NOT NULL, + token BLOB NOT NULL, + PRIMARY KEY (username, token) +); CREATE TABLE IF NOT EXISTS messages ( id BLOB PRIMARY KEY, recipient BLOB NOT NULL, received_at INTEGER NOT NULL, + tier INTEGER NOT NULL, envelope BLOB NOT NULL ); +-- (received_at, id) is FETCH's sort and cursor order (§6.1). CREATE INDEX IF NOT EXISTS messages_by_recipient - ON messages (recipient, received_at); + ON messages (recipient, received_at, id); +-- Identifiers of every envelope accepted within the retention window, so an +-- envelope captured and resent after DELETE does not reappear (§10). +CREATE TABLE IF NOT EXISTS seen ( + id BLOB PRIMARY KEY, + at INTEGER NOT NULL +); """ @@ -277,39 +306,78 @@ class Store: except sqlite3.IntegrityError: return False - def mailbox_bytes(self, keys: list[bytes]) -> int: + def accepted_tokens(self, username: str) -> list[bytes]: + return [ + row[0] + for row in self.db.execute( + "SELECT token FROM accepted WHERE username = ?", (username,) + ) + ] + + def set_accepted(self, username: str, tokens: list[bytes]) -> int: + """§4: an AUTH with sync = 1 replaces the whole set, which is how a + token is both added and removed.""" + with self.db: + self.db.execute("DELETE FROM accepted WHERE username = ?", (username,)) + self.db.executemany( + "INSERT OR IGNORE INTO accepted (username, token) VALUES (?, ?)", + [(username, token) for token in tokens], + ) + return self.count_accepted(username) + + def count_accepted(self, username: str) -> int: + return self.db.execute( + "SELECT COUNT(*) FROM accepted WHERE username = ?", (username,) + ).fetchone()[0] + + def tombstoned(self, mid: bytes) -> bool: + return ( + self.db.execute("SELECT 1 FROM seen WHERE id = ?", (mid,)).fetchone() + is not None + ) + + def mailbox_bytes(self, keys: list[bytes], tier: int) -> int: marks = ",".join("?" * len(keys)) row = self.db.execute( f"SELECT COALESCE(SUM(LENGTH(envelope)), 0) FROM messages " - f"WHERE recipient IN ({marks})", - keys, + f"WHERE tier = ? AND recipient IN ({marks})", + [tier] + keys, ).fetchone() return row[0] - def store_message(self, mid: bytes, recipient: bytes, envelope: bytes) -> None: + def store_message(self, mid: bytes, recipient: bytes, tier: int, envelope: bytes) -> None: with self.db: + now = int(time.time()) self.db.execute( - "INSERT OR IGNORE INTO messages (id, recipient, received_at, envelope) " - "VALUES (?, ?, ?, ?)", - (mid, recipient, int(time.time()), envelope), + "INSERT OR IGNORE INTO messages " + "(id, recipient, received_at, tier, envelope) VALUES (?, ?, ?, ?, ?)", + (mid, recipient, now, tier, envelope), + ) + self.db.execute( + "INSERT OR IGNORE INTO seen (id, at) VALUES (?, ?)", (mid, now) ) - def pending(self, keys: list[bytes], budget: int) -> list[tuple[bytes, int, bytes]]: + def pending( + self, keys: list[bytes], after_time: int, after_id: bytes, budget: int + ) -> list[tuple[bytes, int, int, bytes]]: marks = ",".join("?" * len(keys)) rows = self.db.execute( - f"SELECT id, received_at, envelope FROM messages " - f"WHERE recipient IN ({marks}) ORDER BY received_at, id", - keys, + f"SELECT id, received_at, tier, envelope FROM messages " + f"WHERE recipient IN ({marks}) " + f"AND (received_at > ? OR (received_at = ? AND id > ?)) " + f"ORDER BY received_at, id", + keys + [after_time, after_time, after_id], ) - out: list[tuple[bytes, int, bytes]] = [] + out: list[tuple[bytes, int, int, bytes]] = [] used = 0 - for mid, received_at, envelope in rows: - # Always return at least one message, even if it alone exceeds the + for mid, received_at, tier, envelope in rows: + size = RECORD_OVERHEAD + len(envelope) + # Always return at least one record, even if it alone exceeds the # budget, so an oversized envelope cannot wedge a mailbox shut. - if out and used + len(envelope) > budget: + if out and used + size > budget: break - out.append((mid, received_at, envelope)) - used += len(envelope) + out.append((mid, received_at, tier, envelope)) + used += size return out def delete(self, keys: list[bytes], ids: list[bytes]) -> int: @@ -323,10 +391,16 @@ class Store: ) return cur.rowcount - def purge(self, older_than: int) -> int: + def purge(self, main_cutoff: int, requests_cutoff: int, seen_cutoff: int) -> int: with self.db: - cur = self.db.execute("DELETE FROM messages WHERE received_at < ?", (older_than,)) - return cur.rowcount + cur = self.db.execute( + "DELETE FROM messages WHERE (tier = ? AND received_at < ?) " + "OR (tier = ? AND received_at < ?)", + (TIER_MAIN, main_cutoff, TIER_REQUESTS, requests_cutoff), + ) + gone = cur.rowcount + self.db.execute("DELETE FROM seen WHERE at < ?", (seen_cutoff,)) + return gone # --------------------------------------------------------------------------- @@ -335,10 +409,7 @@ class Store: class RateLimiter: - """Fixed-window per-IP counter, the whole of §10's abuse control. - - A server cannot see senders, so quotas, size caps and this are all it has. - """ + """Fixed-window counter, keyed by IP address or by accept token (§10).""" def __init__(self, limit: int, window: float = 60.0) -> None: self.limit = limit @@ -346,25 +417,25 @@ class RateLimiter: self.lock = threading.Lock() self.hits: dict[str, tuple[float, int]] = {} - def allow(self, ip: str) -> bool: + def allow(self, key: str) -> bool: if self.limit <= 0: return True now = time.monotonic() with self.lock: - start, count = self.hits.get(ip, (now, 0)) + start, count = self.hits.get(key, (now, 0)) if now - start >= self.window: start, count = now, 0 if count >= self.limit: return False - self.hits[ip] = (start, count + 1) + self.hits[key] = (start, count + 1) if len(self.hits) > 4096: self._evict(now) return True def _evict(self, now: float) -> None: - stale = [ip for ip, (start, _) in self.hits.items() if now - start >= self.window] - for ip in stale: - del self.hits[ip] + stale = [key for key, (start, _) in self.hits.items() if now - start >= self.window] + for key in stale: + del self.hits[key] # --------------------------------------------------------------------------- @@ -475,7 +546,9 @@ class Session: def op_auth(self, r: Reader) -> tuple[int, bytes]: username = r.take(r.u8()).decode("utf-8", "strict") identity = r.take(KEY_LEN) - signature = r.take(64) + signature = r.take(SIG_LEN) + sync = r.u8() + tokens = [r.take(TOKEN_LEN) for _ in range(r.u16())] r.done() bound = self.store.identity_of(username) # A wrong username and a wrong signature are both AUTH_FAILED: telling @@ -484,8 +557,16 @@ class Session: return AUTH_FAILED, b"" if not verify(identity, signature, LABEL_AUTH + self.handshake_hash): return AUTH_FAILED, b"" + if sync not in (0, 1) or (sync == 0 and tokens): + return MALFORMED, b"" + # Refused before authenticating, so the client has to trim and retry + # rather than silently running with a truncated set (§4). + if len(tokens) > self.server.max_accepted: + return TOO_LARGE, b"" self.username = username - return OK, b"" + held = self.store.set_accepted(username, tokens) if sync else \ + self.store.count_accepted(username) + return OK, struct.pack(">H", held) def op_resolve(self, r: Reader) -> tuple[int, bytes]: username = r.take(r.u8()).decode("utf-8", "strict") @@ -496,7 +577,22 @@ class Session: chain = self.store.chain(username) return OK, identity + bytes([len(chain)]) + b"".join(chain) + def match_token(self, username: str, mid: bytes, mac: bytes) -> bytes | None: + """§5.8. Trial-match the MAC against the mailbox's tokens. + + Bounded by --max-accepted, so an unmatchable MAC costs a known amount + of HMAC rather than an unbounded scan. + """ + if len(mac) != TOKEN_LEN: + return None + for token in self.store.accepted_tokens(username): + want = hmac.new(token, LABEL_MAC + mid, hashlib.sha256).digest() + if hmac.compare_digest(want, mac): + return token + return None + def op_send(self, r: Reader) -> tuple[int, bytes]: + mac = r.take(r.u8()) envelope = r.rest() if not self.server.send_limiter.allow(self.peer_ip): return RATE_LIMITED, b"" @@ -510,22 +606,40 @@ class Session: username = self.store.username_for_key(recipient) if username is None: return UNKNOWN_USER, b"" + mid = message_id(envelope) + # A resend is answered with the same id and stores nothing, whether the + # original is still here or was deleted (§10). + if self.store.tombstoned(mid): + return OK, mid + # An unmatched MAC is not an error: SEND must not reveal whether a + # token is still accepted (§5.8). + token = self.match_token(username, mid, mac) + if token is None: + tier, quota = TIER_REQUESTS, self.server.requests_quota + else: + tier, quota = TIER_MAIN, self.server.quota + if not self.server.token_limiter.allow(token.hex()): + return RATE_LIMITED, b"" keys = self.store.keys_of(username) - if self.store.mailbox_bytes(keys) + len(envelope) > self.server.quota: + if self.store.mailbox_bytes(keys, tier) + len(envelope) > quota: return QUOTA_EXCEEDED, b"" # The ciphertext is never inspected; the server cannot read it. - mid = message_id(envelope) - self.store.store_message(mid, recipient, envelope) + self.store.store_message(mid, recipient, tier, envelope) return OK, mid def op_fetch(self, r: Reader) -> tuple[int, bytes]: + after_time = r.i64() + after_id = r.take(ID_LEN) r.done() assert self.username is not None keys = self.store.keys_of(self.username) - records = self.store.pending(keys, FETCH_BUDGET) + records = self.store.pending(keys, after_time, after_id, FETCH_BUDGET) out = [struct.pack(">H", len(records))] - for mid, received_at, envelope in records: - out.append(mid + struct.pack(">qI", received_at, len(envelope)) + envelope) + for mid, received_at, tier, envelope in records: + flags = FLAG_REQUESTS if tier == TIER_REQUESTS else 0 + out.append( + mid + struct.pack(">qBI", received_at, flags, len(envelope)) + envelope + ) return OK, b"".join(out) def op_delete(self, r: Reader) -> tuple[int, bytes]: @@ -544,6 +658,7 @@ class Session: def op_register(self, r: Reader) -> tuple[int, bytes]: username = r.take(r.u8()).decode("utf-8", "strict") identity = r.take(KEY_LEN) + signature = r.take(SIG_LEN) token = r.take(r.u8()) cert = r.take(r.u8()) r.done() @@ -554,6 +669,14 @@ class Session: Ed25519PublicKey.from_public_bytes(identity) except ValueError: return MALFORMED, b"" + # §6.1 proof of possession, bound to this server's static key so the + # attestation cannot be replayed elsewhere. + if not verify( + identity, + signature, + LABEL_REGISTER + self.server.static_pub + username.encode() + identity, + ): + return AUTH_FAILED, b"" expected = self.server.invite_token if expected is not None and token != expected: return NOT_PERMITTED, b"" @@ -565,7 +688,8 @@ class Session: if len(cert) != CERT_LEN: return MALFORMED, b"" - old_pub, new_pub, when, signature = cert[:32], cert[32:64], cert[64:72], cert[72:] + old_pub, new_pub, when = cert[:32], cert[32:64], cert[64:72] + sig_old, sig_new = cert[72:136], cert[136:200] if new_pub != identity: return MALFORMED, b"" bound = self.store.identity_of(username) @@ -574,7 +698,10 @@ class Session: if bound != old_pub: # Only the currently bound key may hand the username on (§7). return NOT_PERMITTED, b"" - if not verify(old_pub, signature, LABEL_ROTATE + old_pub + new_pub + when): + # Both keys sign: the old one alone could otherwise rotate to a key + # nobody controls (§7). + signed = LABEL_ROTATE + username.encode() + old_pub + new_pub + when + if not verify(old_pub, sig_old, signed) or not verify(new_pub, sig_new, signed): return AUTH_FAILED, b"" chain = self.store.chain(username) if len(chain) >= MAX_CHAIN: @@ -638,6 +765,14 @@ class Handler(socketserver.BaseRequestHandler): pass +def static_public(static_key: bytes) -> bytes: + return ( + X25519PrivateKey.from_private_bytes(static_key) + .public_key() + .public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + ) + + class MailServer(socketserver.ThreadingTCPServer): allow_reuse_address = True daemon_threads = True @@ -645,13 +780,18 @@ class MailServer(socketserver.ThreadingTCPServer): def __init__(self, address, args: argparse.Namespace, static_key: bytes) -> None: super().__init__(address, Handler) self.static_key = static_key + self.static_pub = static_public(static_key) self.store = Store(args.db) self.max_envelope = args.max_envelope self.quota = args.quota + self.requests_quota = args.requests_quota self.retention = args.retention_days * 86400 + self.requests_retention = args.requests_retention_days * 86400 + self.max_accepted = args.max_accepted self.invite_token = args.invite_token.encode() if args.invite_token else None self.conn_limiter = RateLimiter(args.rate_connections) self.send_limiter = RateLimiter(args.rate_sends) + self.token_limiter = RateLimiter(args.rate_token) threading.Thread(target=self._purge_loop, daemon=True).start() def _purge_loop(self) -> None: @@ -659,7 +799,14 @@ class MailServer(socketserver.ThreadingTCPServer): while True: time.sleep(PURGE_INTERVAL) try: - gone = store.purge(int(time.time()) - self.retention) + now = int(time.time()) + # Tombstones outlive both tiers, so a replay cannot slip in + # between a message expiring and its id being forgotten. + gone = store.purge( + now - self.retention, + now - self.requests_retention, + now - max(self.retention, self.requests_retention), + ) if gone: log.info("expired %d message(s)", gone) except Exception: @@ -707,13 +854,8 @@ def cmd_serve(args: argparse.Namespace) -> int: return 1 server = MailServer((args.host, args.port), args, static_key) - public = ( - X25519PrivateKey.from_private_bytes(static_key) - .public_key() - .public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) - ) log.info("listening on %s:%d", args.host, args.port) - log.info("server public key: %s", b32(public)) + log.info("server public key: %s", b32(server.static_pub)) try: server.serve_forever() except KeyboardInterrupt: @@ -739,13 +881,20 @@ def main(argv: list[str] | None = None) -> int: serve.add_argument("--db", default="mail.db") serve.add_argument("--host", default="127.0.0.1") serve.add_argument("--port", type=int, default=DEFAULT_PORT) - serve.add_argument("--max-envelope", type=int, default=1 << 20, metavar="BYTES") + serve.add_argument("--max-envelope", type=int, default=768 << 10, metavar="BYTES") serve.add_argument("--quota", type=int, default=64 << 20, metavar="BYTES") + serve.add_argument("--requests-quota", type=int, default=2 << 20, metavar="BYTES", + help="quota for mail arriving without an accept token") serve.add_argument("--retention-days", type=int, default=30) + serve.add_argument("--requests-retention-days", type=int, default=7) + serve.add_argument("--max-accepted", type=int, default=1024, metavar="TOKENS", + help="accept tokens a mailbox may hold (SPEC.md §5.8)") serve.add_argument("--invite-token", default=None, help="require this token to register; open registration if unset") serve.add_argument("--rate-connections", type=int, default=120, metavar="PER_MIN") serve.add_argument("--rate-sends", type=int, default=60, metavar="PER_MIN") + serve.add_argument("--rate-token", type=int, default=120, metavar="PER_MIN", + help="sends per minute per accept token") serve.set_defaults(func=cmd_serve) args = parser.parse_args(argv)