From 5dda6567af6d9901691271de5650d10d6d08f0ef Mon Sep 17 00:00:00 2001 From: randogoth Date: Sat, 3 Oct 2026 18:03:43 +0300 Subject: [PATCH] feat: add sender/recipient accept gates, security+stress test harnesses, nix flake, and docs --- README.md | 71 +++++ devbox.json | 4 +- docker-compose.yml | 2 + flake.lock | 78 +++++ flake.nix | 231 ++++++++++++++ scripts/security_test.py | 480 ++++++++++++++++++++++++++++ scripts/stress_test.py | 532 ++++++++++++++++++++++++++++++++ src/pneumatube/__main__.py | 13 +- src/pneumatube/config.py | 21 +- src/pneumatube/email_to_smol.py | 77 ++++- src/pneumatube/lookup_server.py | 24 +- src/pneumatube/smol_to_email.py | 39 ++- src/pneumatube/smtp_client.py | 8 + src/pneumatube/smtp_server.py | 17 +- tests/test_email_to_smol.py | 53 ++++ tests/test_lookup_server.py | 28 +- tests/test_smol_to_email.py | 65 ++++ tests/test_smtp_server.py | 65 +++- 18 files changed, 1764 insertions(+), 44 deletions(-) create mode 100644 README.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 scripts/security_test.py create mode 100644 scripts/stress_test.py create mode 100644 tests/test_smol_to_email.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..3c2dc37 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# pneumatube + +A gateway bridging a smolmail account and an SMTP mailbox: smolmail messages carrying an `Email-To` frontmatter field are relayed to that address over SMTP, and mail sent to `@` is relayed as a smolmail message to `` if that user is registered on the configured smolmail server. See `src/pneumatube/__main__.py` for the full environment variable list, `docker-compose.yml`/`docker/postfix/` for a Postfix-fronted container deployment, and `flake.nix` for a NixOS deployment with no containers. + +## Using the gateway as a registered smolmail user + +This assumes a smolmail account already exists, and that the gateway's smolmail address (e.g. `gateway@smol.example.org`) and the email domain it answers to (e.g. `example.com`) are known — the operator can provide both. Everything below uses the reference client, `smolmail.py`, the same way it's used for anything else. + +### Sending email through the gateway + +Send the gateway a message with an `Email-To` frontmatter field naming where it should go: + +``` +echo "Hi there, this is coming in from smolmail." | \ + smolmail.py send gateway@smol.example.org --subject "Hello" --header "Email-To: someone@example.com" +``` + +A few things that aren't obvious from the outside: + +- **No `Email-To`, no relay.** A message without it is never forwarded — it just sits there for the operator to notice, never bounced back to the sender. +- **The recipient won't see the sender's smolmail identity.** The email arrives `From:` the gateway's own fixed address, the same for every user of that gateway, not something tied to whoever sent it. A sender who wants to be identified needs to say so in the body. +- **By default, a sender has to be accepted first.** Most gateways only relay mail from senders they've explicitly accepted (SPEC.md §5.8) — the sender needs to ask the operator to accept their address. One wrinkle worth knowing about: being accepted isn't quite enough on its own. The accept token that makes the sender's *next* message count only arrives once the gateway has sent something back, so if a message still doesn't go through right after being accepted, the operator needs to send a quick reply first. (Some gateways are run deliberately open with this requirement turned off — in that case none of this applies and anyone can use it.) + +### Receiving email through the gateway + +Mail sent to `@example.com` reaches that registered user as a smolmail message, but by default only once the user has accepted the gateway: + +``` +smolmail.py accept gateway@smol.example.org +echo "hi" | smolmail.py send gateway@smol.example.org +``` + +Both steps matter, not just the first. Accepting the gateway only changes the user's own server's tiering for mail from it; the gateway has no way to know that happened until it sees it reflected in a message from the user (same §5.8 mechanism as above), which means sending it at least one message. Until both are done, inbound email addressed to that user is rejected outright — not queued, not quietly demoted to some low-priority folder, just refused before it's even accepted. + +Once that's done, the smolmail messages received this way won't say who originally sent the email — that's deliberate, since an email's claimed sender is never something the gateway can verify, so it doesn't forward a claim it can't back up. If the original sender identified themselves, that'll only show up in whatever text they wrote, not in a field that can be trusted on its own. + +(Again: some gateways run with this requirement turned off, relaying to anyone registered regardless of acceptance — it depends on how that gateway is configured.) + +## Test Results + +### Unit tests + +`uv run pytest` (or `devbox run test`) — 27/27 passing. Covers address validation and header-injection rejection (`smtp_client`), frontmatter sanitization (`email_to_smol`), MIME body extraction, the accepted-recipient gate (`email_to_smol`), per-recipient LMTP reply assembly and the accepted-recipient RCPT reject (`smtp_server`), the Postfix `tcp_table(5)` lookup protocol including the same gate (`lookup_server`), and failure isolation plus the accepted-sender tier gate in the smolmail → email poll loop (`smol_to_email`). + +### Stress test (`scripts/stress_test.py`, `devbox run stress`) + +Concurrent load against pneumatube directly, and through the real Postfix container rig. + +- **Healthy at low-to-moderate concurrency**: ~5 concurrent senders sustains ~140 msg/s with zero errors on both directions. +- **Found and fixed**: `EmailToSmolBridge.user_exists` resolved a recipient's identity at RCPT time and discarded the result, so `forward()` resolved the same username again moments later — doubling the network round trips (each a full TCP + Noise handshake, smolmail.py has no connection pooling) for every message. Fixed by caching the resolved identity as a contact, the same trust-on-first-use pattern smolmail.py's own CLI already uses. +- **A real scaling ceiling, not a pneumatube bug**: at concurrency ≈30, throughput collapses to mass client-side timeouts. The likely cause (via code inspection, not a clean isolated reproduction): smolmail's reference server (`smolmaild.py`) uses Python's `socketserver.ThreadingTCPServer` with its default backlog of 5, uncorrected, compounding with pneumatube's own single-worker-thread serialization (`worker.py`, by design, for sqlite thread-safety). Worth pinning down further before relying on sustained throughput much above a handful of messages/sec against the reference server. +- Both targets pass cleanly at healthy concurrency: correctness verified (every expected message delivered exactly once, no loss or duplication) on the direct path and through the full Postfix → LMTP → pneumatube path. + +### Security test (`scripts/security_test.py`, `devbox run security`) + +11 checks: 8 PASS (defense verified), 3 RISK (confirmed, accepted design tradeoffs — not bugs, but worth knowing they're real), 0 BUG. + +**PASS:** +- Address validation (`smtp_client.validate`) rejects every CR/LF/NUL injection variant tried, not just the literal `\r\n` case. +- The frontmatter sanitizer (`email_to_smol._single_line`) strips CR/LF from every payload tried. +- Python's `EmailMessage` rejects embedded CR/LF in header values — confirmed as a real backstop, since `smtp_client.send()` has no explicit sanitizer of its own and otherwise relies entirely on smolmail's frontmatter format structurally preventing multi-line values. +- `extract_text` survives a battery of malformed MIME (empty messages, non-UTF8 bytes, deeply nested multipart, multipart with no text part) without raising or stalling. +- **Found and fixed**: the smolmail → email poll loop had no per-message failure isolation — unlike the email → smolmail side (`smtp_server.py`, already correct), one message raising an exception `_relay()`'s own except clause didn't cover would propagate out of `poll_once()` entirely. Since the poisoned message is only ever deleted on success, it would sort first on every subsequent poll and jam the entire direction forever. Not confirmed reachable via the documented frontmatter format today, but fixed regardless as defense-in-depth with a regression test proving later messages still get processed after an earlier one fails. +- **Found and fixed**: the smolmail → email direction relayed *any* sender by default, including first-contact senders with no accept token — confirmed live (15/15 first-contact sends reached the outbound sink). Fixed with `PNEUMATUBE_REQUIRE_ACCEPTED_SENDER` (default `1`): `poll_once()` now polls only the main/accepted tier (SPEC.md §5.8), so a sender the gateway account hasn't explicitly accepted is left alone. Re-verified live: 0/15 first-contact sends reached the sink with the new default. Note the gateway never replies to smolmail senders, so accepting someone is a two-step process — `smolmail.py accept
` alone doesn't retroactively unblock a message already sitting in the local inbox, and the sender's *next* message won't carry a valid token either until the operator also sends them one message back (that's how the token actually reaches them, per the protocol). +- **Found and fixed**: `Email-From` (the email → smolmail direction's frontmatter field carrying the original email's `MAIL FROM`) has been removed entirely. `MAIL FROM` is never authenticated, so forwarding it into a field that looks like gateway-attested metadata let any email sender forge an apparently-trustworthy "from" inside a signed smolmail message. Re-verified live with a forged `security-team@trusted-bank.example`: the relayed message now carries no field reflecting it at all. +- **Found and fixed**: the email → smolmail direction relayed to *any* registered recipient by default, with no throttling and no way for a recipient to opt out. Fixed with `PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT` (default `1`): reuses the real smolmail accept-token mechanism (SPEC.md §5.8) rather than inventing a new one — pneumatube already needs to know locally whether it holds a token for a recipient to attach a valid accept MAC on `cmd_send`, and that same fact is exactly "has this recipient accepted the gateway". A recipient who hasn't is now rejected live at RCPT TO (an all-or-nothing gate, not a demotion: without it, the message would still land in the recipient's own requests tier; with it, they get nothing at all). Checked in both `handle_RCPT` and the Postfix lookup service, so Postfix itself rejects live rather than generating a bounce later. Re-verified live: a registered-but-never-accepted recipient is now rejected with `outcome=rejected`, where it previously always succeeded. + +**RISK (confirmed, accepted tradeoffs):** +- The Postfix lookup service unambiguously reveals which usernames exist, with no auth or rate limiting of its own — correct and necessary for `check_recipient_access` to work, but only safe because it's loopback-only. This requirement is load-bearing, not cosmetic: never expose `PNEUMATUBE_LOOKUP_HOST`/`PNEUMATUBE_LOOKUP_PORT` beyond Postfix. +- **Open relay, smolmail → email, opt-in**: setting `PNEUMATUBE_REQUIRE_ACCEPTED_SENDER=0` deliberately restores the open-bridge behavior above (confirmed live: 15/15 first-contact sends relayed) — an accepted tradeoff for an operator who wants a public bridge, not a default. +- **Open relay, email → smolmail, opt-in**: setting `PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT=0` deliberately restores the "relay to anyone" behavior above (confirmed live: 15/15 distinct-sender messages relayed to an unaccepted recipient) — same tradeoff, operator's explicit choice. Note there is still no *per-sender* throttling in either mode: the gate is per-recipient (have they accepted the gateway at all), not per-sender, so an accepted recipient remains reachable by any external sender once opted in. diff --git a/devbox.json b/devbox.json index 13ff5ea..fe7cf07 100644 --- a/devbox.json +++ b/devbox.json @@ -10,7 +10,9 @@ "scripts": { "sync": ["uv sync"], "test": ["uv run pytest"], - "run": ["uv run pneumatube"] + "run": ["uv run pneumatube"], + "stress": ["uv run python scripts/stress_test.py"], + "security": ["uv run python scripts/security_test.py"] } } } diff --git a/docker-compose.yml b/docker-compose.yml index 1ac9cef..88bf077 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,8 @@ services: PNEUMATUBE_LISTEN_PORT: "2424" PNEUMATUBE_LOOKUP_HOST: 0.0.0.0 PNEUMATUBE_LOOKUP_PORT: "2526" + PNEUMATUBE_REQUIRE_ACCEPTED_SENDER: ${PNEUMATUBE_REQUIRE_ACCEPTED_SENDER:-1} + PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT: ${PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT:-1} PNEUMATUBE_SMTP_RELAY_HOST: ${PNEUMATUBE_SMTP_RELAY_HOST:?set an outbound relay host} PNEUMATUBE_SMTP_RELAY_PORT: ${PNEUMATUBE_SMTP_RELAY_PORT:-587} PNEUMATUBE_SMTP_RELAY_USER: ${PNEUMATUBE_SMTP_RELAY_USER:-} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..8c82f08 --- /dev/null +++ b/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1790822859, + "narHash": "sha256-69xHQhAeMAD2wDXO7T2pcOZIF9Sga2W+JkmY2a11Ops=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c59305bab2065cfecc4944690d9eedbb56f3a9fa", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "smolmail": "smolmail" + } + }, + "smolmail": { + "flake": false, + "locked": { + "lastModified": 1790747231, + "narHash": "sha256-C4Lfqr9hFMQhX17Fe6qP5a03pbin8ZcVvYuk9jnY3eA=", + "ref": "refs/heads/main", + "rev": "864daa0bad30c27976a24391fc2db7e848a749d4", + "revCount": 24, + "type": "git", + "url": "ssh://git@code.randogoth.com:2222/randogoth/smolmail.git" + }, + "original": { + "type": "git", + "url": "ssh://git@code.randogoth.com:2222/randogoth/smolmail.git" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..0341a80 --- /dev/null +++ b/flake.nix @@ -0,0 +1,231 @@ +{ + description = "Gateway bridging a smolmail account and an SMTP mailbox"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + smolmail = { + url = "git+ssh://git@code.randogoth.com:2222/randogoth/smolmail.git"; + flake = false; + }; + }; + + outputs = { self, nixpkgs, flake-utils, smolmail }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; }; + python = pkgs.python313; + + # smolmail.py is a `uv run --script` file with PEP 723 inline deps, + # not an installed package -- loaded by path at runtime (vendor.py), + # same as the devbox workflow's sibling checkout. smolmailEnv gives + # the `smolmail` CLI a working interpreter for one-time account + # bootstrap (keygen/trust/register) without needing uv on the host. + smolmailEnv = python.withPackages (ps: [ ps.pynacl ps.noiseprotocol ]); + smolmailCli = pkgs.writeShellScriptBin "smolmail" '' + exec ${smolmailEnv}/bin/python3 ${smolmail}/smolmail.py "$@" + ''; + + pneumatubePkg = python.pkgs.buildPythonApplication { + pname = "pneumatube"; + version = "0.1.0"; + pyproject = true; + src = ./.; + build-system = [ python.pkgs.setuptools python.pkgs.wheel ]; + dependencies = with python.pkgs; [ pynacl noiseprotocol aiosmtpd ]; + # No unit tests need the real smolmail checkout; the ones that do + # (test_smtp_server, test_lookup_server) stub the bridge instead. + nativeCheckInputs = [ python.pkgs.pytestCheckHook ]; + }; + in + { + packages.default = pneumatubePkg; + packages.smolmail = smolmailCli; + apps.smolmail = { + type = "app"; + program = "${smolmailCli}/bin/smolmail"; + }; + + devShells.default = pkgs.mkShell { + packages = [ python pkgs.uv ]; + }; + } + ) // { + nixosModules.default = { config, lib, pkgs, ... }: + let + cfg = config.services.pneumatube; + stateDir = "/var/lib/pneumatube"; + in + { + options.services.pneumatube = with lib; { + enable = mkEnableOption "the pneumatube smolmail/SMTP gateway"; + + package = mkOption { + type = types.package; + default = self.packages.${pkgs.system}.default; + description = "pneumatube package to run."; + }; + + smolmailSrc = mkOption { + type = types.path; + default = smolmail; + description = '' + smolmail checkout providing smolmail.py, loaded by path at + runtime (PNEUMATUBE_SMOLMAIL_REPO) rather than installed as + a package. + ''; + }; + + smolmailDomain = mkOption { + type = types.str; + description = '' + The smolmail server address used in "user@..." addresses for + both directions (e.g. "smol.example.org" or "host:port"). + ''; + }; + + emailDomain = mkOption { + type = types.str; + description = "Email domain this gateway accepts inbound mail for."; + }; + + fetchInterval = mkOption { + type = types.ints.positive; + default = 30; + description = "Seconds between smolmail inbox polls."; + }; + + listenHost = mkOption { + type = types.str; + default = "127.0.0.1"; + description = "Bind address for the LMTP delivery backend."; + }; + + listenPort = mkOption { + type = types.port; + default = 2424; + }; + + lookupHost = mkOption { + type = types.str; + default = "127.0.0.1"; + description = '' + Bind address for the Postfix check_recipient_access lookup + service (tcp_table(5)), which turns an unknown recipient + into a live RCPT-time reject instead of a queued-then- + bounced message. + ''; + }; + + lookupPort = mkOption { + type = types.port; + default = 2526; + }; + + fromAddress = mkOption { + type = types.str; + description = "From: address used on mail forwarded to SMTP."; + }; + + relayHost = mkOption { + type = types.str; + description = "Outbound SMTP relay for the smolmail -> email direction."; + }; + + relayPort = mkOption { + type = types.port; + default = 587; + }; + + relayUser = mkOption { + type = types.nullOr types.str; + default = null; + }; + + relayPasswordFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Path, outside the Nix store, to a systemd EnvironmentFile + containing a single `PNEUMATUBE_SMTP_RELAY_PASSWORD=...` + line -- e.g. provisioned by sops-nix or agenix. Never set + the password through another option: it would end up + world-readable in the Nix store. + ''; + }; + + relayStartTls = mkOption { + type = types.bool; + default = true; + }; + + postfix.enable = mkEnableOption '' + a local Postfix configured to front this gateway over LMTP: + relay_domains/transport for emailDomain, and + check_recipient_access against the lookup service above. Set + to false to wire up Postfix (on this host or elsewhere) + yourself + ''; + }; + + config = lib.mkIf cfg.enable { + systemd.services.pneumatube = { + description = "pneumatube smolmail/SMTP gateway"; + after = [ "network.target" ] ++ lib.optional cfg.postfix.enable "postfix.service"; + wantedBy = [ "multi-user.target" ]; + environment = { + PNEUMATUBE_SMOLMAIL_REPO = "${cfg.smolmailSrc}"; + PNEUMATUBE_SMOLMAIL_KEY = "${stateDir}/gateway.key"; + PNEUMATUBE_SMOLMAIL_DB = "${stateDir}/gateway.db"; + PNEUMATUBE_SMOLMAIL_DOMAIN = cfg.smolmailDomain; + PNEUMATUBE_EMAIL_DOMAIN = cfg.emailDomain; + PNEUMATUBE_FETCH_INTERVAL = toString cfg.fetchInterval; + PNEUMATUBE_LISTEN_PROTOCOL = "lmtp"; + PNEUMATUBE_LISTEN_HOST = cfg.listenHost; + PNEUMATUBE_LISTEN_PORT = toString cfg.listenPort; + PNEUMATUBE_LOOKUP_HOST = cfg.lookupHost; + PNEUMATUBE_LOOKUP_PORT = toString cfg.lookupPort; + PNEUMATUBE_SMTP_RELAY_HOST = cfg.relayHost; + PNEUMATUBE_SMTP_RELAY_PORT = toString cfg.relayPort; + PNEUMATUBE_SMTP_RELAY_STARTTLS = if cfg.relayStartTls then "1" else "0"; + PNEUMATUBE_GATEWAY_FROM_ADDRESS = cfg.fromAddress; + } // lib.optionalAttrs (cfg.relayUser != null) { + PNEUMATUBE_SMTP_RELAY_USER = cfg.relayUser; + }; + serviceConfig = { + ExecStart = "${cfg.package}/bin/pneumatube"; + DynamicUser = true; + StateDirectory = "pneumatube"; + WorkingDirectory = stateDir; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + Restart = "on-failure"; + RestartSec = "5s"; + } // lib.optionalAttrs (cfg.relayPasswordFile != null) { + EnvironmentFile = cfg.relayPasswordFile; + }; + }; + + services.postfix = lib.mkIf cfg.postfix.enable { + enable = true; + settings.main = { + relay_domains = [ cfg.emailDomain ]; + smtpd_relay_restrictions = [ "permit_mynetworks" "reject_unauth_destination" ]; + smtpd_recipient_restrictions = + [ "check_recipient_access tcp:${cfg.lookupHost}:${toString cfg.lookupPort}" ]; + }; + transport = "${cfg.emailDomain} lmtp:inet:${cfg.listenHost}:${toString cfg.listenPort}\n"; + # Both are loopback-only literal IPs here, so this is purely + # defensive: unlike a container deployment behind a hostname, + # there is no DNS/chroot interaction to break. See the + # project's Docker Postfix notes for why this mattered there. + settings.master = { + smtp_inet.chroot = false; + lmtp.chroot = false; + }; + }; + }; + }; + }; +} diff --git a/scripts/security_test.py b/scripts/security_test.py new file mode 100644 index 0000000..2867d58 --- /dev/null +++ b/scripts/security_test.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +"""Adversarial testing for pneumatube, run against your own deployment. + +Covers (see each test_* function's docstring for detail): + - injection fuzzing of the Email-To / subject paths + - malformed-MIME resilience of extract_text + - per-message failure isolation in the smolmail -> email poll loop + - username enumeration via the Postfix lookup service's own protocol + - abuse/open-relay behaviour and the accept-gate mitigations for each + direction (PNEUMATUBE_REQUIRE_ACCEPTED_SENDER, _RECIPIENT), including + their explicit opt-out modes + - sender spoofing: confirms a forged MAIL FROM is never forwarded into + the relayed smolmail message (email -> smolmail carries no Email-From + field at all, precisely because MAIL FROM is unauthenticated) + +Each check prints PASS (a defense verified), RISK (confirmed, accepted +design tradeoff -- not a bug, but worth knowing it's real), or BUG +(unexpected, worth fixing). + +Usage: + uv run python scripts/security_test.py + uv run python scripts/security_test.py --skip-live # unit-style checks only + +Needs a `smolmail` checkout next to this repo for the live checks (same +default pneumatube itself uses: PNEUMATUBE_SMOLMAIL_REPO, default +"../smolmail"). +""" +from __future__ import annotations + +import argparse +import contextlib +import email as email_module +import logging +import os +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass +from email.message import EmailMessage +from pathlib import Path + +from aiosmtpd.controller import Controller + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from stress_test import ( # noqa: E402 + REPO_ROOT, CountingSink, SmolAccount, SmolServer, free_port, make_account, + send_one_lmtp, send_one_smol, start_smolmaild, wait_for_port, +) + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) +from pneumatube import smtp_client, vendor # noqa: E402 +from pneumatube.email_to_smol import EmailToSmolBridge, _single_line # noqa: E402 +from pneumatube.lookup_server import _lookup # noqa: E402 + + +@dataclass +class Finding: + check: str + kind: str # "pass" (defense verified), "risk" (confirmed accepted tradeoff), "bug" (unexpected) + detail: str + + def print(self) -> None: + label = {"pass": "PASS", "risk": "RISK", "bug": "BUG "}[self.kind] + print(f"[{label}] {self.check}: {self.detail}") + + +FINDINGS: list[Finding] = [] + + +def record(check: str, kind: str, detail: str) -> None: + f = Finding(check, kind, detail) + f.print() + FINDINGS.append(f) + + +# --- injection fuzzing (no live infra needed) ----------------------------------- + + +INJECTION_PAYLOADS = [ + ("bare-lf", "evil@example.com\nX-Injected: 1"), + ("bare-cr", "evil@example.com\rX-Injected: 1"), + ("crlf", "evil@example.com\r\nBcc: evil@example.com"), + ("nul-byte", "evil@example.com\x00X-Injected: 1"), + ("unicode-line-sep", "evil@example.com
X-Injected: 1"), + ("unicode-para-sep", "evil@example.com
X-Injected: 1"), + ("very-long", "a" * 10_000 + "@example.com"), +] + + +def test_address_validation_fuzz() -> None: + """smtp_client.validate() is the only gate between an attacker-supplied + Email-To frontmatter value and smtplib.SMTP's RCPT TO -- it must reject + every CR/LF variant, not just the literal \\r\\n case the unit tests + already cover.""" + failures = [] + for name, payload in INJECTION_PAYLOADS: + try: + smtp_client.validate(payload) + # unicode-line-sep/para-sep and very-long aren't CR/LF and may + # legitimately pass the regex; only bare-lf/bare-cr/crlf/nul-byte + # must always be rejected. + if name in ("bare-lf", "bare-cr", "crlf", "nul-byte"): + failures.append(name) + except smtp_client.InvalidAddress: + pass + record("address validation fuzz", "pass" if not failures else "bug", + "all CR/LF/NUL variants rejected" if not failures + else f"accepted dangerous payloads: {failures}") + + +def test_single_line_fuzz() -> None: + """email_to_smol._single_line() is what stops a forged email Subject + from injecting extra smolmail frontmatter fields (SPEC.md S5.5: one + `Key: value` per line) into an otherwise-trusted message. (MAIL FROM no + longer flows into frontmatter at all -- see email_to_smol.py's module + docstring -- but Subject still does.)""" + failures = [] + for name, payload in INJECTION_PAYLOADS: + cleaned = _single_line(payload) + if "\n" in cleaned or "\r" in cleaned: + failures.append(name) + record("frontmatter sanitizer fuzz", "pass" if not failures else "bug", + "no payload left a raw CR/LF in the sanitized value" if not failures + else f"raw CR/LF survived sanitization: {failures}") + + +def test_outbound_header_injection() -> None: + """Even though frontmatter values can't structurally contain a raw + newline (smolmail.py's parser splits the body into single lines before a + value is ever extracted), smtp_client.send() has no explicit sanitizer + of its own -- it currently relies entirely on that upstream guarantee + plus Python's EmailMessage rejecting embedded CR/LF. Confirm the second + layer actually holds, since the first is an implicit dependency on + vendored code outside this project.""" + try: + m = EmailMessage() + m["Subject"] = "hi\r\nBcc: evil@attacker.example" + rejected = False + except ValueError: + rejected = True + record("stdlib header-injection backstop", "pass" if rejected else "bug", + "EmailMessage rejects embedded CR/LF in header values (confirmed)" if rejected + else "EmailMessage accepted an embedded CR/LF -- smtp_client.send() has no " + "sanitizer of its own and would forward it verbatim") + + +# --- malformed MIME resilience (no live infra needed) --------------------------- + + +def test_malformed_mime_resilience() -> None: + """extract_text() runs on every inbound email before any validation -- + it must not crash or hang on adversarial MIME, since a single crash + taking down the LMTP handler would be a trivial DoS against every other + in-flight recipient in the same message.""" + cases = { + "empty": b"", + "no-body": b"Subject: hi\r\n\r\n", + "non-utf8-bytes": b"Subject: hi\r\nContent-Type: text/plain\r\n\r\n\xff\xfe\x00\x01", + "multipart-no-text-part": ( + b"Content-Type: multipart/mixed; boundary=X\r\n\r\n--X\r\n" + b"Content-Type: application/octet-stream\r\n\r\nbinary\r\n--X--\r\n"), + "deeply-nested-multipart": _nested_multipart(50), + "declared-huge-missing-body": ( + b"Content-Type: text/plain\r\nContent-Length: 999999999\r\n\r\nshort"), + } + failures = [] + for name, raw in cases.items(): + try: + message = email_module.message_from_bytes(raw) + t0 = time.monotonic() + EmailToSmolBridge.extract_text(message) + if time.monotonic() - t0 > 2.0: + failures.append(f"{name} (slow: possible algorithmic DoS)") + except Exception as exc: + failures.append(f"{name} ({exc!r})") + record("malformed MIME resilience", "pass" if not failures else "bug", + "all cases handled without raising or stalling" if not failures + else f"crashed or stalled on: {failures}") + + +def _nested_multipart(depth: int) -> bytes: + body = b"leaf" + for i in range(depth): + boundary = f"B{i}".encode() + body = (b"Content-Type: multipart/mixed; boundary=" + boundary + b"\r\n\r\n--" + + boundary + b"\r\n" + body + b"\r\n--" + boundary + b"--\r\n") + return b"Content-Type: multipart/mixed; boundary=B0\r\n\r\n--B0\r\n" + body + b"\r\n--B0--\r\n" + + +# --- per-message failure isolation (no live infra needed) ---------------------- + + +def test_poll_loop_isolates_failures() -> None: + """A message that makes _relay() raise anything other than + InvalidAddress/OSError must not block every message behind it in the + smolmail -> email direction -- see smol_to_email.py's poll_once() for + why (a poisoned message is never deleted, so it sorts first forever).""" + from types import SimpleNamespace + + from pneumatube.smol_to_email import SmolToEmailBridge + + class StubSmol: + SmolError = RuntimeError + + def cmd_fetch(self, args, store): + pass + + class StubStore: + def mail(self, box): + return [(b"poison", b"", 0, None), (b"ok", b"", 1, None)] + + bridge = SmolToEmailBridge(StubSmol(), StubStore(), account=None, + config=SimpleNamespace(smolmail_key="k", smolmail_db="d", + require_accepted_sender=True)) + relayed = [] + + def fake_relay(mid, envelope): + if mid == b"poison": + raise ValueError("simulated") + relayed.append(mid) + + bridge._relay = fake_relay + # poll_once() logs the simulated failure via logger.exception(), which is + # the whole point of the fix -- quiet it here so it doesn't look like an + # unhandled crash in this script's own output. + logging.getLogger("pneumatube.smol_to_email").setLevel(logging.CRITICAL) + try: + bridge.poll_once() + record("poll loop failure isolation", "pass" if relayed == [b"ok"] else "bug", + "later messages still processed after an earlier one raised" if relayed == [b"ok"] + else "poll_once stopped after the first failure -- queue-jam DoS") + except Exception as exc: + record("poll loop failure isolation", "bug", + f"poll_once raised instead of isolating the failure: {exc!r}") + + +# --- username enumeration via the lookup service (no live infra needed) -------- + + +def test_lookup_enumeration() -> None: + """The Postfix lookup service (lookup_server.py) answers definitively + per username with no authentication and no rate limiting of its own -- + by design, it's meant to be reachable only from Postfix on a private + network. Confirm that property so the deployment docs' "loopback only" + requirement is understood as load-bearing, not cosmetic.""" + import asyncio + + class StubBridge: + def user_exists(self, username): + return username in ("alice", "bob") + + def recipient_accepted_gateway(self, username): + return True + + class InlineWorker: + async def run(self, fn, *args): + return fn(*args) + + from types import SimpleNamespace + config = SimpleNamespace(email_domain="gateway.example", require_accepted_recipient=True) + bridge, worker = StubBridge(), InlineWorker() + + # tcp_table(5) status is "200" either way (it means "I have an answer"); + # the OK-vs-reject distinction Postfix actually acts on is in the value. + results = {} + for name in ("alice", "bob", "carol", "admin", "root", "postmaster"): + reply = asyncio.run(_lookup(bridge, worker, config, f"{name}@gateway.example")) + _code, _, value = reply.partition(" ") + results[name] = value == "OK" + distinguishable = results["alice"] and results["bob"] and not any( + results[n] for n in ("carol", "admin", "root", "postmaster")) + record("lookup service enumeration", "risk" if distinguishable else "bug", + "the lookup protocol unambiguously reveals which usernames exist -- expected, " + "it must for check_recipient_access to work, but this is only safe because the " + "service is loopback-only; exposing it beyond Postfix would be a username " + "enumeration oracle" if distinguishable else + "lookup responses did not distinguish known from unknown usernames as expected " + "-- check_recipient_access would misbehave") + + +# --- live tests: abuse/throttling and sender spoofing --------------------------- + + +def _start_pneumatube(smolmail_repo: Path, gateway: SmolAccount, server: SmolServer, + sink_port: int, require_accepted_sender: bool = True, + require_accepted_recipient: bool = True + ) -> tuple[subprocess.Popen, int]: + listen_port = free_port() + env = {**os.environ, + "PNEUMATUBE_SMOLMAIL_REPO": str(smolmail_repo), + "PNEUMATUBE_SMOLMAIL_KEY": str(gateway.key_path), + "PNEUMATUBE_SMOLMAIL_DB": str(gateway.db_path), + "PNEUMATUBE_SMOLMAIL_DOMAIN": f"{server.host}:{server.port}", + "PNEUMATUBE_EMAIL_DOMAIN": "gateway.example", + "PNEUMATUBE_FETCH_INTERVAL": "2", + "PNEUMATUBE_REQUIRE_ACCEPTED_SENDER": "1" if require_accepted_sender else "0", + "PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT": "1" if require_accepted_recipient else "0", + "PNEUMATUBE_LISTEN_PROTOCOL": "lmtp", + "PNEUMATUBE_LISTEN_HOST": "127.0.0.1", + "PNEUMATUBE_LISTEN_PORT": str(listen_port), + "PNEUMATUBE_LOOKUP_HOST": "127.0.0.1", + "PNEUMATUBE_LOOKUP_PORT": str(free_port()), + "PNEUMATUBE_SMTP_RELAY_HOST": "127.0.0.1", + "PNEUMATUBE_SMTP_RELAY_PORT": str(sink_port), + "PNEUMATUBE_SMTP_RELAY_STARTTLS": "0", + "PNEUMATUBE_GATEWAY_FROM_ADDRESS": "gateway@gateway.example"} + proc = subprocess.Popen(["uv", "run", "pneumatube"], cwd=REPO_ROOT, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + wait_for_port("127.0.0.1", listen_port) + return proc, listen_port + + +def _stop(proc: subprocess.Popen) -> None: + proc.terminate() + with contextlib.suppress(Exception): + proc.wait(timeout=5) + + +def run_live_tests(smolmail_repo: Path) -> None: + with tempfile.TemporaryDirectory(prefix="pneumatube-sectest-") as tmp: + tmp_dir = Path(tmp) + server = start_smolmaild(smolmail_repo, tmp_dir, rate_connections=6000, rate_sends=6000) + smol = vendor.load(smolmail_repo) + gateway = make_account(smol, tmp_dir, "gatewayuser", server) + victim = make_account(smol, tmp_dir, "victim", server) + spoof_victim = make_account(smol, tmp_dir, "spoofvictim", server) + + sink = CountingSink() + sink_port = free_port() + sink_controller = Controller(sink, hostname="127.0.0.1", port=sink_port) + sink_controller.start() + + # -- smolmail -> email, default config (require_accepted_sender=True) -- + # 15 messages from 15 DIFFERENT, never-before-seen sender identities + # (no accept token, no prior relationship), each addressed to a + # different external destination. None should reach the sink. + proc, listen_port = _start_pneumatube(smolmail_repo, gateway, server, sink_port, + require_accepted_sender=True) + try: + senders = [make_account(smol, tmp_dir, f"stranger{i}", server) for i in range(15)] + for i, sender in enumerate(senders): + send_one_smol(smol, sender, gateway.address, f"dest{i}@external.example", + f"relay test {i}") + time.sleep(4) # >= one PNEUMATUBE_FETCH_INTERVAL + delivered = sink.count() + record("open relay blocked by default (smolmail -> email)", + "pass" if delivered == 0 else "bug", + f"{delivered}/{len(senders)} first-contact messages reached the outbound " + "sink with PNEUMATUBE_REQUIRE_ACCEPTED_SENDER at its default (1) -- " + + ("none did, as intended" if delivered == 0 else + "some did; the accept gate in smol_to_email.py is not holding")) + + # -- email -> smolmail, default config (require_accepted_recipient=True) -- + # "victim" is registered on the smolmail server (so user_exists + # passes) but has never accepted the gateway (no token held) -- + # the all-or-nothing gate should reject this live, not queue it. + outcome, _lat = send_one_lmtp( + "127.0.0.1", listen_port, "sender@external.example", + "victim@gateway.example", b"Subject: hi\r\n\r\nbody\r\n") + record("relay blocked for unaccepted recipient (email -> smolmail)", + "pass" if outcome == "rejected" else "bug", + f"outcome={outcome} with PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT at its " + "default (1) for a registered-but-never-accepted recipient -- " + + ("rejected live at RCPT TO, as intended" if outcome == "rejected" else + "was not rejected; the gate in handle_RCPT/_lookup is not holding")) + finally: + _stop(proc) + + # -- both directions, explicit opt-in to the open-bridge mode -- + # Confirms both flags at 0 still do what they document: restore + # "relay anyone" for an operator who deliberately wants it. victim + # and spoof_victim never accepted the gateway, so this also doubles + # as the opt-out confirmation for the recipient gate. + proc, listen_port = _start_pneumatube(smolmail_repo, gateway, server, sink_port, + require_accepted_sender=False, + require_accepted_recipient=False) + try: + senders = [make_account(smol, tmp_dir, f"openbridge{i}", server) for i in range(15)] + for i, sender in enumerate(senders): + send_one_smol(smol, sender, gateway.address, f"dest2-{i}@external.example", + f"open bridge test {i}") + deadline = time.monotonic() + 15 + while sink.count() < len(senders) and time.monotonic() < deadline: + time.sleep(0.5) + delivered = sink.count() + record("open relay via explicit opt-in (smolmail -> email)", "risk", + f"{delivered}/{len(senders)} first-contact senders were relayed to " + "arbitrary external addresses with PNEUMATUBE_REQUIRE_ACCEPTED_SENDER=0 " + "-- confirms the opt-in open-bridge mode works as documented; an operator " + "choosing it should understand this is what they're choosing") + + # -- open relay / no throttling (email -> smolmail) -- + ok = 0 + total = 15 + for i in range(total): + outcome, _lat = send_one_lmtp("127.0.0.1", listen_port, + f"stranger{i}@external.example", + "victim@gateway.example", + f"Subject: relay {i}\r\n\r\nbody {i}\r\n".encode()) + ok += outcome == "ok" + record("open relay via explicit opt-in (email -> smolmail)", "risk", + f"{ok}/{total} messages from {total} distinct external senders were all " + "relayed to an unaccepted recipient with PNEUMATUBE_REQUIRE_ACCEPTED_" + "RECIPIENT=0 -- confirms the opt-out works as documented; an operator " + "choosing it should understand this is what they're choosing. There is " + "also no per-sender throttling regardless of this flag's setting.") + + # -- sender spoofing -- + # Own recipient account, untouched by the open-relay tests above, + # so finding the message isn't a race against unrelated mail. + # email_to_smol.py forwards no claim about the original sender at + # all (see its module docstring) specifically because MAIL FROM + # is unauthenticated; confirm a forged one doesn't leak through + # into any frontmatter field of the relayed message. + forged = "security-team@trusted-bank.example" + outcome, _lat = send_one_lmtp( + "127.0.0.1", listen_port, forged, "spoofvictim@gateway.example", + b"Subject: Urgent: verify your account\r\n\r\nClick here...\r\n") + opened = None + deadline = time.monotonic() + 15 + account = smol.load_account(argparse.Namespace(key=str(spoof_victim.key_path)), + smol.Store(str(spoof_victim.db_path))) + while time.monotonic() < deadline and opened is None: + store = smol.Store(str(spoof_victim.db_path)) + smol.cmd_fetch(argparse.Namespace(key=str(spoof_victim.key_path), + db=str(spoof_victim.db_path), + reset=False, keep=False), store) + rows = store.mail("all") + if rows: + mid, envelope, _at, _r = rows[0] + opened = smol.describe(store, envelope, account.keys) + else: + time.sleep(0.5) + + if opened is None: + record("sender spoofing (no Email-From claim forwarded)", "bug", + f"the message never arrived (outcome={outcome}) -- could not confirm") + else: + leaked = forged in opened["fields"].values() or "email-from" in opened["fields"] + record("sender spoofing (no Email-From claim forwarded)", + "pass" if not leaked else "bug", + "the relayed message carries no field reflecting the forged MAIL " + "FROM -- nothing here for a recipient to mistake for a verified " + "sender claim" if not leaked else + f"the forged MAIL FROM {forged!r} leaked into the relayed message's " + f"frontmatter: {opened['fields']}") + finally: + _stop(proc) + sink_controller.stop() + _stop(server.process) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--skip-live", action="store_true", + help="run only the unit-style checks (no subprocess/network)") + parser.add_argument("--smolmail-repo", default="../smolmail") + args = parser.parse_args() + + test_address_validation_fuzz() + test_single_line_fuzz() + test_outbound_header_injection() + test_malformed_mime_resilience() + test_poll_loop_isolates_failures() + test_lookup_enumeration() + + if not args.skip_live: + run_live_tests(Path(args.smolmail_repo).resolve()) + + print() + bugs = [f for f in FINDINGS if f.kind == "bug"] + risks = [f for f in FINDINGS if f.kind == "risk"] + print(f"{len(FINDINGS)} check(s): {len(FINDINGS) - len(bugs) - len(risks)} pass, " + f"{len(risks)} confirmed accepted risk(s), {len(bugs)} bug(s)") + + +if __name__ == "__main__": + main() diff --git a/scripts/stress_test.py b/scripts/stress_test.py new file mode 100644 index 0000000..cdb7997 --- /dev/null +++ b/scripts/stress_test.py @@ -0,0 +1,532 @@ +#!/usr/bin/env python3 +"""Concurrent load test for pneumatube. + +Two targets: + + direct Talks to a `pneumatube` subprocess directly over LMTP/SMTP, + bypassing Postfix. Isolates pneumatube's own concurrency + behaviour -- in particular, SmolWorker (worker.py) serializes + every smolmail operation onto one thread, so this is where that + design's throughput ceiling actually shows up. + postfix Drives the same load through the real docker-compose.yml rig + (Postfix -> LMTP -> pneumatube), to confirm nothing changes with + a real MTA in front. Needs podman (or docker) + compose. + +Both targets spin up a throwaway smolmaild, register a gateway account plus +N recipient/sender accounts against it, generate concurrent load, and verify +every expected message actually arrived exactly once -- throughput numbers +are meaningless if messages were silently dropped or duplicated. + +Usage: + uv run python scripts/stress_test.py --target direct + uv run python scripts/stress_test.py --target postfix + uv run python scripts/stress_test.py --target both --concurrency 40 + +Needs a `smolmail` checkout next to this repo (same default pneumatube +itself uses: PNEUMATUBE_SMOLMAIL_REPO, default "../smolmail"). +""" +from __future__ import annotations + +import argparse +import contextlib +import os +import shutil +import smtplib +import socket +import subprocess +import sys +import tempfile +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path + +from aiosmtpd.controller import Controller + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) +from pneumatube import vendor # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def wait_for_port(host: str, port: int, timeout: float = 15.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with socket.create_connection((host, port), timeout=0.5): + return + except OSError: + time.sleep(0.1) + raise TimeoutError(f"nothing listening on {host}:{port} after {timeout}s") + + +# --- reporting ---------------------------------------------------------------- + + +@dataclass +class Report: + label: str + attempted: int = 0 + succeeded: int = 0 + rejected: int = 0 + errors: int = 0 + latencies: list[float] = field(default_factory=list) + + def record(self, outcome: str, latency: float) -> None: + self.attempted += 1 + self.latencies.append(latency) + if outcome == "ok": + self.succeeded += 1 + elif outcome == "rejected": + self.rejected += 1 + else: + self.errors += 1 + + def print(self) -> None: + lat = sorted(self.latencies) + p50 = lat[len(lat) // 2] if lat else 0.0 + p99 = lat[int(len(lat) * 0.99)] if lat else 0.0 + print(f"-- {self.label} --") + print(f" attempted={self.attempted} succeeded={self.succeeded} " + f"rejected={self.rejected} errors={self.errors}") + print(f" latency p50={p50 * 1000:.1f}ms p99={p99 * 1000:.1f}ms " + f"max={max(lat, default=0) * 1000:.1f}ms") + + +# --- smolmail test fixtures ---------------------------------------------------- + + +@dataclass +class SmolServer: + host: str + port: int + pubkey_b32: str + process: subprocess.Popen | None = None + + +def parse_pubkey(text: str) -> str: + for line in text.splitlines(): + if "public key" in line: + return line.strip().rsplit(" ", 1)[-1] + raise RuntimeError(f"no public key line in output:\n{text}") + + +def start_smolmaild(smolmail_repo: Path, tmp_dir: Path, rate_connections: int, + rate_sends: int) -> SmolServer: + port = free_port() + key_path = tmp_dir / "server.key" + subprocess.run( + ["uv", "run", str(smolmail_repo / "smolmaild.py"), "keygen", "--key", str(key_path)], + cwd=smolmail_repo, check=True, capture_output=True, text=True) + process = subprocess.Popen( + ["uv", "run", str(smolmail_repo / "smolmaild.py"), "serve", + "--key", str(key_path), "--db", str(tmp_dir / "server.db"), + "--host", "127.0.0.1", "--port", str(port), + # smolmaild's connection/send rate limits are keyed by source IP + # (SPEC.md S10); every sender in this script shares one IP, so the + # defaults (120 conn/min, 60 sends/min) would measure the rate + # limiter, not pneumatube, unless raised for a many-sender test. + "--rate-connections", str(rate_connections), "--rate-sends", str(rate_sends)], + cwd=smolmail_repo, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + pubkey = None + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + line = process.stdout.readline() + if not line: + break + if "server public key" in line: + pubkey = line.strip().rsplit(" ", 1)[-1] + break + if pubkey is None: + process.kill() + raise RuntimeError("smolmaild never printed its public key") + wait_for_port("127.0.0.1", port) + return SmolServer("127.0.0.1", port, pubkey, process) + + +@dataclass +class SmolAccount: + username: str + key_path: Path + db_path: Path + address: str # "user@host:port" + + +def make_account(smol, tmp_dir: Path, name: str, server: SmolServer) -> SmolAccount: + """Keygen + trust + register, reusing smolmail.py's own CLI commands + directly (the same functions the gateway itself calls) rather than + reimplementing the protocol or shelling out per account. + + A bare TCP connect (wait_for_port) can succeed slightly before a + containerized smolmaild can actually complete a Noise handshake, so + register is retried a few times rather than treating that race as fatal. + """ + key_path, db_path = tmp_dir / f"{name}.key", tmp_dir / f"{name}.db" + store = smol.Store(str(db_path)) + smol.cmd_keygen(argparse.Namespace(key=str(key_path), force=True), store) + smol.cmd_trust(argparse.Namespace(host=f"{server.host}:{server.port}", + key_b32=server.pubkey_b32, force=True), store) + address = f"{name}@{server.host}:{server.port}" + register_args = argparse.Namespace(key=str(key_path), address=address, token=None) + attempts = 20 + for attempt in range(attempts): + try: + smol.cmd_register(register_args, store) + break + except (smol.SmolError, ConnectionError, OSError): + if attempt == attempts - 1: + raise + time.sleep(1.0) + return SmolAccount(name, key_path, db_path, address) + + +def repoint_gateway_account(smol, gateway: SmolAccount, host: str, port: int, + pubkey_b32: str) -> None: + """Point the gateway's own remembered server connection at a different + host string (e.g. a compose service name) without a real round trip -- + `trust` only ever stores a pin locally (smolmail.py's own cmd_trust).""" + store = smol.Store(str(gateway.db_path)) + smol.cmd_trust(argparse.Namespace(host=host, key_b32=pubkey_b32, force=True), store) + store.set_account(smol.Address(gateway.username, host, port)) + + +# --- outbound SMTP sink (stands in for the real relay) ------------------------- + + +class CountingSink: + def __init__(self) -> None: + self.lock = threading.Lock() + self.received: list[tuple[str, tuple[str, ...]]] = [] + + async def handle_DATA(self, server, session, envelope): + with self.lock: + self.received.append((envelope.mail_from, tuple(envelope.rcpt_tos))) + return "250 OK" + + def count(self) -> int: + with self.lock: + return len(self.received) + + +# --- load generators (each measures its own latency) --------------------------- + + +def send_one_lmtp(host: str, port: int, mail_from: str, rcpt: str, body: bytes) -> tuple[str, float]: + t0 = time.monotonic() + try: + with smtplib.LMTP(host, port, timeout=10) as client: + client.ehlo("stress-test") + client.mail(mail_from) + code, _ = client.rcpt(rcpt) + if code >= 500: + return "rejected", time.monotonic() - t0 + if code >= 400: + return "error", time.monotonic() - t0 + client.data(body) + return "ok", time.monotonic() - t0 + except Exception: + return "error", time.monotonic() - t0 + + +def send_one_smtp(host: str, port: int, mail_from: str, rcpt: str, body: bytes) -> tuple[str, float]: + t0 = time.monotonic() + try: + with smtplib.SMTP(host, port, timeout=10) as client: + client.ehlo("stress-test") + client.mail(mail_from) + code, _ = client.rcpt(rcpt) + if code >= 500: + return "rejected", time.monotonic() - t0 + if code >= 400: + return "error", time.monotonic() - t0 + client.data(body) + return "ok", time.monotonic() - t0 + except Exception: + return "error", time.monotonic() - t0 + + +def send_one_smol(smol, account: SmolAccount, to_address: str, email_to: str, + subject: str) -> tuple[str, float]: + t0 = time.monotonic() + store = smol.Store(str(account.db_path)) + send_args = argparse.Namespace( + key=str(account.key_path), address=to_address, body=f"load from {account.username}", + subject=subject, reply_to=None, header=[f"Email-To: {email_to}"], + anonymous=True, no_pad=False) + try: + smol.cmd_send(send_args, store) + return "ok", time.monotonic() - t0 + except Exception: + return "error", time.monotonic() - t0 + + +def run_concurrent(report: Report, concurrency: int, jobs: list) -> list[bool]: + """jobs: list of (fn, args, expected_valid). Returns per-job valid flags + in submission order so the caller can cross-check expected vs actual.""" + valid_flags = [valid for _, _, valid in jobs] + with ThreadPoolExecutor(max_workers=concurrency) as pool: + start = time.monotonic() + futures = [pool.submit(fn, *fn_args) for fn, fn_args, _ in jobs] + for future in futures: + outcome, latency = future.result() + report.record(outcome, latency) + elapsed = time.monotonic() - start + report.print() + print(f" throughput={len(jobs) / elapsed:.1f} msg/s over {elapsed:.1f}s") + return valid_flags + + +# --- direct target --------------------------------------------------------------- + + +def run_direct(args: argparse.Namespace) -> None: + smolmail_repo = Path(args.smolmail_repo).resolve() + with tempfile.TemporaryDirectory(prefix="pneumatube-stress-") as tmp: + tmp_dir = Path(tmp) + print(f"[direct] starting smolmaild in {tmp_dir}") + server = start_smolmaild(smolmail_repo, tmp_dir, args.server_rate_connections, + args.server_rate_sends) + smol = vendor.load(smolmail_repo) + + gateway = make_account(smol, tmp_dir, "gatewayuser", server) + recipients = [make_account(smol, tmp_dir, f"recipient{i}", server) + for i in range(args.recipients)] + senders = [make_account(smol, tmp_dir, f"sender{i}", server) + for i in range(args.concurrency)] + print(f"[direct] {len(recipients)} recipient account(s), " + f"{len(senders)} sender account(s) ready") + + sink = CountingSink() + sink_port = free_port() + sink_controller = Controller(sink, hostname="127.0.0.1", port=sink_port) + sink_controller.start() + + listen_port, lookup_port = free_port(), free_port() + env = {**os.environ, + "PNEUMATUBE_SMOLMAIL_REPO": str(smolmail_repo), + "PNEUMATUBE_SMOLMAIL_KEY": str(gateway.key_path), + "PNEUMATUBE_SMOLMAIL_DB": str(gateway.db_path), + "PNEUMATUBE_SMOLMAIL_DOMAIN": f"{server.host}:{server.port}", + "PNEUMATUBE_EMAIL_DOMAIN": "gateway.example", + "PNEUMATUBE_FETCH_INTERVAL": "2", + # Throughput, not the accept-gate features, is this script's + # concern -- the throwaway recipient/sender accounts here + # never go through the accept dance with the gateway. + "PNEUMATUBE_REQUIRE_ACCEPTED_SENDER": "0", + "PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT": "0", + "PNEUMATUBE_LISTEN_PROTOCOL": "lmtp", + "PNEUMATUBE_LISTEN_HOST": "127.0.0.1", + "PNEUMATUBE_LISTEN_PORT": str(listen_port), + "PNEUMATUBE_LOOKUP_HOST": "127.0.0.1", + "PNEUMATUBE_LOOKUP_PORT": str(lookup_port), + "PNEUMATUBE_SMTP_RELAY_HOST": "127.0.0.1", + "PNEUMATUBE_SMTP_RELAY_PORT": str(sink_port), + "PNEUMATUBE_SMTP_RELAY_STARTTLS": "0", + "PNEUMATUBE_GATEWAY_FROM_ADDRESS": "gateway@gateway.example"} + print("[direct] starting pneumatube") + proc = subprocess.Popen(["uv", "run", "pneumatube"], cwd=REPO_ROOT, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + try: + wait_for_port("127.0.0.1", listen_port) + wait_for_port("127.0.0.1", lookup_port) + + # Phase A: email -> smolmail, concurrent LMTP, mixed valid/invalid. + total = args.concurrency * args.messages + jobs = [] + for i in range(total): + valid = i % 3 != 0 # 2/3 valid, 1/3 unknown-user + rcpt = (f"{recipients[i % len(recipients)].username}@gateway.example" + if valid else "nobody-at-all@gateway.example") + body = f"Subject: load {i}\r\n\r\nmessage {i}\r\n".encode() + jobs.append((send_one_lmtp, ("127.0.0.1", listen_port, + "stress@sender.example", rcpt, body), valid)) + report_a = Report("email -> smolmail (LMTP, direct)") + valid_flags = run_concurrent(report_a, args.concurrency, jobs) + expected_ok, expected_rejected = sum(valid_flags), valid_flags.count(False) + ok = report_a.succeeded == expected_ok and report_a.rejected == expected_rejected + print(f" correctness: {'PASS' if ok else 'FAIL'} " + f"(expected ok={expected_ok} rejected={expected_rejected})") + + # Phase B: smolmail -> email, concurrent SEND, then drain via the + # gateway's own poll loop (already running in the subprocess). + jobs = [(send_one_smol, (smol, senders[i % len(senders)], gateway.address, + f"dest{i}@example.org", f"load {i}"), True) + for i in range(total)] + report_b = Report("smolmail -> email (SEND + poll drain)") + run_concurrent(report_b, args.concurrency, jobs) + + drain_deadline = time.monotonic() + max(30.0, total * 0.05) + while sink.count() < report_b.succeeded and time.monotonic() < drain_deadline: + time.sleep(0.5) + drained = sink.count() + print(f" drained {drained}/{report_b.succeeded} into the outbound sink " + f"({'PASS' if drained == report_b.succeeded else 'FAIL: lost or stuck messages'})") + finally: + proc.terminate() + with contextlib.suppress(Exception): + proc.wait(timeout=5) + sink_controller.stop() + server.process.terminate() + with contextlib.suppress(Exception): + server.process.wait(timeout=5) + + +# --- postfix target --------------------------------------------------------------- + + +def podman(*cmd: str, **kwargs) -> subprocess.CompletedProcess: + return subprocess.run(["podman", *cmd], text=True, capture_output=True, **kwargs) + + +def run_postfix(args: argparse.Namespace) -> None: + if shutil.which("podman") is None: + print("[postfix] podman not found; skipping", file=sys.stderr) + return + + smolmail_repo = Path(args.smolmail_repo).resolve() + smol = vendor.load(smolmail_repo) + network = "pneumatube-stress-net" + env = {**os.environ, + "PNEUMATUBE_SMOLMAIL_DOMAIN": "smolmail-dev:1961", + "PNEUMATUBE_EMAIL_DOMAIN": "gateway.example", + # Throughput, not the accept-gate features, is this script's + # concern -- the throwaway recipient accounts here never go + # through the accept dance with the gateway. + "PNEUMATUBE_REQUIRE_ACCEPTED_SENDER": "0", + "PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT": "0", + "PNEUMATUBE_SMTP_RELAY_HOST": "smolmail-dev", + "PNEUMATUBE_SMTP_RELAY_PORT": "2626", + "PNEUMATUBE_SMTP_RELAY_STARTTLS": "0", + "PNEUMATUBE_GATEWAY_FROM_ADDRESS": "gateway@gateway.example"} + + with tempfile.TemporaryDirectory(prefix="pneumatube-stress-pf-") as tmp: + tmp_dir = Path(tmp) + data_dir = REPO_ROOT / "data" + data_dir.mkdir(exist_ok=True) + try: + print("[postfix] building images") + subprocess.run(["podman", "compose", "build"], cwd=REPO_ROOT, env=env, check=True) + + print("[postfix] starting throwaway smolmaild-dev container") + podman("network", "create", network) + server_dir = tmp_dir / "server" + server_dir.mkdir() + keygen = podman("run", "--rm", "-v", f"{smolmail_repo}:/smolmail:ro", + "-v", f"{server_dir}:/data", + "localhost/pneumatube_pneumatube:latest", + "uv", "run", "/smolmail/smolmaild.py", "keygen", "--key", "/data/server.key") + keygen.check_returncode() + pubkey = parse_pubkey(keygen.stdout) + + host_port = free_port() + podman("run", "-d", "--name", "pneumatube-stress-smolmaild", + "--network", network, "--network-alias", "smolmail-dev", + "-p", f"{host_port}:1961", + "-v", f"{smolmail_repo}:/smolmail:ro", "-v", f"{server_dir}:/data", + "localhost/pneumatube_pneumatube:latest", + "uv", "run", "/smolmail/smolmaild.py", "serve", + "--key", "/data/server.key", "--db", "/data/server.db", + "--host", "0.0.0.0", "--port", "1961", + "--rate-connections", str(args.server_rate_connections), + "--rate-sends", str(args.server_rate_sends)).check_returncode() + wait_for_port("127.0.0.1", host_port) + + server = SmolServer("127.0.0.1", host_port, pubkey) + gateway = make_account(smol, tmp_dir, "gatewayuser", server) + recipients = [make_account(smol, tmp_dir, f"recipient{i}", server) + for i in range(args.recipients)] + repoint_gateway_account(smol, gateway, "smolmail-dev", 1961, pubkey) + (data_dir / "gateway.key").write_bytes(gateway.key_path.read_bytes()) + (data_dir / "gateway.db").write_bytes(gateway.db_path.read_bytes()) + + print("[postfix] starting pneumatube + postfix") + subprocess.run(["podman", "compose", "up", "-d"], cwd=REPO_ROOT, env=env, check=True) + # Join the compose-started containers to the network carrying + # smolmail-dev (they start on compose's own default network, + # which smolmail-dev was never part of) -- not the other way + # around, which would just reconnect smolmail-dev to a network + # it's already on and leave pneumatube unable to resolve it. + podman("network", "connect", network, "pneumatube_pneumatube_1") + podman("network", "connect", network, "pneumatube_postfix_1") + wait_for_port("127.0.0.1", 2525, timeout=60.0) + # Port 25 accepting doesn't mean postconf's chroot/transport_maps + # setup (entrypoint.sh) has finished; give it a moment to settle. + time.sleep(2) + + total = args.concurrency * args.messages + jobs = [] + for i in range(total): + valid = i % 3 != 0 + rcpt = (f"{recipients[i % len(recipients)].username}@gateway.example" + if valid else "nobody-at-all@gateway.example") + body = f"Subject: pf-load {i}\r\n\r\nmessage {i}\r\n".encode() + jobs.append((send_one_smtp, ("127.0.0.1", 2525, + "stress@sender.example", rcpt, body), valid)) + report = Report("email -> smolmail (through Postfix)") + valid_flags = run_concurrent(report, args.concurrency, jobs) + expected_ok, expected_rejected = sum(valid_flags), valid_flags.count(False) + print(f" correctness (live reject, not bounce): " + f"{'PASS' if report.rejected == expected_rejected else 'FAIL'} " + f"(expected ok={expected_ok} rejected={expected_rejected})") + + deadline = time.monotonic() + max(30.0, total * 0.1) + delivered = 0 + while time.monotonic() < deadline: + delivered = 0 + for r in recipients: + store = smol.Store(str(r.db_path)) + smol.cmd_fetch(argparse.Namespace(key=str(r.key_path), db=str(r.db_path), + reset=False, keep=False), store) + delivered += len(store.mail("all")) + if delivered >= expected_ok: + break + time.sleep(1) + print(f" delivered {delivered}/{expected_ok} across recipient inboxes " + f"({'PASS' if delivered == expected_ok else 'FAIL: lost or stuck messages'})") + finally: + print("[postfix] tearing down") + subprocess.run(["podman", "compose", "down"], cwd=REPO_ROOT, env=env) + podman("rm", "-f", "pneumatube-stress-smolmaild") + podman("network", "rm", network) + with contextlib.suppress(Exception): + (data_dir / "gateway.key").unlink() + (data_dir / "gateway.db").unlink() + data_dir.rmdir() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--target", choices=["direct", "postfix", "both"], default="direct") + parser.add_argument("--concurrency", type=int, default=20, + help="concurrent sender workers per phase") + parser.add_argument("--messages", type=int, default=10, + help="messages sent by each worker") + parser.add_argument("--recipients", type=int, default=10, + help="number of pre-registered valid recipient accounts") + parser.add_argument("--smolmail-repo", default="../smolmail") + parser.add_argument("--server-rate-connections", type=int, default=6000, + help="smolmaild --rate-connections for the throwaway test server " + "(default 120/min would rate-limit this whole test, since every " + "sender shares one source IP; pass 120 to test against that " + "realistic default instead)") + parser.add_argument("--server-rate-sends", type=int, default=6000, + help="smolmaild --rate-sends for the throwaway test server " + "(see --server-rate-connections; realistic default is 60)") + args = parser.parse_args() + + if args.target in ("direct", "both"): + run_direct(args) + if args.target in ("postfix", "both"): + run_postfix(args) + + +if __name__ == "__main__": + main() diff --git a/src/pneumatube/__main__.py b/src/pneumatube/__main__.py index 1bf4131..14b7e9d 100644 --- a/src/pneumatube/__main__.py +++ b/src/pneumatube/__main__.py @@ -1,12 +1,19 @@ """pneumatube: a gateway bridging one smolmail account and an SMTP mailbox. smolmail -> email: a message the gateway receives with an Email-To -frontmatter field is relayed to that address over SMTP. +frontmatter field is relayed to that address over SMTP. By default this only +happens for senders the gateway account has accepted (SPEC.md S5.8; +PNEUMATUBE_REQUIRE_ACCEPTED_SENDER=0 to relay first-contact senders too). email -> smolmail: mail sent to @PNEUMATUBE_EMAIL_DOMAIN is relayed as a smolmail message to @PNEUMATUBE_SMOLMAIL_DOMAIN, if -that username is registered there, with the original sender recorded in an -Email-From frontmatter field. +that username is registered there and (by default) has accepted the gateway +(SPEC.md S5.8; PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT=0 for the softer, +smolmail-native requests-tier demotion instead of an outright reject -- see +email_to_smol.py's module docstring). The relayed message carries no claim +about the original email sender -- an email's MAIL FROM is unauthenticated, +so forwarding it into the message would let any sender forge an +apparently-trustworthy "from" inside a signed smolmail message. See config.py for the full list of environment variables, including PNEUMATUBE_LISTEN_PROTOCOL (lmtp, behind a real MTA, or smtp for local-dev diff --git a/src/pneumatube/config.py b/src/pneumatube/config.py index c01971e..85fdc1e 100644 --- a/src/pneumatube/config.py +++ b/src/pneumatube/config.py @@ -5,10 +5,11 @@ import os from dataclasses import dataclass from pathlib import Path -# Frontmatter is case-insensitive and flat (smolmail SPEC.md S5.5); these two -# custom keys are the only contract between the two directions of the bridge. +# Frontmatter is case-insensitive and flat (smolmail SPEC.md S5.5); this is +# the only contract the smolmail -> email direction relies on. The reverse +# direction deliberately carries no analogous custom field -- see +# email_to_smol.py's module docstring for why. FRONTMATTER_EMAIL_KEY = "Email-To" # smolmail -> email: where to deliver -FRONTMATTER_SENDER_KEY = "Email-From" # email -> smolmail: who originally sent it def _env(name: str) -> str: @@ -54,6 +55,17 @@ class Config: gateway_from_address: str # From: header used on mail forwarded to SMTP + # If true (the default), only relay smolmail messages from senders the + # gateway account has accepted (SPEC.md S5.8) -- a first-contact sender + # with no accept token lands in the requests tier and is left alone. + # Set to false to run an intentionally open bridge instead. + require_accepted_sender: bool + + # If true (the default), only relay inbound email to a smolmail user who + # has accepted the gateway (SPEC.md S5.8) -- an all-or-nothing RCPT-time + # reject, not a demotion. See email_to_smol.py's module docstring. + require_accepted_recipient: bool + def load() -> Config: listen_protocol = os.environ.get("PNEUMATUBE_LISTEN_PROTOCOL", "lmtp") @@ -80,4 +92,7 @@ def load() -> Config: smtp_relay_password=os.environ.get("PNEUMATUBE_SMTP_RELAY_PASSWORD"), smtp_relay_starttls=os.environ.get("PNEUMATUBE_SMTP_RELAY_STARTTLS", "1") != "0", gateway_from_address=_env("PNEUMATUBE_GATEWAY_FROM_ADDRESS"), + require_accepted_sender=os.environ.get("PNEUMATUBE_REQUIRE_ACCEPTED_SENDER", "1") != "0", + require_accepted_recipient=os.environ.get( + "PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT", "1") != "0", ) diff --git a/src/pneumatube/email_to_smol.py b/src/pneumatube/email_to_smol.py index 8bafbb0..eedf640 100644 --- a/src/pneumatube/email_to_smol.py +++ b/src/pneumatube/email_to_smol.py @@ -1,9 +1,37 @@ """email -> smolmail direction. -Confirms a username is registered on the configured smolmail server and, if -so, relays an inbound email to it as a smolmail message, carrying the -original sender's email address in the Email-From frontmatter field -(SPEC.md S5.5). +Confirms a username is registered on the configured smolmail server, and +that they have accepted the gateway's own smolmail identity (SPEC.md S5.8), +before relaying an inbound email to them as a smolmail message. This is an +all-or-nothing gate, not a demotion: without it, the gateway's SEND would +still go through and land in the recipient's own requests tier (still +delivered, just quota-limited); with it, pneumatube refuses the RCPT +entirely, so an unaccepted recipient gets nothing at all. Set +PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT=0 to fall back to that softer, +smolmail-native tiering instead. + +Acceptance is checked locally, with no network round trip: cmd_send already +needs to know whether it holds a token for the recipient (SELECT token FROM +tokens WHERE address = ?) to decide whether to attach a valid accept MAC, +and that same row's presence is exactly "has this recipient accepted the +gateway". The recipient must do two things before this check passes, not +one: accept the gateway's address on their own smolmail account (which only +affects their own server's tiering), and then actually send the gateway +something -- the token only reaches pneumatube's local `tokens` table via a +message's signed Accept frontmatter field (learn_token(), run at FETCH time). +Accepting alone does not make pneumatube aware of it. + +The relayed message deliberately carries no claim about the original email +sender: smolmail's own signature already tells the recipient this message +is authentically from the gateway's registered identity, which is the only +thing anyone here can vouch for. An email's MAIL FROM is never authenticated +(SMTP has no mechanism for that without SPF/DKIM enforcement upstream, which +this project does not configure), so forwarding it into a frontmatter field +that looks like gateway-attested metadata would let any email sender forge +an apparently-trustworthy "from" inside a signed smolmail message -- a +sender-spoofing vector, not a feature. If the recipient needs to know who +sent the original email, that has to come from the body text itself, which +is clearly just quoted content rather than something the signature backs. """ from __future__ import annotations @@ -12,7 +40,7 @@ import logging from email.message import Message from types import ModuleType -from .config import Config, FRONTMATTER_SENDER_KEY +from .config import Config logger = logging.getLogger(__name__) @@ -30,28 +58,51 @@ class EmailToSmolBridge: def user_exists(self, username: str) -> bool | None: """True/False for a definite answer; None when resolving itself failed (pinning/network), which is not the same as "no such user" - and should not be treated as a permanent bounce.""" + and should not be treated as a permanent bounce. + + Caches the resolved identity as a contact (trust-on-first-use, same + as smolmail.py's own trust_key) so forward()'s cmd_send -- called a + moment later for the same username -- hits that cache instead of + paying for a second full connect-and-RESOLVE round trip. Each one is + a fresh TCP + Noise handshake (smolmail.py has no connection + pooling), so skipping the duplicate roughly halves the network cost + of this direction's hot path. + """ try: addr = self.smol.Address.parse(f"{username}@{self.config.smolmail_domain}") with self.smol.connect(self.store, addr, require_pin=True) as conn: - self.smol.do_resolve(conn, username) + identity, _chain = self.smol.do_resolve(conn, username) except self.smol.SmolError as exc: if "unknown user" in str(exc): return False logger.warning("resolve for %s failed: %s", username, exc) return None + self.store.save_contact(addr.short, identity, verified=False) return True - def forward(self, username: str, sender: str, subject: str, body: str) -> None: - # Subject and sender end up as frontmatter values (SPEC.md S5.5: one - # `Key: value` per line); a bare CR/LF smuggled through email headers - # would otherwise let a sender inject extra frontmatter fields. + def recipient_accepted_gateway(self, username: str) -> bool: + """Whether pneumatube holds an accept token for this recipient -- + see this module's docstring for exactly what that does and doesn't + mean. A pure local lookup against the gateway's own db, no network. + + Addresses are stored (by learn_token()) in Address.short's + normalized form, which drops an explicit default port -- build the + lookup key the same way rather than concatenating smolmail_domain + directly, or a configured "host:1961" would never match. + """ + addr = self.smol.Address.parse(f"{username}@{self.config.smolmail_domain}") + return self.store.one("SELECT token FROM tokens WHERE address = ?", addr.short) is not None + + def forward(self, username: str, subject: str, body: str) -> None: + # Subject ends up as a frontmatter value (SPEC.md S5.5: one + # `Key: value` per line); a bare CR/LF smuggled through the email + # Subject header would otherwise let a sender inject extra + # frontmatter fields. send_args = argparse.Namespace( key=str(self.config.smolmail_key), db=str(self.config.smolmail_db), address=f"{username}@{self.config.smolmail_domain}", body=body, subject=_single_line(subject), reply_to=None, - header=[f"{FRONTMATTER_SENDER_KEY}: {_single_line(sender)}"], - anonymous=False, no_pad=False) + header=[], anonymous=False, no_pad=False) self.smol.cmd_send(send_args, self.store) @staticmethod diff --git a/src/pneumatube/lookup_server.py b/src/pneumatube/lookup_server.py index a0a1c11..cb9c4e5 100644 --- a/src/pneumatube/lookup_server.py +++ b/src/pneumatube/lookup_server.py @@ -1,10 +1,11 @@ """Postfix tcp_table(5) lookup service, backing check_recipient_access. Without this, Postfix's smtpd accepts a recipient into its queue before ever -asking pneumatube whether the username exists — the live RESOLVE check only -runs later, when the queue manager delivers over LMTP, so an unknown user -gets a backscatter-prone bounce instead of a live SMTP-time reject. Wiring -this service into smtpd_recipient_restrictions moves that same RESOLVE check +asking pneumatube whether the username exists and has accepted the gateway +(SPEC.md S5.8) — those checks only run later, when the queue manager +delivers over LMTP, so a reject becomes a backscatter-prone bounce instead +of a live SMTP-time reject. Wiring this service into +smtpd_recipient_restrictions moves the same checks handle_RCPT already does to RCPT TO time, before Postfix commits to accepting the message. Protocol (tcp_table(5)): one request per line, `get `, where key and @@ -33,12 +34,19 @@ async def _lookup(bridge: EmailToSmolBridge, worker: SmolWorker, config: Config, # to a bare-domain lookup after a miss) — no opinion. return "500 not found" - exists = await worker.run(bridge.user_exists, username.lower()) + username = username.lower() + exists = await worker.run(bridge.user_exists, username) if exists is None: return "400 temporary failure resolving recipient" - if exists: - return "200 " + quote("OK") - return "200 " + quote("550 5.1.1 no such user") + if not exists: + return "200 " + quote("550 5.1.1 no such user") + + if config.require_accepted_recipient: + accepted = await worker.run(bridge.recipient_accepted_gateway, username) + if not accepted: + return "200 " + quote("550 5.7.1 recipient has not accepted mail from this gateway") + + return "200 " + quote("OK") async def _handle_connection(bridge, worker, config, reader, writer) -> None: diff --git a/src/pneumatube/smol_to_email.py b/src/pneumatube/smol_to_email.py index 1956661..d9a5d37 100644 --- a/src/pneumatube/smol_to_email.py +++ b/src/pneumatube/smol_to_email.py @@ -5,6 +5,27 @@ the Email-To frontmatter field (SPEC.md S5.5) to a real mailbox over SMTP. A message without that field is left in the local inbox rather than dropped, so it stays visible to `smolmail.py list`/`read` against the gateway's own db for manual follow-up. + +By default (PNEUMATUBE_REQUIRE_ACCEPTED_SENDER=1, config.require_accepted_ +sender), only the main tier is relayed: a first-contact sender with no +accept token (SPEC.md S5.8) lands in the requests tier and is left alone, +so the gateway can't be used as an open relay by anyone who merely learns +its smolmail address. Setting the flag to false restores the earlier "relay +anyone" behavior for a deliberately open/public bridge. + +Unblocking a sender takes two steps, not one -- confirmed live, since it is +easy to assume `accept` alone is enough: + 1. `smolmail.py --key gateway.key --db gateway.db accept ` + records the acceptance locally and uploads the token to the server, but + does NOT retroactively reclassify a message already sitting in this + gateway's local inbox (its tier was decided at fetch time) or do + anything the sender can see. + 2. The sender only learns the token when it arrives in a reply's Accept + frontmatter field (SPEC.md S5.8) -- which pneumatube itself never sends, + since this direction only relays outward to SMTP. The operator must + send at least one message back from the gateway's own identity (e.g. + `echo hi | smolmail.py --key gateway.key --db gateway.db send
`) + before the sender's *next* message will actually carry a valid token. """ from __future__ import annotations @@ -29,10 +50,20 @@ class SmolToEmailBridge: def poll_once(self) -> None: self.smol.cmd_fetch(self.fetch_args, self.store) - # "all" rather than "inbox": a gateway must relay unsolicited senders - # too, so requests-tier mail (SPEC.md S5.8) is not silently skipped. - for mid, envelope, _received_at, _recipient in self.store.mail("all"): - self._relay(mid, envelope) + # "inbox" (main tier only) when gating on acceptance; "all" includes + # the requests tier too, for a deliberately open bridge. + box = "inbox" if self.config.require_accepted_sender else "all" + for mid, envelope, _received_at, _recipient in self.store.mail(box): + try: + self._relay(mid, envelope) + except Exception: + # One message's failure must not block every message behind + # it: _relay() only deletes on success, so a message that + # keeps raising here (e.g. smtp_client.send() rejecting a + # value _relay()'s own except clause doesn't cover) would + # otherwise jam this entire direction forever -- it always + # sorts first by received_at on the next poll. + logger.exception("failed to relay %s; left for manual review", mid.hex()[:16]) def _relay(self, mid: bytes, envelope: bytes) -> None: try: diff --git a/src/pneumatube/smtp_client.py b/src/pneumatube/smtp_client.py index 0d447ec..0879abd 100644 --- a/src/pneumatube/smtp_client.py +++ b/src/pneumatube/smtp_client.py @@ -26,6 +26,14 @@ def validate(address: str) -> str: def send(config: Config, *, to_addr: str, subject: str, body: str, info_header: str) -> None: to_addr = validate(to_addr) message = EmailMessage() + # Fixed, not derived from the smolmail sender: that sender's address + # isn't a deliverable email address and isn't authorized (SPF/DKIM) to + # send as anything but this one, operator-configured address. A + # per-sender plus-address under this same domain (e.g. + # "gateway+alice@example.com") would stay within that authorization + # while still distinguishing senders and could route a reply back to + # the originating smolmail identity -- a real option if useful later, + # just not implemented. message["From"] = config.gateway_from_address message["To"] = to_addr message["Subject"] = subject diff --git a/src/pneumatube/smtp_server.py b/src/pneumatube/smtp_server.py index 6d87df1..7e667e9 100644 --- a/src/pneumatube/smtp_server.py +++ b/src/pneumatube/smtp_server.py @@ -8,9 +8,12 @@ the internet it has no TLS, rate limiting or rDNS checks. handle_RCPT does a live smolmail RESOLVE per recipient, so an unknown username is rejected inline during the conversation rather than bounced -later. handle_DATA replies once per accepted recipient (joined by CRLF) -instead of a single pass/fail for the whole message, which is what makes -LMTP's per-recipient status semantics real rather than cosmetic. +later. By default it also requires the recipient to have accepted the +gateway (PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT; see email_to_smol.py's +module docstring) -- an all-or-nothing reject, not a demotion. handle_DATA +replies once per accepted recipient (joined by CRLF) instead of a single +pass/fail for the whole message, which is what makes LMTP's per-recipient +status semantics real rather than cosmetic. """ from __future__ import annotations @@ -42,12 +45,16 @@ class GatewayHandler: if not exists: return "550 no such user" + if self.config.require_accepted_recipient: + accepted = await self.worker.run(self.bridge.recipient_accepted_gateway, username) + if not accepted: + return "550 5.7.1 recipient has not accepted mail from this gateway" + envelope.rcpt_tos.append(f"{username}@{domain}") return "250 OK" async def handle_DATA(self, server, session, envelope): message = email_module.message_from_bytes(envelope.content) - sender = envelope.mail_from subject = str(message.get("Subject", "(no subject)")) body = self.bridge.extract_text(message) @@ -55,7 +62,7 @@ class GatewayHandler: for address in envelope.rcpt_tos: username = address.split("@", 1)[0] try: - await self.worker.run(self.bridge.forward, username, sender, subject, body) + await self.worker.run(self.bridge.forward, username, subject, body) replies.append("250 2.1.5 delivered") except Exception: logger.exception("failed to forward mail for %s", username) diff --git a/tests/test_email_to_smol.py b/tests/test_email_to_smol.py index 4ce93f5..7fbae6a 100644 --- a/tests/test_email_to_smol.py +++ b/tests/test_email_to_smol.py @@ -1,7 +1,60 @@ from email.message import EmailMessage +from types import SimpleNamespace from pneumatube.email_to_smol import EmailToSmolBridge, _single_line +_DEFAULT_PORT = 1961 + + +class _Address: + def __init__(self, user, host, port): + self.user, self.host, self.port = user, host, port + + @property + def short(self): + return f"{self.user}@{self.host}" + ("" if self.port == _DEFAULT_PORT else f":{self.port}") + + +class _StubAddressClass: + @staticmethod + def parse(text): + user, _, hostport = text.partition("@") + host, _, port = hostport.partition(":") + return _Address(user, host, int(port) if port else _DEFAULT_PORT) + + +class _StubSmol: + Address = _StubAddressClass + + +class _StubStore: + def __init__(self, tokens): + self.tokens = tokens + + def one(self, sql, address): + return (b"token",) if address in self.tokens else None + + +def test_recipient_accepted_gateway_true_when_token_held(): + bridge = EmailToSmolBridge(_StubSmol(), _StubStore({"alice@smol.example"}), + config=SimpleNamespace(smolmail_domain="smol.example")) + assert bridge.recipient_accepted_gateway("alice") is True + + +def test_recipient_accepted_gateway_false_when_no_token(): + bridge = EmailToSmolBridge(_StubSmol(), _StubStore(set()), + config=SimpleNamespace(smolmail_domain="smol.example")) + assert bridge.recipient_accepted_gateway("alice") is False + + +def test_recipient_accepted_gateway_normalizes_default_port(): + """A configured smolmail_domain carrying an explicit default port must + still match a token stored under Address.short's normalized form (which + drops it) -- naive string concatenation would never match.""" + bridge = EmailToSmolBridge(_StubSmol(), _StubStore({"alice@smol.example"}), + config=SimpleNamespace(smolmail_domain="smol.example:1961")) + assert bridge.recipient_accepted_gateway("alice") is True + def test_single_line_strips_crlf(): assert _single_line("alice@example.org\r\nAccept: forged") == "alice@example.org Accept: forged" diff --git a/tests/test_lookup_server.py b/tests/test_lookup_server.py index 6c2bd6f..a01fadb 100644 --- a/tests/test_lookup_server.py +++ b/tests/test_lookup_server.py @@ -8,22 +8,27 @@ from pneumatube.lookup_server import _lookup class _StubBridge: - def __init__(self, known): + def __init__(self, known, accepted=None): self.known = known + self.accepted = known if accepted is None else accepted def user_exists(self, username): if username == "broken": return None return username in self.known + def recipient_accepted_gateway(self, username): + return username in self.accepted + class _InlineWorker: async def run(self, fn, *args): return fn(*args) -def _config(): - return SimpleNamespace(email_domain="gateway.example") +def _config(**overrides): + defaults = {"email_domain": "gateway.example", "require_accepted_recipient": True} + return SimpleNamespace(**{**defaults, **overrides}) def test_lookup_known_user_returns_ok(): @@ -52,3 +57,20 @@ def test_lookup_other_domain_has_no_opinion(): bridge = _StubBridge({"alice"}) reply = asyncio.run(_lookup(bridge, _InlineWorker(), _config(), "alice@other.example")) assert reply == "500 not found" + + +def test_lookup_rejects_unaccepted_recipient_by_default(): + bridge = _StubBridge({"alice"}, accepted=set()) + reply = asyncio.run(_lookup(bridge, _InlineWorker(), _config(), "alice@gateway.example")) + code, _, text = reply.partition(" ") + assert code == "200" + assert unquote(text) == "550 5.7.1 recipient has not accepted mail from this gateway" + + +def test_lookup_allows_unaccepted_recipient_when_disabled(): + bridge = _StubBridge({"alice"}, accepted=set()) + config = _config(require_accepted_recipient=False) + reply = asyncio.run(_lookup(bridge, _InlineWorker(), config, "alice@gateway.example")) + code, _, text = reply.partition(" ") + assert code == "200" + assert unquote(text) == "OK" diff --git a/tests/test_smol_to_email.py b/tests/test_smol_to_email.py new file mode 100644 index 0000000..fcf89c0 --- /dev/null +++ b/tests/test_smol_to_email.py @@ -0,0 +1,65 @@ +from types import SimpleNamespace + +from pneumatube.smol_to_email import SmolToEmailBridge + + +class _StubSmol: + SmolError = RuntimeError + + def cmd_fetch(self, args, store): + pass + + +class _StubStore: + def __init__(self, rows): + self._rows = rows + self.requested_box = None + + def mail(self, box): + self.requested_box = box + return self._rows + + +def _config(**overrides): + defaults = {"smolmail_key": "k", "smolmail_db": "d", "require_accepted_sender": True} + return SimpleNamespace(**{**defaults, **overrides}) + + +def test_poll_once_does_not_stop_after_one_message_fails(): + """A message that keeps raising out of _relay() (e.g. a value + smtp_client.send() rejects that _relay()'s own except clause doesn't + cover) must not block every message behind it -- it always sorts first + on the next poll, since only a successful _relay() deletes it.""" + rows = [(b"poison", b"", 0, None), (b"ok", b"", 1, None)] + bridge = SmolToEmailBridge(_StubSmol(), _StubStore(rows), account=None, config=_config()) + + relayed = [] + + def fake_relay(mid, envelope): + if mid == b"poison": + raise ValueError("simulated failure unrelated to InvalidAddress/OSError") + relayed.append(mid) + + bridge._relay = fake_relay + bridge.poll_once() + + assert relayed == [b"ok"] + + +def test_poll_once_uses_main_tier_only_by_default(): + """require_accepted_sender=True (the default) must poll "inbox" (main + tier), not "all" -- "all" would include requests-tier mail from + first-contact senders with no accept token (SPEC.md S5.8), which is + exactly what this setting exists to keep the gateway from relaying.""" + store = _StubStore([]) + bridge = SmolToEmailBridge(_StubSmol(), store, account=None, config=_config()) + bridge.poll_once() + assert store.requested_box == "inbox" + + +def test_poll_once_uses_all_tiers_when_open_bridge_enabled(): + store = _StubStore([]) + bridge = SmolToEmailBridge(_StubSmol(), store, account=None, + config=_config(require_accepted_sender=False)) + bridge.poll_once() + assert store.requested_box == "all" diff --git a/tests/test_smtp_server.py b/tests/test_smtp_server.py index 775307e..e47eec4 100644 --- a/tests/test_smtp_server.py +++ b/tests/test_smtp_server.py @@ -7,14 +7,22 @@ from pneumatube.smtp_server import GatewayHandler class _StubBridge: - def __init__(self): + def __init__(self, known=("alice", "bob", "broken"), accepted=("alice", "bob", "broken")): self.forwarded = [] + self.known = set(known) + self.accepted = set(accepted) - def forward(self, username, sender, subject, body): + def forward(self, username, subject, body): self.forwarded.append(username) if username == "broken": raise RuntimeError("smolmail server unreachable") + def user_exists(self, username): + return username in self.known + + def recipient_accepted_gateway(self, username): + return username in self.accepted + @staticmethod def extract_text(message): return message.get_payload(decode=True).decode("utf-8", "replace") @@ -36,9 +44,58 @@ def _envelope(rcpt_tos): ) +def _config(**overrides): + defaults = {"email_domain": "gateway.example", "require_accepted_recipient": True} + return SimpleNamespace(**{**defaults, **overrides}) + + +def test_handle_rcpt_rejects_wrong_domain(): + handler = GatewayHandler(_StubBridge(), _InlineWorker(), config=_config()) + reply = asyncio.run(handler.handle_RCPT(None, None, _envelope([]), + "alice@other.example", None)) + assert reply.startswith("550") + + +def test_handle_rcpt_rejects_unknown_user(): + handler = GatewayHandler(_StubBridge(known=()), _InlineWorker(), config=_config()) + reply = asyncio.run(handler.handle_RCPT(None, None, _envelope([]), + "ghost@gateway.example", None)) + assert reply.startswith("550") + + +def test_handle_rcpt_rejects_unaccepted_recipient_by_default(): + """The all-or-nothing gate: a recipient who exists but has not accepted + the gateway (no token held) must be rejected live, not silently demoted + to the recipient's own requests tier.""" + bridge = _StubBridge(known=("alice",), accepted=()) + handler = GatewayHandler(bridge, _InlineWorker(), config=_config()) + reply = asyncio.run(handler.handle_RCPT(None, None, _envelope([]), + "alice@gateway.example", None)) + assert reply.startswith("550") + + +def test_handle_rcpt_allows_unaccepted_recipient_when_disabled(): + bridge = _StubBridge(known=("alice",), accepted=()) + handler = GatewayHandler(bridge, _InlineWorker(), + config=_config(require_accepted_recipient=False)) + envelope = _envelope([]) + reply = asyncio.run(handler.handle_RCPT(None, None, envelope, "alice@gateway.example", None)) + assert reply.startswith("250") + assert envelope.rcpt_tos == ["alice@gateway.example"] + + +def test_handle_rcpt_accepts_known_accepted_recipient(): + bridge = _StubBridge(known=("alice",), accepted=("alice",)) + handler = GatewayHandler(bridge, _InlineWorker(), config=_config()) + envelope = _envelope([]) + reply = asyncio.run(handler.handle_RCPT(None, None, envelope, "alice@gateway.example", None)) + assert reply.startswith("250") + assert envelope.rcpt_tos == ["alice@gateway.example"] + + def test_handle_data_replies_once_per_recipient(): bridge = _StubBridge() - handler = GatewayHandler(bridge, _InlineWorker(), config=SimpleNamespace()) + handler = GatewayHandler(bridge, _InlineWorker(), config=_config()) envelope = _envelope(["alice@gateway.example", "broken@gateway.example"]) reply = asyncio.run(handler.handle_DATA(None, None, envelope)) @@ -52,7 +109,7 @@ def test_handle_data_replies_once_per_recipient(): def test_handle_data_all_succeed(): bridge = _StubBridge() - handler = GatewayHandler(bridge, _InlineWorker(), config=SimpleNamespace()) + handler = GatewayHandler(bridge, _InlineWorker(), config=_config()) envelope = _envelope(["alice@gateway.example", "bob@gateway.example"]) reply = asyncio.run(handler.handle_DATA(None, None, envelope))