randogoth/pneumatube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: port pneumatube to Rust on fumi-core

Browse source
This commit is contained in:
randogoth 2026-10-05 11:25:11 +03:00
parent 5dda6567af
commit 65d52bd0e9
37 changed files with 4232 additions and 1912 deletions
Download patch file Download diff file Expand all files Collapse all files

386
scripts/security_test.py
View file

@ -1,58 +1,57 @@
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.11"
# dependencies = ["aiosmtpd>=1.4", "pynacl>=1.5", "noiseprotocol>=0.3.1"]
# ///
"""Adversarial testing for pneumatube, run against your own deployment.
Covers (see each test_* function's docstring for detail):
- injection fuzzing of the Email-To / subject paths
- malformed-MIME resilience of extract_text
- per-message failure isolation in the smolmail -> email poll loop
- username enumeration via the Postfix lookup service's own protocol
Covers (see each check's docstring for detail):
- abuse/open-relay behaviour and the accept-gate mitigations for each
direction (PNEUMATUBE_REQUIRE_ACCEPTED_SENDER, _RECIPIENT), including
their explicit opt-out modes
- username enumeration via the Postfix lookup service's own protocol
- sender spoofing: confirms a forged MAIL FROM is never forwarded into
the relayed smolmail message (email -> smolmail carries no Email-From
field at all, precisely because MAIL FROM is unauthenticated)
The unit-style checks the Python version used to run here (address
validation fuzz, the frontmatter sanitizer, the CR/LF header backstop,
malformed-MIME resilience, poll-loop failure isolation) live in the Rust
implementation's `cargo test` suite now -- see the tests in
src/smtp_client.rs, src/email_to_smol.rs and src/smol_to_email.rs.
Each check prints PASS (a defense verified), RISK (confirmed, accepted
design tradeoff -- not a bug, but worth knowing it's real), or BUG
(unexpected, worth fixing).
Usage:
uv run python scripts/security_test.py
uv run python scripts/security_test.py --skip-live # unit-style checks only
uv run scripts/security_test.py
Needs a `smolmail` checkout next to this repo for the live checks (same
default pneumatube itself uses: PNEUMATUBE_SMOLMAIL_REPO, default
"../smolmail").
Needs a `bunshin` and a `fumi` binary on PATH (or --server-bin/--fumi-bin),
a built pneumatube (--pneumatube-bin, default target/release/pneumatube),
and a `smolmail` checkout next to this repo for the load-driving client
(--smolmail-repo, default "../smolmail").
"""
from __future__ import annotations
import argparse
import contextlib
import email as email_module
import logging
import os
import subprocess
import sys
import socket
import tempfile
import time
from dataclasses import dataclass
from email.message import EmailMessage
from pathlib import Path
from urllib.parse import unquote
from aiosmtpd.controller import Controller
sys.path.insert(0, str(Path(__file__).resolve().parent))
from stress_test import ( # noqa: E402
REPO_ROOT, CountingSink, SmolAccount, SmolServer, free_port, make_account,
send_one_lmtp, send_one_smol, start_smolmaild, wait_for_port,
from stress_test import (
REPO_ROOT, CountingSink, free_port, gateway_env,
load_smolmail, make_account, make_gateway_account, send_one_lmtp,
send_one_smol, start_bunshin, start_pneumatube, stop_proc,
)
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
from pneumatube import smtp_client, vendor # noqa: E402
from pneumatube.email_to_smol import EmailToSmolBridge, _single_line # noqa: E402
from pneumatube.lookup_server import _lookup # noqa: E402
@dataclass
class Finding:
@ -74,255 +73,19 @@ def record(check: str, kind: str, detail: str) -> None:
FINDINGS.append(f)
# --- injection fuzzing (no live infra needed) -----------------------------------
def lookup_get(host: str, port: int, key: str) -> str:
with socket.create_connection((host, port), timeout=10) as s:
s.sendall(f"get {key}\n".encode())
return s.recv(4096).decode().strip()
INJECTION_PAYLOADS = [
("bare-lf", "evil@example.com\nX-Injected: 1"),
("bare-cr", "evil@example.com\rX-Injected: 1"),
("crlf", "evil@example.com\r\nBcc: evil@example.com"),
("nul-byte", "evil@example.com\x00X-Injected: 1"),
("unicode-line-sep", "evil@example.com
X-Injected: 1"),
("unicode-para-sep", "evil@example.com
X-Injected: 1"),
("very-long", "a" * 10_000 + "@example.com"),
]
def test_address_validation_fuzz() -> None:
"""smtp_client.validate() is the only gate between an attacker-supplied
Email-To frontmatter value and smtplib.SMTP's RCPT TO -- it must reject
every CR/LF variant, not just the literal \\r\\n case the unit tests
already cover."""
failures = []
for name, payload in INJECTION_PAYLOADS:
try:
smtp_client.validate(payload)
# unicode-line-sep/para-sep and very-long aren't CR/LF and may
# legitimately pass the regex; only bare-lf/bare-cr/crlf/nul-byte
# must always be rejected.
if name in ("bare-lf", "bare-cr", "crlf", "nul-byte"):
failures.append(name)
except smtp_client.InvalidAddress:
pass
record("address validation fuzz", "pass" if not failures else "bug",
"all CR/LF/NUL variants rejected" if not failures
else f"accepted dangerous payloads: {failures}")
def test_single_line_fuzz() -> None:
"""email_to_smol._single_line() is what stops a forged email Subject
from injecting extra smolmail frontmatter fields (SPEC.md S5.5: one
`Key: value` per line) into an otherwise-trusted message. (MAIL FROM no
longer flows into frontmatter at all -- see email_to_smol.py's module
docstring -- but Subject still does.)"""
failures = []
for name, payload in INJECTION_PAYLOADS:
cleaned = _single_line(payload)
if "\n" in cleaned or "\r" in cleaned:
failures.append(name)
record("frontmatter sanitizer fuzz", "pass" if not failures else "bug",
"no payload left a raw CR/LF in the sanitized value" if not failures
else f"raw CR/LF survived sanitization: {failures}")
def test_outbound_header_injection() -> None:
"""Even though frontmatter values can't structurally contain a raw
newline (smolmail.py's parser splits the body into single lines before a
value is ever extracted), smtp_client.send() has no explicit sanitizer
of its own -- it currently relies entirely on that upstream guarantee
plus Python's EmailMessage rejecting embedded CR/LF. Confirm the second
layer actually holds, since the first is an implicit dependency on
vendored code outside this project."""
try:
m = EmailMessage()
m["Subject"] = "hi\r\nBcc: evil@attacker.example"
rejected = False
except ValueError:
rejected = True
record("stdlib header-injection backstop", "pass" if rejected else "bug",
"EmailMessage rejects embedded CR/LF in header values (confirmed)" if rejected
else "EmailMessage accepted an embedded CR/LF -- smtp_client.send() has no "
"sanitizer of its own and would forward it verbatim")
# --- malformed MIME resilience (no live infra needed) ---------------------------
def test_malformed_mime_resilience() -> None:
"""extract_text() runs on every inbound email before any validation --
it must not crash or hang on adversarial MIME, since a single crash
taking down the LMTP handler would be a trivial DoS against every other
in-flight recipient in the same message."""
cases = {
"empty": b"",
"no-body": b"Subject: hi\r\n\r\n",
"non-utf8-bytes": b"Subject: hi\r\nContent-Type: text/plain\r\n\r\n\xff\xfe\x00\x01",
"multipart-no-text-part": (
b"Content-Type: multipart/mixed; boundary=X\r\n\r\n--X\r\n"
b"Content-Type: application/octet-stream\r\n\r\nbinary\r\n--X--\r\n"),
"deeply-nested-multipart": _nested_multipart(50),
"declared-huge-missing-body": (
b"Content-Type: text/plain\r\nContent-Length: 999999999\r\n\r\nshort"),
}
failures = []
for name, raw in cases.items():
try:
message = email_module.message_from_bytes(raw)
t0 = time.monotonic()
EmailToSmolBridge.extract_text(message)
if time.monotonic() - t0 > 2.0:
failures.append(f"{name} (slow: possible algorithmic DoS)")
except Exception as exc:
failures.append(f"{name} ({exc!r})")
record("malformed MIME resilience", "pass" if not failures else "bug",
"all cases handled without raising or stalling" if not failures
else f"crashed or stalled on: {failures}")
def _nested_multipart(depth: int) -> bytes:
body = b"leaf"
for i in range(depth):
boundary = f"B{i}".encode()
body = (b"Content-Type: multipart/mixed; boundary=" + boundary + b"\r\n\r\n--"
+ boundary + b"\r\n" + body + b"\r\n--" + boundary + b"--\r\n")
return b"Content-Type: multipart/mixed; boundary=B0\r\n\r\n--B0\r\n" + body + b"\r\n--B0--\r\n"
# --- per-message failure isolation (no live infra needed) ----------------------
def test_poll_loop_isolates_failures() -> None:
"""A message that makes _relay() raise anything other than
InvalidAddress/OSError must not block every message behind it in the
smolmail -> email direction -- see smol_to_email.py's poll_once() for
why (a poisoned message is never deleted, so it sorts first forever)."""
from types import SimpleNamespace
from pneumatube.smol_to_email import SmolToEmailBridge
class StubSmol:
SmolError = RuntimeError
def cmd_fetch(self, args, store):
pass
class StubStore:
def mail(self, box):
return [(b"poison", b"", 0, None), (b"ok", b"", 1, None)]
bridge = SmolToEmailBridge(StubSmol(), StubStore(), account=None,
config=SimpleNamespace(smolmail_key="k", smolmail_db="d",
require_accepted_sender=True))
relayed = []
def fake_relay(mid, envelope):
if mid == b"poison":
raise ValueError("simulated")
relayed.append(mid)
bridge._relay = fake_relay
# poll_once() logs the simulated failure via logger.exception(), which is
# the whole point of the fix -- quiet it here so it doesn't look like an
# unhandled crash in this script's own output.
logging.getLogger("pneumatube.smol_to_email").setLevel(logging.CRITICAL)
try:
bridge.poll_once()
record("poll loop failure isolation", "pass" if relayed == [b"ok"] else "bug",
"later messages still processed after an earlier one raised" if relayed == [b"ok"]
else "poll_once stopped after the first failure -- queue-jam DoS")
except Exception as exc:
record("poll loop failure isolation", "bug",
f"poll_once raised instead of isolating the failure: {exc!r}")
# --- username enumeration via the lookup service (no live infra needed) --------
def test_lookup_enumeration() -> None:
"""The Postfix lookup service (lookup_server.py) answers definitively
per username with no authentication and no rate limiting of its own --
by design, it's meant to be reachable only from Postfix on a private
network. Confirm that property so the deployment docs' "loopback only"
requirement is understood as load-bearing, not cosmetic."""
import asyncio
class StubBridge:
def user_exists(self, username):
return username in ("alice", "bob")
def recipient_accepted_gateway(self, username):
return True
class InlineWorker:
async def run(self, fn, *args):
return fn(*args)
from types import SimpleNamespace
config = SimpleNamespace(email_domain="gateway.example", require_accepted_recipient=True)
bridge, worker = StubBridge(), InlineWorker()
# tcp_table(5) status is "200" either way (it means "I have an answer");
# the OK-vs-reject distinction Postfix actually acts on is in the value.
results = {}
for name in ("alice", "bob", "carol", "admin", "root", "postmaster"):
reply = asyncio.run(_lookup(bridge, worker, config, f"{name}@gateway.example"))
_code, _, value = reply.partition(" ")
results[name] = value == "OK"
distinguishable = results["alice"] and results["bob"] and not any(
results[n] for n in ("carol", "admin", "root", "postmaster"))
record("lookup service enumeration", "risk" if distinguishable else "bug",
"the lookup protocol unambiguously reveals which usernames exist -- expected, "
"it must for check_recipient_access to work, but this is only safe because the "
"service is loopback-only; exposing it beyond Postfix would be a username "
"enumeration oracle" if distinguishable else
"lookup responses did not distinguish known from unknown usernames as expected "
"-- check_recipient_access would misbehave")
# --- live tests: abuse/throttling and sender spoofing ---------------------------
def _start_pneumatube(smolmail_repo: Path, gateway: SmolAccount, server: SmolServer,
sink_port: int, require_accepted_sender: bool = True,
require_accepted_recipient: bool = True
) -> tuple[subprocess.Popen, int]:
listen_port = free_port()
env = {**os.environ,
"PNEUMATUBE_SMOLMAIL_REPO": str(smolmail_repo),
"PNEUMATUBE_SMOLMAIL_KEY": str(gateway.key_path),
"PNEUMATUBE_SMOLMAIL_DB": str(gateway.db_path),
"PNEUMATUBE_SMOLMAIL_DOMAIN": f"{server.host}:{server.port}",
"PNEUMATUBE_EMAIL_DOMAIN": "gateway.example",
"PNEUMATUBE_FETCH_INTERVAL": "2",
"PNEUMATUBE_REQUIRE_ACCEPTED_SENDER": "1" if require_accepted_sender else "0",
"PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT": "1" if require_accepted_recipient else "0",
"PNEUMATUBE_LISTEN_PROTOCOL": "lmtp",
"PNEUMATUBE_LISTEN_HOST": "127.0.0.1",
"PNEUMATUBE_LISTEN_PORT": str(listen_port),
"PNEUMATUBE_LOOKUP_HOST": "127.0.0.1",
"PNEUMATUBE_LOOKUP_PORT": str(free_port()),
"PNEUMATUBE_SMTP_RELAY_HOST": "127.0.0.1",
"PNEUMATUBE_SMTP_RELAY_PORT": str(sink_port),
"PNEUMATUBE_SMTP_RELAY_STARTTLS": "0",
"PNEUMATUBE_GATEWAY_FROM_ADDRESS": "gateway@gateway.example"}
proc = subprocess.Popen(["uv", "run", "pneumatube"], cwd=REPO_ROOT, env=env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
wait_for_port("127.0.0.1", listen_port)
return proc, listen_port
def _stop(proc: subprocess.Popen) -> None:
proc.terminate()
with contextlib.suppress(Exception):
proc.wait(timeout=5)
def run_live_tests(smolmail_repo: Path) -> None:
def run_live_tests(args: argparse.Namespace) -> None:
smolmail_repo = Path(args.smolmail_repo).resolve()
smol = load_smolmail(smolmail_repo)
with tempfile.TemporaryDirectory(prefix="pneumatube-sectest-") as tmp:
tmp_dir = Path(tmp)
server = start_smolmaild(smolmail_repo, tmp_dir, rate_connections=6000, rate_sends=6000)
smol = vendor.load(smolmail_repo)
gateway = make_account(smol, tmp_dir, "gatewayuser", server)
server = start_bunshin(args.server_bin, tmp_dir, rate_connections=6000, rate_sends=6000)
gateway = make_gateway_account(args.fumi_bin, tmp_dir, "gatewayuser", server)
victim = make_account(smol, tmp_dir, "victim", server)
spoof_victim = make_account(smol, tmp_dir, "spoofvictim", server)
@ -331,13 +94,16 @@ def run_live_tests(smolmail_repo: Path) -> None:
sink_controller = Controller(sink, hostname="127.0.0.1", port=sink_port)
sink_controller.start()
# -- smolmail -> email, default config (require_accepted_sender=True) --
# 15 messages from 15 DIFFERENT, never-before-seen sender identities
# (no accept token, no prior relationship), each addressed to a
# different external destination. None should reach the sink.
proc, listen_port = _start_pneumatube(smolmail_repo, gateway, server, sink_port,
require_accepted_sender=True)
# -- default config: both accept gates on --
listen_port, lookup_port = free_port(), free_port()
env = gateway_env(gateway, server, require_accepted_sender=True,
require_accepted_recipient=True)
proc = start_pneumatube(args.pneumatube_bin, env, listen_port, lookup_port, sink_port)
try:
# 15 messages from 15 DIFFERENT, never-before-seen sender
# identities (no accept token, no prior relationship), each
# addressed to a different external destination. None should
# reach the sink.
senders = [make_account(smol, tmp_dir, f"stranger{i}", server) for i in range(15)]
for i, sender in enumerate(senders):
send_one_smol(smol, sender, gateway.address, f"dest{i}@external.example",
@ -349,9 +115,8 @@ def run_live_tests(smolmail_repo: Path) -> None:
f"{delivered}/{len(senders)} first-contact messages reached the outbound "
"sink with PNEUMATUBE_REQUIRE_ACCEPTED_SENDER at its default (1) -- "
+ ("none did, as intended" if delivered == 0 else
"some did; the accept gate in smol_to_email.py is not holding"))
"some did; the accept gate in smol_to_email is not holding"))
# -- email -> smolmail, default config (require_accepted_recipient=True) --
# "victim" is registered on the smolmail server (so user_exists
# passes) but has never accepted the gateway (no token held) --
# the all-or-nothing gate should reject this live, not queue it.
@ -363,18 +128,48 @@ def run_live_tests(smolmail_repo: Path) -> None:
f"outcome={outcome} with PNEUMATUBE_REQUIRE_ACCEPTED_RECIPIENT at its "
"default (1) for a registered-but-never-accepted recipient -- " +
("rejected live at RCPT TO, as intended" if outcome == "rejected" else
"was not rejected; the gate in handle_RCPT/_lookup is not holding"))
"was not rejected; the gate in the LMTP handler and lookup service "
"is not holding"))
# The lookup service answers definitively per username with no
# authentication and no rate limiting of its own -- by design,
# it's meant to be reachable only from Postfix on a private
# network. Confirm that property so the deployment docs'
# "loopback only" requirement is understood as load-bearing,
# not cosmetic. Both accept-gate replies ("550 5.7.1 ...") and
# "550 5.1.1 no such user" reveal which usernames exist, just
# as an "OK" would; the OK-vs-reject distinction Postfix acts
# on is a different axis from the exists-or-not distinction an
# enumerator wants.
results = {}
for name in ("victim", "spoofvictim", "carol", "admin", "root", "postmaster"):
reply = lookup_get("127.0.0.1", lookup_port, f"{name}@gateway.example")
# The reply's action text is %XX-quoted on the wire; a
# known username answers anything but "no such user".
action = unquote(reply.split(" ", 1)[1]) if " " in reply else reply
results[name] = action != "550 5.1.1 no such user"
distinguishable = (results["victim"] and results["spoofvictim"]
and not any(results[n] for n in
("carol", "admin", "root", "postmaster")))
record("lookup service enumeration", "risk" if distinguishable else "bug",
"the lookup protocol unambiguously reveals which usernames exist -- "
"expected, it must for check_recipient_access to work, but this is "
"only safe because the service is loopback-only; exposing it beyond "
"Postfix would be a username enumeration oracle" if distinguishable else
"lookup responses did not distinguish known from unknown usernames as "
"expected -- check_recipient_access would misbehave")
finally:
_stop(proc)
stop_proc(proc)
# -- both directions, explicit opt-in to the open-bridge mode --
# Confirms both flags at 0 still do what they document: restore
# "relay anyone" for an operator who deliberately wants it. victim
# and spoof_victim never accepted the gateway, so this also doubles
# as the opt-out confirmation for the recipient gate.
proc, listen_port = _start_pneumatube(smolmail_repo, gateway, server, sink_port,
require_accepted_sender=False,
require_accepted_recipient=False)
listen_port, lookup_port = free_port(), free_port()
env = gateway_env(gateway, server, require_accepted_sender=False,
require_accepted_recipient=False)
proc = start_pneumatube(args.pneumatube_bin, env, listen_port, lookup_port, sink_port)
try:
senders = [make_account(smol, tmp_dir, f"openbridge{i}", server) for i in range(15)]
for i, sender in enumerate(senders):
@ -409,9 +204,9 @@ def run_live_tests(smolmail_repo: Path) -> None:
# -- sender spoofing --
# Own recipient account, untouched by the open-relay tests above,
# so finding the message isn't a race against unrelated mail.
# email_to_smol.py forwards no claim about the original sender at
# all (see its module docstring) specifically because MAIL FROM
# is unauthenticated; confirm a forged one doesn't leak through
# email_to_smol forwards no claim about the original sender at
# all (see its module docs) specifically because MAIL FROM is
# unauthenticated; confirm a forged one doesn't leak through
# into any frontmatter field of the relayed message.
forged = "security-team@trusted-bank.example"
outcome, _lat = send_one_lmtp(
@ -446,34 +241,29 @@ def run_live_tests(smolmail_repo: Path) -> None:
f"the forged MAIL FROM {forged!r} leaked into the relayed message's "
f"frontmatter: {opened['fields']}")
finally:
_stop(proc)
stop_proc(proc)
sink_controller.stop()
_stop(server.process)
stop_proc(server.process)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--skip-live", action="store_true",
help="run only the unit-style checks (no subprocess/network)")
parser.add_argument("--smolmail-repo", default="../smolmail")
parser.add_argument("--server-bin", default="bunshin")
parser.add_argument("--fumi-bin", default="fumi")
parser.add_argument("--pneumatube-bin", default=str(REPO_ROOT / "target/release/pneumatube"))
args = parser.parse_args()
test_address_validation_fuzz()
test_single_line_fuzz()
test_outbound_header_injection()
test_malformed_mime_resilience()
test_poll_loop_isolates_failures()
test_lookup_enumeration()
if not args.skip_live:
run_live_tests(Path(args.smolmail_repo).resolve())
run_live_tests(args)
print()
bugs = [f for f in FINDINGS if f.kind == "bug"]
risks = [f for f in FINDINGS if f.kind == "risk"]
print(f"{len(FINDINGS)} check(s): {len(FINDINGS) - len(bugs) - len(risks)} pass, "
f"{len(risks)} confirmed accepted risk(s), {len(bugs)} bug(s)")
if bugs:
raise SystemExit(1)
if __name__ == "__main__":