From 0b670697601758e2d9cda6ba228643f2bc1cc0f1 Mon Sep 17 00:00:00 2001 From: randogoth Date: Sat, 26 Sep 2026 15:13:43 +0300 Subject: [PATCH] feat: add Nex and Gopher listeners Reuses md2txt's existing nex and text renderers (both already ship unmodified) -- rendered unwrapped gemtext still works for Gemini/Spartan clients that wrap themselves, but Nex and Gopher clients don't, so those two get the classic 80-column pre-wrap instead. Neither protocol has a redirect status of its own, so Site.resolve_flat() follows the /foo.md -> /foo canonical bounce and the WML-card-only -> parent redirect server-side and hands back real content directly, rather than leaving the two new listeners to invent a redirect convention that doesn't exist on the wire. Co-Authored-By: Claude Sonnet 5 --- README.md | 25 ++++++++--- devbox.json | 2 +- flake.nix | 21 +++++++-- src/smolweb/cli.py | 14 +++++- src/smolweb/servers/gopher.py | 81 +++++++++++++++++++++++++++++++++++ src/smolweb/servers/nex.py | 63 +++++++++++++++++++++++++++ src/smolweb/site.py | 45 ++++++++++++++++++- tests/test_site.py | 52 ++++++++++++++++++++++ 8 files changed, 290 insertions(+), 13 deletions(-) create mode 100644 src/smolweb/servers/gopher.py create mode 100644 src/smolweb/servers/nex.py diff --git a/README.md b/README.md index 722ba43..7e68022 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,19 @@ # smolweb Serve a folder of Markdown as [Gemini](https://geminiprotocol.net/) capsules, -[Spartan](https://portal.mozz.us/gemini/spartan.mozz.us/) capsules, and HTTP -pages content-negotiated between WML (WAP 1.x) and XHTML-MP (WAP 2.0 and -modern browsers) — all rendered on the fly, no build step. +[Spartan](https://portal.mozz.us/gemini/spartan.mozz.us/) capsules, +[Nex](https://nightfall.city/nex/info/specification.txt) and +[Gopher](https://www.rfc-editor.org/rfc/rfc1436) documents, and HTTP pages +content-negotiated between WML (WAP 1.x) and XHTML-MP (WAP 2.0 and modern +browsers) — all rendered on the fly, no build step. ```bash smolweb --root ./content --host example.com ``` Every listener binds by default: Gemini on `:1965` (TLS), Spartan on `:300`, -HTTP on `:8080`. An empty address disables one: +HTTP on `:8080`, Nex on `:1900`, Gopher on `:70`. An empty address disables +one: ```bash smolweb --root ./content --host example.com --spartan "" @@ -21,9 +24,19 @@ smolweb --root ./content --host example.com --spartan "" Markdown is parsed once per file (cached by mtime, re-parsed the moment the file changes) and rendered into every format from the same source: -- **gemtext** via [md2txt](https://code.randogoth.com/randogoth/md2txt)'s `gemini` renderer. +- **gemtext**, **Nex**, and **Gopher text** via [md2txt](https://code.randogoth.com/randogoth/md2txt)'s `gemini`, `nex`, and `text` renderers respectively. - **WML** and **XHTML-MP** via [wapdown](https://code.randogoth.com/randogoth/wapdown). +Gemini and Spartan clients wrap gemtext themselves, so it's rendered +unwrapped (`width=0`). Nex and Gopher clients don't, so those two are +pre-wrapped at the classic 80-column convention instead. + +Neither Nex nor Gopher has a redirect status of its own, so the `/foo.md -> +/foo` canonical-URL bounce and the WML-card-only-path-redirects-to-its- +parent behaviour (see below) are both resolved server-side instead of +bounced back to the client — the client always gets real content back +directly, on the first request. + Both libraries read the same leading `---` frontmatter block independently and ignore keys they don't recognise, so one file can freely mix wapdown's deck-structure keys (`title`, `split_level`, `deck_per_card`, ...) with @@ -53,6 +66,8 @@ A document with `deck_per_card: true` gets its own URL space for WML only: `/trail/weather` serves one card's own deck directly. Gemini, Spartan, and an HTTP client negotiating gemtext/XHTML-MP all redirect `/trail/weather` back to `/trail` instead — that URL space only ever makes sense for WML. +Nex and Gopher serve `/trail`'s own content directly for the same request, +since neither has a redirect status to bounce with. ## HTTP content negotiation diff --git a/devbox.json b/devbox.json index a3908c1..5ba6439 100644 --- a/devbox.json +++ b/devbox.json @@ -10,7 +10,7 @@ "scripts": { "sync": ["uv sync"], "test": ["uv run pytest"], - "run": ["uv run smolweb --root ./content --host localhost --spartan :3000"] + "run": ["uv run smolweb --root ./content --host localhost --spartan :3000 --gopher :7070"] } } } diff --git a/flake.nix b/flake.nix index 4748833..55bcf95 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "Serve a folder of Markdown as gemtext, Spartan, WML, and XHTML-MP"; + description = "Serve a folder of Markdown as gemtext, Nex, Gopher, WML, and XHTML-MP"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; @@ -93,6 +93,16 @@ default = ":8080"; description = "HTTP listen address; empty disables it."; }; + nexAddr = mkOption { + type = types.str; + default = ":1900"; + description = "Nex listen address; empty disables it."; + }; + gopherAddr = mkOption { + type = types.str; + default = ":70"; + description = "Gopher listen address; empty disables it."; + }; }; config = lib.mkIf cfg.enable { @@ -112,12 +122,15 @@ ++ lib.optionals (cfg.geminiAddr != "") [ "--gemini" cfg.geminiAddr ] ++ lib.optionals (cfg.spartanAddr != "") [ "--spartan" cfg.spartanAddr ] ++ lib.optionals (cfg.httpAddr != "") [ "--http" cfg.httpAddr ] + ++ lib.optionals (cfg.nexAddr != "") [ "--nex" cfg.nexAddr ] + ++ lib.optionals (cfg.gopherAddr != "") [ "--gopher" cfg.gopherAddr ] ); DynamicUser = true; StateDirectory = "smolweb"; - # Spartan's default port (300) and Gemini's TLS handshake - # both need this rather than running the whole service as - # root -- DynamicUser alone can't bind a privileged port. + # Spartan's default port (300), Gopher's default port (70), + # and Gemini's TLS handshake all need this rather than + # running the whole service as root -- DynamicUser alone + # can't bind a privileged port. AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; NoNewPrivileges = true; ProtectSystem = "strict"; diff --git a/src/smolweb/cli.py b/src/smolweb/cli.py index 00648cd..d7aff01 100644 --- a/src/smolweb/cli.py +++ b/src/smolweb/cli.py @@ -1,4 +1,4 @@ -"""smolweb: serve a folder of Markdown as gemtext, Spartan, WML, and XHTML-MP.""" +"""smolweb: serve a folder of Markdown as gemtext, Nex, Gopher, WML, and XHTML-MP.""" from __future__ import annotations import argparse @@ -10,7 +10,9 @@ from typing import Tuple from .cert import ensure_certificate from .servers.gemini import GeminiServer +from .servers.gopher import GopherServer from .servers.http import SmolwebHTTPServer +from .servers.nex import NexServer from .servers.spartan import SpartanServer from .site import Site @@ -31,6 +33,8 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--gemini", default=":1965", metavar="[HOST]:PORT", help="Gemini listen address; empty disables it (default: :1965).") parser.add_argument("--spartan", default=":300", metavar="[HOST]:PORT", help="Spartan listen address; empty disables it (default: :300).") parser.add_argument("--http", default=":8080", metavar="[HOST]:PORT", help="HTTP listen address; empty disables it (default: :8080).") + parser.add_argument("--nex", default=":1900", metavar="[HOST]:PORT", help="Nex listen address; empty disables it (default: :1900).") + parser.add_argument("--gopher", default=":70", metavar="[HOST]:PORT", help="Gopher listen address; empty disables it (default: :70).") parser.add_argument("--cert", type=Path, default=Path("cert.pem"), help="TLS certificate path, generated on first run if missing.") parser.add_argument("--key", type=Path, default=Path("key.pem"), help="TLS key path, generated on first run if missing.") return parser @@ -67,6 +71,14 @@ def main(argv=None) -> int: http_addr = _parse_address(args.http) threads.append(_serve_in_thread(SmolwebHTTPServer(http_addr, site), "http", http_addr)) + if args.nex: + nex_addr = _parse_address(args.nex) + threads.append(_serve_in_thread(NexServer(nex_addr, site), "nex", nex_addr)) + + if args.gopher: + gopher_addr = _parse_address(args.gopher) + threads.append(_serve_in_thread(GopherServer(gopher_addr, site), "gopher", gopher_addr)) + if not threads: sys.stderr.write("smolweb: every listener disabled, nothing to do\n") return 2 diff --git a/src/smolweb/servers/gopher.py b/src/smolweb/servers/gopher.py new file mode 100644 index 0000000..0bae210 --- /dev/null +++ b/src/smolweb/servers/gopher.py @@ -0,0 +1,81 @@ +"""Gopher protocol listener (:70, plaintext TCP, RFC 1436). + +Only type-0 (text) document retrieval is implemented -- no gophermap +directory listings, no type-7 search. A selector is read up to the first +CR-LF or TAB (a TAB introduces a search query, which a static Markdown +folder has no use for); the response for a text document is dot-stuffed +and terminated with a lone "." line per RFC 1436, then the connection +closes. Raw files are streamed as-is with no terminator, matching how +binary Gopher items are conventionally served. +""" +from __future__ import annotations + +import socketserver + +from ..site import NotFound, RawFile, Site + + +MAX_REQUEST_BYTES = 512 # RFC 1436: selectors should be no longer than 255 chars +SOCKET_TIMEOUT = 10.0 +DEFAULT_PORT = 70 + + +def _dot_stuff(body: bytes) -> bytes: + """Double a leading '.' on any line, so it can't be read as the Lastline.""" + lines = body.split(b"\n") + return b"\n".join(b"." + line if line.startswith(b".") else line for line in lines) + + +class GopherHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + self.request.settimeout(SOCKET_TIMEOUT) + try: + raw = self.rfile.readline(MAX_REQUEST_BYTES + 1) + except OSError: + return + selector = raw.split(b"\t", 1)[0].rstrip(b"\r\n") + try: + path = selector.decode("utf-8") + except UnicodeDecodeError: + return + + try: + # No redirect status exists on this protocol, so a canonical + # /foo.md -> /foo bounce or a WML-card-only sub-path is + # resolved down to real content server-side instead. + resource = self.server.site.resolve_flat(path or "/") + except NotFound: + self._send_text(b"Not found.\n") + return + except Exception: + self._send_text(b"Internal error.\n") + return + + if isinstance(resource, RawFile): + self._send_raw(resource.data) + else: + self._send_text(resource.gopher) + + def _send_text(self, body: bytes) -> None: + try: + self.wfile.write(_dot_stuff(body)) + if not body.endswith(b"\n"): + self.wfile.write(b"\n") + self.wfile.write(b".\r\n") + except OSError: + pass + + def _send_raw(self, data: bytes) -> None: + try: + self.wfile.write(data) + except OSError: + pass + + +class GopherServer(socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__(self, address, site: Site) -> None: + self.site = site + super().__init__(address, GopherHandler) diff --git a/src/smolweb/servers/nex.py b/src/smolweb/servers/nex.py new file mode 100644 index 0000000..c937491 --- /dev/null +++ b/src/smolweb/servers/nex.py @@ -0,0 +1,63 @@ +"""Nex protocol listener (:1900, plaintext TCP). + +Nex is deliberately simpler than Gemini or Spartan: the client sends a bare +path and the server sends back text or binary data and closes the +connection -- no headers, no status codes, no MIME type, no redirect +status. See https://nightfall.city/nex/info/specification.txt. +""" +from __future__ import annotations + +import socketserver + +from ..site import NotFound, RawFile, Site + + +MAX_REQUEST_BYTES = 2048 +SOCKET_TIMEOUT = 10.0 +DEFAULT_PORT = 1900 + + +class NexHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + self.request.settimeout(SOCKET_TIMEOUT) + try: + raw = self.rfile.readline(MAX_REQUEST_BYTES + 1) + except OSError: + return + try: + path = raw.rstrip(b"\r\n").decode("utf-8") + except UnicodeDecodeError: + self._send(b"Bad request\n") + return + + try: + # No redirect status exists on this protocol, so a canonical + # /foo.md -> /foo bounce or a WML-card-only sub-path is + # resolved down to real content server-side instead. + resource = self.server.site.resolve_flat(path or "/") + except NotFound: + self._send(b"Not found\n") + return + except Exception: + self._send(b"Internal error\n") + return + + if isinstance(resource, RawFile): + self._send(resource.data) + else: + self._send(resource.nex) + + def _send(self, body: bytes) -> None: + try: + self.wfile.write(body) + except OSError: + pass + + +class NexServer(socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__(self, address, site: Site) -> None: + self.site = site + super().__init__(address, NexHandler) diff --git a/src/smolweb/site.py b/src/smolweb/site.py index 976afc5..56e9b1e 100644 --- a/src/smolweb/site.py +++ b/src/smolweb/site.py @@ -1,7 +1,8 @@ """Path resolution, live-reload caching, and multi-format rendering. -A folder of Markdown is rendered on request into gemtext (md2txt), and WML -plus XHTML-MP (wapdown). Both libraries parse the same leading `---` +A folder of Markdown is rendered on request into gemtext, Nex, and Gopher +text (md2txt), and WML plus XHTML-MP (wapdown). Both libraries parse the +same leading `---` frontmatter block independently and ignore keys they don't recognise, so one file can carry both a wapdown deck-structure key (`title`, `split_level`, `deck_per_card`) and an md2txt typography key @@ -60,6 +61,8 @@ class Document: xhtml: bytes html: bytes # xhtml with the XML prolog stripped, for the text/html fallback wml_index: bytes # the deck served at the document's own URL + nex: bytes # md2txt's nex renderer, for the Nex protocol + gopher: bytes # md2txt's text renderer, for a Gopher type-0 response wml_cards: Dict[str, bytes] = field(default_factory=dict) # deck_per_card only cache_control: Optional[int] = None @@ -145,6 +148,29 @@ class Site: raise Redirect(self._url_for(parent_source)) raise NotFound(url_path) + def resolve_flat(self, url_path: str) -> Union[Document, RawFile]: + """Resolve a request path the way `resolve()` does, but never hands + the caller a Redirect or a WmlCard. + + Nex and Gopher have no redirect status of their own -- there is + nothing to bounce the client back with -- so instead of a 3xx-style + response, the canonical target's content is resolved and served + directly. The loop cap is defensive: both Redirect and WmlCard + currently only ever point at a real, directly-resolvable document, + so one extra hop always suffices in practice. + """ + for _ in range(5): + try: + resource = self.resolve(url_path) + except Redirect as redirect: + url_path = redirect.location + continue + if isinstance(resource, WmlCard): + url_path = resource.parent_url + continue + return resource + raise NotFound(url_path) + # -- Path resolution -------------------------------------------------- def _clean_path(self, url_path: str) -> str: @@ -243,6 +269,19 @@ class Site: ) gemtext = ("\n".join(gemtext_lines) + "\n").encode("utf-8") + # Nex and Gopher clients don't wrap on their own the way Gemini and + # Spartan clients do, so unlike gemtext these are pre-wrapped at the + # classic 80-column convention rather than passed width=0. + nex_lines = convert_markdown( + md_body, width=80, frontmatter=md_fm, renderer_name="nex", base_path=base_path + ) + nex = ("\n".join(nex_lines) + "\n").encode("utf-8") + + gopher_lines = convert_markdown( + md_body, width=80, frontmatter=md_fm, renderer_name="text", base_path=base_path + ) + gopher = ("\n".join(gopher_lines) + "\n").encode("utf-8") + xhtml = wapdown_convert( wap_body, config=config, renderer_name="xhtmlmp", base_path=base_path ).encode("utf-8") @@ -258,6 +297,8 @@ class Site: xhtml=xhtml, html=html, wml_index=wml_index, + nex=nex, + gopher=gopher, wml_cards=wml_cards, cache_control=config.cache_control, ) diff --git a/tests/test_site.py b/tests/test_site.py index ed61ee4..b533b3b 100644 --- a/tests/test_site.py +++ b/tests/test_site.py @@ -114,6 +114,58 @@ class TestWmlCardUrls: doc = site.resolve("/trail") assert b"{.card" not in doc.xhtml + def test_directive_line_does_not_leak_into_nex_or_gopher(self, site: Site): + doc = site.resolve("/trail") + assert b"{.card" not in doc.nex + assert b"{.card" not in doc.gopher + assert b"Cold and clear" in doc.nex + assert b"Cold and clear" in doc.gopher + + +class TestNexAndGopherRendering: + def test_nex_and_gopher_fields_are_rendered(self, site: Site): + doc = site.resolve("/about") + assert isinstance(doc, Document) + assert b"About" in doc.nex + assert b"Body." in doc.nex + # md2txt's `text` renderer (unlike `nex`) still FIGlet-banners + # h1-h3 by default, so only plain paragraph text is checked here. + assert b"Body." in doc.gopher + + def test_nex_heading_has_no_figlet_banner(self, site: Site): + # The nex renderer uses a plain setext-style underline, never a + # FIGlet banner -- unlike md2txt's own `text` renderer, which does + # use one for h1-h3 by default. + doc = site.resolve("/") + assert b"# Home" not in doc.nex + assert b"Home" in doc.nex + assert b"====" in doc.nex + + +class TestResolveFlat: + def test_canonical_md_redirect_is_resolved_not_bounced(self, site: Site): + # Nex and Gopher have no redirect status: a request for the .md + # form must come back with the real document, not a Redirect. + resource = site.resolve_flat("/about.md") + assert isinstance(resource, Document) + assert b"About" in resource.nex + + def test_unmatched_card_is_resolved_to_parent_document(self, site: Site): + resource = site.resolve_flat("/trail/nonexistent-card") + assert isinstance(resource, Document) + assert b"North: open" in resource.gopher + + def test_card_subpath_is_resolved_to_parent_document_not_the_wml_card(self, site: Site): + # /trail/weather is WML-only URL space; resolve_flat must land on + # trail.md's own Document (which has .nex/.gopher), never leak a + # WmlCard (which has neither) out to the Nex/Gopher servers. + resource = site.resolve_flat("/trail/weather") + assert isinstance(resource, Document) + + def test_still_raises_not_found_for_a_missing_path(self, site: Site): + with pytest.raises(NotFound): + site.resolve_flat("/nope") + class TestHtmlFallbackHasNoXmlProlog: def test_html_field_has_no_prolog(self, site: Site):