fix: strip XML prolog from the text/html fallback body

src/smolweb/servers/http.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from urllib.parse import parse_qs, urlsplit
 
-from ..negotiate import FORMAT_WML, content_type_for, negotiate
+from ..negotiate import FORMAT_HTML, FORMAT_WML, content_type_for, negotiate
 from ..site import NotFound, RawFile, Redirect, Site, WmlCard
 
 
@@ -55,7 +55,12 @@ class SmolwebHTTPHandler(BaseHTTPRequestHandler):
             )
             return
 
-        body = resource.wml_index if fmt == FORMAT_WML else resource.xhtml
+        if fmt == FORMAT_WML:
+            body = resource.wml_index
+        elif fmt == FORMAT_HTML:
+            body = resource.html
+        else:
+            body = resource.xhtml
         self._send(
             200,
             content_type_for(fmt),

src/smolweb/site.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import mimetypes
 import posixpath
+import re
 import threading
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -25,6 +26,15 @@ from wapdown.conversion.core import strip_frontmatter as wapdown_strip_frontmatt
 from wapdown.parsers.markdown import CARD_BREAK_PATTERN
 
 
+# XHTML-MP's leading `<?xml ...?>` processing instruction is valid XML but
+# not valid HTML5: a browser parsing it as text/html (the fallback for a
+# client that didn't ask for application/vnd.wap.xhtml+xml) tolerates it as
+# a bogus comment and still renders the page, but logs a console warning.
+# Stripped only for that fallback -- real XHTML-MP and WAP 2.0 clients get
+# the untouched, fully valid XML document.
+_XML_PROLOG_RE = re.compile(rb"^<\?xml[^>]*\?>\r?\n?")
+
+
 class NotFound(Exception):
     """No resource exists at the requested path."""
 
@@ -48,6 +58,7 @@ class Document:
 
     gemtext: bytes
     xhtml: bytes
+    html: bytes  # xhtml with the XML prolog stripped, for the text/html fallback
     wml_index: bytes  # the deck served at the document's own URL
     wml_cards: Dict[str, bytes] = field(default_factory=dict)  # deck_per_card only
     cache_control: Optional[int] = None
@@ -235,6 +246,7 @@ class Site:
         xhtml = wapdown_convert(
             wap_body, config=config, renderer_name="xhtmlmp", base_path=base_path
         ).encode("utf-8")
+        html = _XML_PROLOG_RE.sub(b"", xhtml, count=1)
 
         wml_output = wapdown_convert(
             wap_body, config=config, renderer_name="wml", base_path=base_path
@@ -244,6 +256,7 @@ class Site:
         return Document(
             gemtext=gemtext,
             xhtml=xhtml,
+            html=html,
             wml_index=wml_index,
             wml_cards=wml_cards,
             cache_control=config.cache_control,

tests/test_site.py

@@ -115,6 +115,26 @@ class TestWmlCardUrls:
         assert b"{.card" not in doc.xhtml
 
 
+class TestHtmlFallbackHasNoXmlProlog:
+    def test_html_field_has_no_prolog(self, site: Site):
+        # A browser parsing this as text/html (rather than XML) tolerates
+        # a leading <?xml ...?> as a bogus comment and still renders the
+        # page, but logs a console warning -- so the text/html fallback
+        # must not carry it.
+        doc = site.resolve("/")
+        assert not doc.html.startswith(b"<?xml")
+        assert doc.html.startswith(b"<!DOCTYPE html")
+
+    def test_xhtml_field_keeps_the_prolog(self, site: Site):
+        # Real XHTML-MP/WAP 2.0 clients still need well-formed XML.
+        doc = site.resolve("/")
+        assert doc.xhtml.startswith(b'<?xml version="1.0"')
+
+    def test_html_and_xhtml_are_otherwise_identical(self, site: Site):
+        doc = site.resolve("/")
+        assert doc.html == doc.xhtml[doc.xhtml.index(b"<!DOCTYPE"):]
+
+
 class TestLiveReload:
     def test_edited_file_is_rerendered(self, tmp_path: Path):
         target = tmp_path / "index.md"