sites/plugins/smolnet.py

"""Render a Pelican site as a small-net capsule beside its HTML.

A capsule is a directory of plain Markdown that a protocol daemon serves over
Gemini, Spartan, Nex, Gopher and WAP, converting per request. This writes one
out of the same content the HTML build uses, so the two never drift.

Written against `itsybitsy`, which parses **no frontmatter at all**: a document
is pure Markdown and its metadata lives beside it in a per-directory sidecar.
Pelican's own metadata is a block of bare `Title:`/`Date:` lines at the head of
every file, so served unchanged it becomes the first paragraph of every page.
Stripping that is most of what this plugin does.

Nothing here is specific to one site. Everything that differs between sites is a
setting, and every setting has a default, so a site can switch it on with just
`PLUGINS = ['smolnet']`.

    SMOLNET_OUTPUT        'smolnet'   directory under OUTPUT_PATH
    SMOLNET_SECTION_DIR   'section'   subdirectory holding one index per category
    SMOLNET_LATEST        15          entries on the front page; 0 drops the section
    SMOLNET_META   [date, category]   fields for the line under an article title
    SMOLNET_SECTION_ORDER {}          category name -> sort key, else alphabetical
    SMOLNET_MIN_SECTIONS  2           below this, the section index is dropped
    SMOLNET_LABELS        {}          override any heading, see LABELS
    SMOLNET_ASSETS        None        directories to copy; defaults to STATIC_PATHS
    SMOLNET_SIDECAR       '.itsybitsy.toml'
    SMOLNET_ROT13_ATTR    None        attribute holding a rot13 address to reveal
    SMOLNET_TAGLINE       None        one line at the foot of the index
    SMOLNET_COMMENTS      None        thread URL per article, `{slug}` substituted
    SMOLNET_COMMENTS_FORMATS          formats that get it, default gemtext and text
    SMOLNET_FOOTNOTES     False   rewrite [^key] footnotes into a numbered
                                  Notes section; off by default since not
                                  every site using this plugin has been
                                  checked for footnotes
    SMOLNET_CALLOUTS      {}          callout kind -> label, for `> [!kind]`
                                  blocks; see rewrite_callouts

Every run of links here is a **Markdown list**, which is what each format needs:
Mews Profile reads a dated link as a feed entry only when it starts a list item
(mews.page/spec 6.1), and the `text` renderer used for Nex and Gopher keeps one
item per line instead of reflowing a paragraph into a run-on line. gemtext has
one kind of link, a line beginning `=>`, and itsybitsy prints only those lines
for a list item that holds nothing but links, so no entry is listed twice.

That last part is a floor on the daemon version: an itsybitsy before the fix in
its gemtext renderer prints a bullet *and* a link line for every entry here.

Gemini clients number link lines themselves, which means a menu of areas is
dialled by number without this plugin writing a single bracket.
"""

import os
import re
import shutil

from pelican import signals
from pelican.generators import Generator

LABELS = {
    "sections": "Sections",
    "latest": "Latest",
    "archive": "All entries",
    "pages": "Pages",
    "back": "Back to the index",
    "comments": "Comments",
    "notes": "Notes",
}

# Pelican's metadata block: bare `Key: value` lines from the first line on,
# with indented continuations, ending at the first blank line. This is
# Python-Markdown's Meta extension, not YAML, and it has no delimiters.
META_LINE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*:")
META_CONT = re.compile(r"^\s+\S")

# Pelican's internal link syntax. Left alone these reach the capsule verbatim.
INTERNAL = re.compile(r"\{(filename|static|attach)\}([^)\s]+)")

# Links written against the HTML build, e.g. ](/go-west.html). Pelican's
# {filename} form is the portable one, but plenty of content predates knowing
# that and hardcodes the output URL instead. A capsule has no .html, and the
# daemon resolves a name carrying an extension literally, so these 404 unless
# they are rewritten.
HARDCODED = re.compile(r"\]\(\s*(/[^)\s]*?\.html)([^)]*)\)")

SUMMARY_MARKER = "<!-- PELICAN_END_SUMMARY -->"

# Markdown image syntax, for dropping the ones whose host is gone. Matched on
# the source rather than the rendered HTML because this plugin reads the
# Markdown directly, so lostimages.py, which works on Pelican's output, never
# sees the capsule.
MD_IMAGE = re.compile(r'!\[[^\]]*\]\(\s*(https?://[^\s)]+)[^)]*\)')

# A footnote definition, Markdown-Extra style: "[^key]: text". The text runs
# to the next blank line or the next definition, not to the next newline: a
# definition's own citation link is often hand-wrapped onto an unindented
# second physical line, which CommonMark treats as a lazy continuation of the
# same paragraph (rendered with no line break at all). An earlier version of
# this regex stopped at the first `$` and left that second line behind as an
# orphaned, bracket-broken paragraph in the body -- caught by building a real
# site and reading the output, not by the narrower check (indented
# continuations only) done beforehand.
FOOTNOTE_DEF = re.compile(
    r"^\[\^([^\]]+)\]:[ \t]*(.*(?:\n(?![ \t]*\n|\[\^)[^\n]*)*)",
    re.MULTILINE,
)

# An inline reference, "[^key]". No exclusion for a trailing ":" here: by the
# time this runs, FOOTNOTE_DEF.sub has already stripped every genuine
# definition line out of the text, so a "[^key]" that is still present can
# only be an inline reference -- including the fairly common case of one
# followed by an ordinary colon in the author's own prose ("Gnosis[^gnosis]:
# collective critique and experimentation..."), which an earlier version of
# this regex wrongly left unrewritten by trying to guard against a confusion
# that FOOTNOTE_DEF already resolves upstream.
FOOTNOTE_REF = re.compile(r"\[\^([^\]]+)\]")

# A CommonMark reference-link definition using a bare integer as its id, e.g.
# "[18]: https://...". At least one article here (sixnames.md) cites sources
# this way. A renderer resolves a later bare "[18]" in running text as a
# shortcut reference link to that target -- so a footnote number assigned
# without checking this would, if it ever landed on the same integer, turn an
# inserted footnote marker into a silent, wrong hyperlink instead of the
# citation it was meant to carry.
LINKREF_DEF = re.compile(r"^\[(\d+)\]:", re.MULTILINE)

# The opening line of a GitHub/Obsidian-style callout, "> [!kind] title".
# The trailing newline is part of the match, so a marker with no label can be
# dropped whole instead of leaving the quote to open on a blank line.
CALLOUT = re.compile(r"^([ ]{0,3}>[ ]?)\[!(\w+)\][ \t]*(.*)\n?", re.MULTILINE)


def rot13(text):
    """The obfuscation used on addresses in the HTML templates."""
    out = []
    for ch in text:
        if "a" <= ch <= "z":
            out.append(chr((ord(ch) - 97 + 13) % 26 + 97))
        elif "A" <= ch <= "Z":
            out.append(chr((ord(ch) - 65 + 13) % 26 + 65))
        else:
            out.append(ch)
    return "".join(out)


def strip_metadata(text):
    """Drop the leading metadata block, returning the body alone.

    Stops at the first line that is neither a `Key:` line nor an indented
    continuation of one, so a document that happens to open on prose is
    returned untouched rather than having its first paragraph eaten.
    """
    lines = text.splitlines()
    i = 0
    seen = False
    while i < len(lines):
        line = lines[i]
        if not line.strip():
            if seen:
                i += 1
            break
        if META_LINE.match(line):
            seen = True
        elif seen and META_CONT.match(line):
            pass
        else:
            break
        i += 1
    if not seen:
        return text
    return "\n".join(lines[i:]).lstrip("\n")


def rewrite_internal(text, by_source, warn):
    """Resolve `{filename}other.md` against the documents actually built.

    The target is a path relative to the referring file, which for a flat
    content directory means a bare basename. Anything that does not resolve is
    reported rather than silently emitted, because a literal `{filename}` in a
    capsule is both broken and ugly.
    """

    def sub(match):
        kind, target = match.group(1), match.group(2)
        key = os.path.basename(target)
        if kind == "filename" and key in by_source:
            return "/" + by_source[key]
        warn("smolnet: unresolved {%s}%s", kind, target)
        return "/" + key.rsplit(".", 1)[0]

    return INTERNAL.sub(sub, text)


def rewrite_html_links(text, urls, warn):
    """Point links written against the HTML build at the capsule instead.

    Matched against each document's own rendered URL (Pelican's ARTICLE_URL /
    PAGE_URL settings resolved per document), not by stripping ".html" off the
    link's basename and hoping that equals the slug. That shortcut only holds
    for a site whose URL *is* a bare "{slug}.html" -- true for randogoth.com,
    where this was first written, false for a site like sublunar.space whose
    ARTICLE_URL prefixes the date ("2018-06-chaxiraxi.html" for slug
    "chaxiraxi"). There the stripped string never matched any slug, and every
    article-to-article link passed through unrewritten, pointing the capsule
    at a path itsybitsy has no document for.

    `/pages/glossary.html` and `/go-west.html` both become `/<slug>`, since the
    capsule is flat. A target whose rendered URL matches no document built is
    reported and left alone rather than silently rewritten into a second broken
    link.
    """

    def sub(match):
        target, rest = match.group(1), match.group(2)
        key = target.lstrip("/")
        if key in urls:
            return "](/%s%s)" % (urls[key], rest)
        warn("smolnet: link to %s matches no document", target)
        return match.group(0)

    return HARDCODED.sub(sub, text)


def reveal_rot13(text, attr):
    """Replace a block of HTML carrying a rot13 address with the address itself.

    Address obfuscation defends against web crawlers and depends on JavaScript,
    so over Gemini it is both pointless and impossible. The renderer drops raw
    HTML blocks outright, which would take the address with them: on a page
    whose purpose is to carry it, that is a silent loss rather than a tidy-up.

    The whole block is replaced rather than the tag that carries the attribute.
    Matching tags means guessing where the element ends, and the first attempt
    at this stopped at the first closing tag and left the rest of the markup
    stranded in the output. Markdown already separates block-level HTML by blank
    lines, so the block is a boundary the format gives us for free.
    """
    if not attr:
        return text
    carrier = re.compile(r"\b" + re.escape(attr) + r'="([^"]+)"', re.IGNORECASE)
    blocks = re.split(r"(\n[ \t]*\n)", text)
    out = []
    for block in blocks:
        found = carrier.search(block)
        out.append(rot13(found.group(1)) if found else block)
    return "".join(out)


def drop_dead_images(text, dead):
    """Remove images hotlinked from hosts that no longer exist.

    It matters more here than on the web. A browser shows a broken image as a
    silent gap, but gemtext has no inline images: every one becomes a visible
    `=>` link line, so the page would advertise a dead host as though it were
    somewhere worth going.

    Shares LOSTIMAGES_DEAD_HOSTS with lostimages.py so there is one list, and
    matches on the host rather than fetching, because a build must not depend on
    the network.
    """
    if not dead:
        return text

    def sub(match):
        host = match.group(1).split("//", 1)[1].split("/")[0].lower().split(":")[0]
        if any(host == d or host.endswith("." + d) for d in dead):
            return ""
        return match.group(0)

    text = MD_IMAGE.sub(sub, text)
    # a line that held nothing but the dropped image would leave a blank
    return re.sub(r"\n[ \t]*\n[ \t]*\n+", "\n\n", text)


def rewrite_footnotes(text, labels, warn):
    """Turn Markdown-Extra footnotes into something a footnote-less renderer
    can show: inline refs become "[1]", "[2]", ... in order of first
    appearance, and the definitions move to a "## Notes" section at the end.

    itsybitsy's Markdown parser has no footnote extension enabled, so left
    alone, both the inline "[^key]" and the "[^key]: text" definition line
    print as literal bracket text in the capsule. Off by default (see
    SMOLNET_FOOTNOTES) since not every site using this plugin has had its
    content checked for footnotes.
    """
    defs = {}

    def strip_def(match):
        # Collapse any lazy-continuation line breaks back into the single
        # flowing line Markdown would have rendered them as, the way a line
        # break folds to nothing right after an opening "[": the citation
        # link itself can start on the wrapped line with nothing between the
        # bracket and the break.
        joined = re.sub(r"\s*\n\s*", " ", match.group(2)).strip()
        joined = re.sub(r"\[\s+", "[", joined)
        defs[match.group(1)] = joined
        return ""

    body = FOOTNOTE_DEF.sub(strip_def, text)
    # The def lines are gone; squeeze the blank run they leave, same as
    # drop_dead_images does for a dropped image's own empty line.
    body = re.sub(r"\n[ \t]*\n[ \t]*\n+", "\n\n", body)

    # Numbers assigned skip any integer a reference-link definition already
    # owns elsewhere in this same document, so an inserted marker can never
    # collide with one.
    reserved = {int(n) for n in LINKREF_DEF.findall(body)}
    numbers = {}
    next_number = [1]

    def number_ref(match):
        key = match.group(1)
        if key not in defs:
            warn("smolnet: footnote ref [^%s] has no definition", key)
            return match.group(0)
        if key not in numbers:
            while next_number[0] in reserved:
                next_number[0] += 1
            numbers[key] = next_number[0]
            next_number[0] += 1
        return "[%d]" % numbers[key]

    body = FOOTNOTE_REF.sub(number_ref, body)

    if not numbers:
        return body

    notes = ["## %s" % labels["notes"], ""]
    for key, n in sorted(numbers.items(), key=lambda kv: kv[1]):
        notes.append("%d. %s" % (n, defs[key]))
    return body.rstrip() + "\n\n" + "\n".join(notes) + "\n"


def rewrite_callouts(text, kinds):
    """Turn a `> [!kind]` marker into a bold label line, leaving the rest of
    the passage an ordinary blockquote.

    The HTML build renders these as boxes (an <aside>, via a Markdown
    extension in the site itself). gemtext has nothing to draw a box with, and
    a quote is the one block it does set apart, so the passage stays one; only
    the marker would otherwise print literally as "[!aside]". Kinds not in
    `kinds` are left alone, matching the HTML side, which also only boxes the
    kinds it was told about.
    """
    if not kinds:
        return text
    kinds = {k.lower(): v for k, v in kinds.items()}

    def sub(match):
        kind = match.group(2).lower()
        if kind not in kinds:
            return match.group(0)
        title = match.group(3).strip() or kinds[kind]
        if not title:
            # No label: the marker line goes, the quote stays.
            return ""
        # The label as a paragraph of its own inside the quote, so it can't run
        # into the passage's first sentence.
        return "%s**%s**\n>\n" % (match.group(1), title)

    return CALLOUT.sub(sub, text)


def toml_escape(value):
    return str(value).replace("\\", "\\\\").replace('"', '\\"')


def sidecar(titles):
    """A `.itsybitsy.toml` naming each file beside it.

    The daemon falls back to the first heading and then to the filename, so
    this is belt and braces, but it is the only place a title can live that
    survives a document having no heading of its own.
    """
    lines = []
    for name in sorted(titles):
        lines.append('[page."%s"]' % toml_escape(name))
        lines.append('title = "%s"' % toml_escape(titles[name]))
        lines.append("")
    return "\n".join(lines)


def link(label, href):
    return "[%s](%s)" % (label.replace("]", ""), href)


def bullets(items):
    """A Markdown list, one item per line. See the module header."""
    return "\n".join("- %s" % item for item in items)


def gate(block, formats):
    """Wrap a block in an itsybitsy format gate.

    A gate names output format ids, not protocols: `gemtext` is the one Gemini,
    Spartan, Nex and Gopher are all served from, so there is no gating on a
    protocol name. Blank lines because a gate encloses *blocks*, and a
    directive run together with the line under it is one block.

    Note which way this can cut. Pelican does not implement the directive and
    renders it as an inert HTML comment, so a gate can only ever subtract from
    the capsule, never from the site. That is the right direction for anything
    this plugin writes, because the capsule is all it writes.
    """
    return "<!-- only %s -->\n\n%s\n\n<!-- end -->" % (" ".join(formats), block)


class CapsuleGenerator(Generator):
    """Writes the capsule once the content generators have read everything."""

    def generate_context(self):
        pass

    def generate_output(self, writer=None):
        s = self.settings
        out = os.path.join(self.output_path, s.get("SMOLNET_OUTPUT", "smolnet"))
        section_dir = s.get("SMOLNET_SECTION_DIR", "section")
        labels = dict(LABELS, **s.get("SMOLNET_LABELS", {}))
        order = s.get("SMOLNET_SECTION_ORDER", {}) or {}
        latest_n = s.get("SMOLNET_LATEST", 15)
        rot_attr = s.get("SMOLNET_ROT13_ATTR")
        dead_hosts = {h.lower().lstrip(".") for h in
                      s.get("LOSTIMAGES_DEAD_HOSTS", [])}
        sidecar_name = s.get("SMOLNET_SIDECAR", ".itsybitsy.toml")
        footnotes = s.get("SMOLNET_FOOTNOTES", False)
        callouts = s.get("SMOLNET_CALLOUTS", {})
        # kapara serves the same remark42 thread the HTML widget shows, over
        # Gemini, Spartan and Nex. Articles only: the widget is on articles
        # only, and a page with a thread nobody can reach from the site is a
        # second conversation, which is the one thing the gateway exists to
        # avoid. Gated because this tree can also be served as WML or
        # XHTML-MP, and a handset cannot open a gemini:// link.
        comments = s.get("SMOLNET_COMMENTS")
        comments_formats = s.get("SMOLNET_COMMENTS_FORMATS", ("gemtext", "text"))

        articles = [a for a in self.context.get("articles", [])]
        pages = [p for p in self.context.get("pages", [])]
        if not articles and not pages:
            return

        # Map source basename -> slug, so {filename} can be resolved. Built from
        # everything published, pages included, since an article may link one.
        by_source = {}
        urls = {}
        for doc in articles + pages:
            by_source[os.path.basename(doc.source_path)] = doc.slug
            urls[doc.url] = doc.slug

        os.makedirs(out, exist_ok=True)
        root_titles = {}

        def emit(doc, trail, thread=False):
            body = strip_metadata(self._read(doc))
            body = body.replace(SUMMARY_MARKER, "")
            body = rewrite_internal(body, by_source, self._warn)
            body = rewrite_html_links(body, urls, self._warn)
            body = reveal_rot13(body, rot_attr)
            body = drop_dead_images(body, dead_hosts)
            body = rewrite_callouts(body, callouts)
            if footnotes:
                body = rewrite_footnotes(body, labels, self._warn)
            # Blank lines, not newlines: a heading and the line under it are
            # separate blocks only if a blank line divides them, otherwise the
            # date runs into the first sentence as one paragraph.
            head = "# %s\n\n" % doc.title
            if trail:
                head += "%s\n\n" % trail
            text = head + body.strip() + "\n"
            if thread and comments:
                text += "\n%s\n" % gate(
                    link(labels["comments"], comments.format(slug=doc.slug)),
                    comments_formats,
                )
            name = "%s.md" % doc.slug
            with open(os.path.join(out, name), "w", encoding="utf-8") as fh:
                fh.write(text)
            root_titles[name] = doc.title

        # Which metadata goes under the title. Default is date and category,
        # but a site that files by tag has no useful category and may well have
        # a field that means more: zonetoast carries `era`, which is the point
        # of the site in a way that Pelican's default `misc` is not.
        meta_fields = s.get("SMOLNET_META", ["date", "category"])
        for article in articles:
            bits = []
            for field in meta_fields:
                value = getattr(article, field, None)
                if not value:
                    continue
                if field == "date":
                    bits.append(value.strftime("%d %b %Y"))
                else:
                    bits.append(str(value))
            emit(article, " · ".join(bits), thread=True)
        for page in pages:
            emit(page, "")

        # One index per category.
        sections = sorted(
            self.context.get("categories", []),
            key=lambda pair: (order.get(str(pair[0]), 99), str(pair[0])),
        )
        # A site that files by tag rather than category still has one category,
        # Pelican's DEFAULT_CATEGORY, holding everything. Offering "misc, 14" as
        # the only way in is worse than offering nothing, so one bucket is not a
        # taxonomy and the whole section index goes.
        if len(sections) < s.get("SMOLNET_MIN_SECTIONS", 2):
            sections = []
        if sections:
            os.makedirs(os.path.join(out, section_dir), exist_ok=True)
            section_titles = {}
            for category, items in sections:
                lines = ["# %s" % category, ""]
                lines.append(self._links(sorted(items, key=lambda a: a.date, reverse=True)))
                lines += ["", link(labels["back"], "/")]
                name = "%s.md" % category.slug
                with open(os.path.join(out, section_dir, name), "w", encoding="utf-8") as fh:
                    fh.write("\n".join(lines).rstrip() + "\n")
                section_titles[name] = str(category)
            self._write_sidecar(os.path.join(out, section_dir, sidecar_name), section_titles)

        by_date = sorted(articles, key=lambda a: a.date, reverse=True)

        # The full listing, so the index can stay short without hiding anything.
        archive = ["# %s" % labels["archive"], "", self._links(by_date), "",
                   link(labels["back"], "/")]
        with open(os.path.join(out, "archive.md"), "w", encoding="utf-8") as fh:
            fh.write("\n".join(archive).rstrip() + "\n")
        root_titles["archive.md"] = labels["archive"]

        index = ["# %s" % s.get("SITENAME", "index"), ""]
        if sections:
            index += ["## %s" % labels["sections"], ""]
            index.append(bullets(
                link("%s · %d" % (category, len(items)),
                     "/%s/%s" % (section_dir, category.slug))
                for category, items in sections
            ))
            index.append("")
        if latest_n and by_date:
            index += ["## %s" % labels["latest"], "",
                      self._links(by_date[:latest_n]), "",
                      link(labels["archive"], "/archive"), ""]
        if pages:
            index += ["## %s" % labels["pages"], "",
                      bullets(link(p.title, "/%s" % p.slug) for p in pages), ""]
        tagline = s.get("SMOLNET_TAGLINE")
        if tagline:
            index += ["---", "", tagline, ""]
        with open(os.path.join(out, "index.md"), "w", encoding="utf-8") as fh:
            fh.write("\n".join(index).rstrip() + "\n")
        root_titles["index.md"] = s.get("SITENAME", "index")

        self._write_sidecar(os.path.join(out, sidecar_name), root_titles)
        self._copy_assets(out)

    def _links(self, docs):
        """A list of dated links, one entry per item.

        Targets are root-relative. A section index sits one directory down, so
        relative targets would need `../` and would break again at any further
        depth; `/slug` is correct from anywhere and the daemon resolves it
        against the site root.

        The date is ISO 8601 and it leads the label, which is protocol surface
        rather than a style choice. Both subscription conventions this capsule
        is read through take a link as a feed entry when its label begins with
        a `YYYY-MM-DD` date and read the rest as the title: Gemini's over the
        `=>` lines, Mews Profile's over the list items (6.1). That one format is
        what makes each index a feed: the front page, the archive and all six
        areas, subscribable in a client and submittable to an aggregator. A
        board's `09 Oct 26` parses as nothing and registers zero entries.

        Nothing separates the date from the title. Gemini's examples put a dash
        there and ask clients to strip whatever they find, but Mews reads the
        label from the date to the end of the line, so a separator arrives as
        part of the title -- as does a `·` in a Gemini client that skips the
        sanitising step.
        """
        rows = []
        for doc in docs:
            label = doc.title
            if getattr(doc, "date", None):
                label = "%s %s" % (doc.date.strftime("%Y-%m-%d"), doc.title)
            rows.append(link(label, "/%s" % doc.slug))
        return bullets(rows)

    def _read(self, doc):
        with open(doc.source_path, encoding="utf-8") as fh:
            return fh.read()

    def _write_sidecar(self, path, titles):
        with open(path, "w", encoding="utf-8") as fh:
            fh.write(sidecar(titles))

    def _copy_assets(self, out):
        """Copy the static directories in, so the capsule stands on its own.

        A capsule whose images are fetched over HTTPS from the web site is not
        really a capsule, and link targets are resolved against the capsule
        root, so pointing outside it would not work anyway.
        """
        s = self.settings
        names = s.get("SMOLNET_ASSETS")
        if names is None:
            names = [n for n in s.get("STATIC_PATHS", []) if not n.startswith(".")]
        for name in names:
            src = os.path.join(self.path, name)
            if not os.path.isdir(src):
                continue
            shutil.copytree(src, os.path.join(out, name), dirs_exist_ok=True)

    def _warn(self, fmt, *args):
        try:
            from pelican.log import logger

            logger.warning(fmt, *args)
        except Exception:
            pass


def get_generators(pelican_object):
    return CapsuleGenerator


def register():
    signals.get_generators.connect(get_generators)