From 4f1a67bed347abb2c3ef170a4e364579aac65288 Mon Sep 17 00:00:00 2001 From: randogoth Date: Wed, 14 Oct 2026 11:54:26 +0300 Subject: [PATCH] Feed the F-Droid repo from Forgejo releases automatically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scaffold here was a manual pipeline: copy an APK into repo/, run `fdroid update`, rsync the result somewhere. Every app already builds and signs its own APK in CI and publishes it as a Forgejo release, so all of that copying was re-doing by hand what the forge already had. Both halves of a listing now come from the same release: the APK from the release asset, and the title, summary, description, changelog, icon and screenshots from the fastlane/ tree at the same tag. A listing can no longer describe a build other than the one it ships, because there is no second copy of the text to drift. fdroid-sync.py is the whole reconcile. It is a pure function of what the forge currently holds — work out what should be published, make it so, re-sign the index only if something moved — which is what makes it safe to drive from a webhook that has no delivery guarantee and no retry. A missed delivery costs latency, not correctness. Three things were only found by running the real thing with an empty environment, and all three fail late and confusingly: - fdroidserver bundles apksigner but no JDK, and shells out to jar, keytool and jarsigner by bare name. The PATH is now set outright rather than inherited, because the bundled apksigner is a Gradle-style launcher that also needs xargs and sed — and without xargs fdroidserver reads every APK as having an invalid signature and quarantines the lot into archive/. - index-v2 only carries a changelog for a version matching the last Builds entry, and a repo of prebuilt APKs has no build recipes. Without a declared entry every release shipped an empty "What's New". - fdroidserver does not fall back to the APK's launcher icon, so an app whose fastlane tree has no images/icon.png was listed with no icon at all, which is two of these four. The sync now recovers one from the APK. module.nix keeps the keystore password out of the Nix store via fdroidserver's {env: ...} indirection, and serves the repo rooted at repo/ rather than at the working directory, which would also have handed out config.yml and metadata/. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 10 +- apps.nix | 56 +++++ bin/register-webhooks.sh | 74 ++++++ fdroid-sync.py | 505 +++++++++++++++++++++++++++++++++++++++ flake.lock | 8 +- flake.nix | 33 ++- module.nix | 309 ++++++++++++++++++++++++ nixos-module.nix | 40 ---- sync.sh | 16 -- 9 files changed, 982 insertions(+), 69 deletions(-) create mode 100644 apps.nix create mode 100755 bin/register-webhooks.sh create mode 100755 fdroid-sync.py create mode 100644 module.nix delete mode 100644 nixos-module.nix delete mode 100755 sync.sh diff --git a/.gitignore b/.gitignore index 2c837ac..c26a20c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,15 @@ +# The repo's index signing key and the config that holds its password. The key +# is the repo's identity to every subscriber: back it up off this machine. +# Server-side it lives in bucur's sops store, not here. keystore.p12 config.yml + +# Local scratch from running fdroid by hand; the real tree lives on the server. repo/ -unsigned/ +metadata/ archive/ +unsigned/ tmp/ stats/ +result +__pycache__/ diff --git a/apps.nix b/apps.nix new file mode 100644 index 0000000..ab24d36 --- /dev/null +++ b/apps.nix @@ -0,0 +1,56 @@ +# The apps this repo serves. One entry per app; adding an app is adding an +# entry here and pushing. +# +# Everything in this file is the part of a listing that does not change when a +# release is cut: the package identity, where its releases live, its licence +# and where it belongs in F-Droid's category list. Everything that *does* +# change per release — name, summary, description, changelog, icon, +# screenshots — is deliberately absent, because it is read from the app repo's +# own fastlane/ tree at the released tag. Writing a description here would +# create a second copy of it that silently drifts from the one the app ships. +# +# Fields: +# id short slug; names log lines and the --app filter, nothing else +# packageId the Android applicationId. This is the app's identity to +# F-Droid and to every phone that has it installed, so it has to +# match `applicationId` in android/app/build.gradle{,.kts} +# exactly — a typo here publishes a second, unrelated app rather +# than an update to the existing one. +# repo owner/name on the forge +# license SPDX identifier, shown in the client +# categories F-Droid's fixed category vocabulary, not free text +# fastlane where the fastlane tree sits inside the repo (default +# "fastlane"; the-mews keeps its Flutter app under app/) +# webSite optional; omitted means the client shows only the source link +[ + { + id = "muzzlevelocity"; + packageId = "vision.flux.muzzlevelocity"; + repo = "randogoth/muzzle-velocity"; + license = "Apache-2.0"; + categories = [ "Writing" ]; + } + { + id = "plasmoid"; + packageId = "com.randogoth.plasmoid"; + repo = "randogoth/plasmoid"; + license = "Apache-2.0"; + categories = [ "System" ]; + } + { + id = "themews"; + packageId = "page.mews.mews"; + repo = "randogoth/the-mews"; + license = "Apache-2.0"; + categories = [ "Internet" ]; + fastlane = "app/fastlane"; + webSite = "https://mews.page"; + } + { + id = "kirakira"; + packageId = "com.app.smol_mail"; + repo = "randogoth/kirakira"; + license = "MIT"; + categories = [ "Internet" ]; + } +] diff --git a/bin/register-webhooks.sh b/bin/register-webhooks.sh new file mode 100755 index 0000000..2d7e84d --- /dev/null +++ b/bin/register-webhooks.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Point every app repo's release webhook at the F-Droid rebuild endpoint. +# +# Forgejo webhooks are instance state, not repo content, so they cannot be +# declared in a flake and have to be registered once per repo. This does that +# from the single source of truth — apps.nix — so adding an app means editing +# that file and re-running this, not remembering four web forms. +# +# Idempotent: a hook already pointing at the same URL is updated in place +# rather than duplicated, so re-running after a secret rotation is the +# supported way to roll the secret. +# +# The secret must be the same value the serving host reads from +# webhookSecretFile; a mismatch makes every delivery fail the HMAC check and +# the repo silently stops updating. Pipe it straight out of sops so it reaches +# neither the terminal nor the shell history: +# +# +# FORGEJO_TOKEN= \ +# FDROID_WEBHOOK_SECRET=$(= \ +# nix run nixpkgs#sops -- -d --extract '["fdroid_webhook_secret"]' \ +# ) \ +# ../fdroid-repo/bin/register-webhooks.sh +# +# Usage: register-webhooks.sh [base-url] +set -euo pipefail + +cd "$(dirname "$0")/.." + +: "${FORGEJO_TOKEN:?set FORGEJO_TOKEN to a Forgejo access token with repo write access}" +# Env, not an argument: an argv is visible to every other process on the box +# for as long as the script runs. +SECRET="${FDROID_WEBHOOK_SECRET:?set FDROID_WEBHOOK_SECRET to the shared webhook secret}" +BASE="${1:-https://apps.randogoth.com}" +FORGE="${FORGE:-https://code.randogoth.com}" +URL="$BASE/hooks/rebuild-fdroid" + +api() { + local method=$1 path=$2 + shift 2 + curl -sS -X "$method" \ + -H "Authorization: token $FORGEJO_TOKEN" \ + -H "Content-Type: application/json" \ + "$FORGE/api/v1$path" "$@" +} + +for repo in $(nix eval --json --file ./apps.nix | jq -r '.[].repo'); do + # Forgejo returns the full hook list; match on the target URL because the + # hook id is assigned by the server and is not knowable in advance. + existing=$(api GET "/repos/$repo/hooks" | jq -r --arg u "$URL" \ + '.[] | select(.config.url == $u) | .id' | head -1) + + payload=$(jq -n --arg url "$URL" --arg secret "$SECRET" '{ + type: "gitea", + active: true, + # "release" alone: a tag push fires before CI has built anything, so a + # push-triggered rebuild would always find the old release and do nothing. + events: ["release"], + config: { url: $url, content_type: "json", secret: $secret } + }') + + if [ -n "$existing" ]; then + api PATCH "/repos/$repo/hooks/$existing" -d "$payload" >/dev/null + echo "updated $repo (hook $existing) -> $URL" + else + api POST "/repos/$repo/hooks" -d "$payload" >/dev/null + echo "created $repo -> $URL" + fi +done + +echo +echo "Note: the release webhook fires when the release object is created," +echo "which in these workflows is one API call before the APK is attached." +echo "fdroid-sync waits for the upload to land, so that race is handled." diff --git a/fdroid-sync.py b/fdroid-sync.py new file mode 100755 index 0000000..fab33d6 --- /dev/null +++ b/fdroid-sync.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 +"""Reconcile a self-hosted F-Droid repo with the latest Forgejo releases. + +Nothing is built here. Each app's APK is the one its own CI built and signed, +and its listing text comes from the fastlane/ tree in the same repo at the same +tag, so the store entry and the binary can never describe different releases. + +A run is a pure function of what the forge currently holds: it works out what +should be in the repo, makes it so, and rebuilds the index only if something +actually moved. That is what makes it safe to drive from a webhook, which has +no delivery guarantee and no retry — a missed delivery costs latency, not +correctness, because the next run (or the timer) converges anyway. + +Usage: fdroid-sync [app-id ...] +""" + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +# Both halves of a release filename come from the git tag, which the app +# workflows write as v+ (Flutter's own pubspec +# version syntax). The code is what F-Droid orders versions by and what names +# the changelog file, so a tag without one is an error rather than a guess. +TAG_SEP = "+" + +# How long to keep waiting for an APK that is still uploading. The release +# webhook fires when the release object is created, which in these workflows is +# one API call *before* the APK is attached to it, so a webhook-triggered run +# routinely arrives at a release with no asset yet. That is a race to wait out, +# not an error: a 50-80 MB upload off a CI runner takes a while. +ASSET_WAIT_SECONDS = 600 +ASSET_POLL_SECONDS = 20 + +# A run can be triggered while an earlier one is still going, and systemd +# merges the second `start` into the first rather than queueing it, so the +# release that triggered it would be missed until the daily timer. Re-checking +# the forge after a pass that changed something closes that window; a pass that +# changes nothing ends the loop, so the steady state is still a single pass. +MAX_PASSES = 3 + +DEFAULT_LOCALE = "en-US" + +# Per-read, not per-transfer, so this is not a cap on how long an 80 MB APK may +# take — only on how long the forge may go silent mid-stream. +HTTP_TIMEOUT = 60 + + +def log(*args): + print(*args, flush=True) + + +class AppError(Exception): + """A failure confined to one app, so the other three still get a chance.""" + + +# -------------------------------------------------------------------------- +# forge API + + +def api_get(url): + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp: + return json.load(resp) + + +def download(url, dest: Path, expect_size=None): + """Fetch to a sibling temp file, then rename. + + Never streams straight onto the destination: a half-written file under + repo/ is one `fdroid update` away from being published as a real release, + and a phone that downloads it gets a signature failure it cannot explain. + """ + tmp = dest.with_name(dest.name + ".part") + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp, open(tmp, "wb") as out: + shutil.copyfileobj(resp, out, 1 << 20) + size = tmp.stat().st_size + if expect_size is not None and size != expect_size: + tmp.unlink() + raise AppError(f"{dest.name}: got {size} bytes, forge advertised {expect_size}") + tmp.rename(dest) + return size + + +def latest_release(forge, repo): + """The newest published, non-draft, non-prerelease release.""" + url = f"{forge}/api/v1/repos/{repo}/releases/latest" + try: + return api_get(url) + except urllib.error.HTTPError as e: + if e.code == 404: + raise AppError("no published release yet") from None + raise + + +def version_code(tag): + name, sep, code = tag.lstrip("v").partition(TAG_SEP) + if not sep or not code.isdigit(): + raise AppError( + f"tag {tag!r} is not v{TAG_SEP}; " + "cannot tell which version this is" + ) + return name, int(code) + + +def apk_asset(release): + apks = [a for a in release.get("assets", []) if a["name"].endswith(".apk")] + if len(apks) > 1: + raise AppError( + f"release {release['tag_name']} has {len(apks)} APKs " + f"({', '.join(a['name'] for a in apks)}); expected exactly one" + ) + return apks[0] if apks else None + + +def await_apk_asset(forge, repo, release): + """Poll until the APK shows up, or give up loudly.""" + asset = apk_asset(release) + if asset: + return asset + deadline = time.monotonic() + ASSET_WAIT_SECONDS + log(f" {release['tag_name']}: no APK attached yet, waiting for the upload") + while time.monotonic() < deadline: + time.sleep(ASSET_POLL_SECONDS) + asset = apk_asset(latest_release(forge, repo)) + if asset: + return asset + raise AppError( + f"release {release['tag_name']} still has no APK after " + f"{ASSET_WAIT_SECONDS}s; did the build fail after creating the release?" + ) + + +# -------------------------------------------------------------------------- +# listing metadata, straight out of the app's own fastlane tree + + +def tree_manifest(root: Path): + """(relpath, sha256) for every file under root, sorted. + + Used to decide whether a freshly fetched listing differs from the one + already in place. The alternative — always replacing it — would mark every + run as changed and rebuild the index daily for nothing, which rewrites + every index timestamp and makes every client re-download it. + """ + out = [] + for path in sorted(root.rglob("*")): + if path.is_file(): + out.append((str(path.relative_to(root)), hashlib.sha256(path.read_bytes()).hexdigest())) + return out + + +def extract_fastlane(forge, repo, tag, fastlane_dir, dest: Path): + """Unpack /metadata/android/ from the source tag into dest. + + Copied verbatim, subdirectories and all: fdroidserver reads the fastlane + layout natively — title.txt, short_description.txt, full_description.txt, + changelogs/.txt and images/{icon,featureGraphic}.png plus + images/*Screenshots/ — so there is nothing to translate and no mapping + table to keep in step with upstream. + """ + url = f"{forge}/{repo}/archive/{urllib.parse.quote(tag, safe='')}.tar.gz" + prefix = f"{fastlane_dir.strip('/')}/metadata/android/" + with tempfile.TemporaryDirectory() as td: + tarball = Path(td) / "src.tar.gz" + download(url, tarball) + found = 0 + with tarfile.open(tarball) as tf: + for member in tf: + if not member.isfile(): + continue + # Drop the archive's top-level directory, which Forgejo names + # after the repo rather than after anything we control. + _, _, rel = member.name.partition("/") + if not rel.startswith(prefix): + continue + rel = rel[len(prefix):] + target = (dest / rel).resolve() + # A tarball is attacker-controlled input in principle, and this + # one is unpacked as root. Refuse any member that resolves + # outside dest rather than trusting the forge to never serve a + # `../` path. + if not str(target).startswith(str(dest.resolve()) + os.sep): + raise AppError(f"archive member escapes the destination: {member.name}") + target.parent.mkdir(parents=True, exist_ok=True) + src = tf.extractfile(member) + if src is None: + continue + with open(target, "wb") as out: + shutil.copyfileobj(src, out) + found += 1 + if not found: + raise AppError(f"{fastlane_dir}/metadata/android/ is empty or missing at tag {tag}") + return found + + +def open_apk(apk: Path): + """androguard's APK reader, or None if androguard is not installed.""" + try: + from androguard.core.apk import APK + except ImportError: + return None + # androguard logs through loguru, not the stdlib, and at DEBUG it prints a + # line per resource-table oddity — hundreds of them per Flutter APK, which + # buries the one line we actually emit here. + try: + from loguru import logger + + logger.remove() + except ImportError: + pass + return APK(str(apk)) + + +def check_version_code(apk: Path, code): + """Fail if the tag and the APK disagree about which version this is. + + Everything downstream keys off the version code taken from the git tag: + it names the APK in repo/, it is what pruning sorts on, and it is what + resolves the changelog. F-Droid, meanwhile, reads the code out of the APK's + own manifest. Normally both come from the same pubspec line and agree, but + nothing enforces that — a hand-moved tag is enough to split them, and then + the repo publishes a file whose name claims one version while the index + claims another, and pruning starts ordering by a number that means nothing. + """ + pkg = open_apk(apk) + if pkg is None: + return + actual = int(pkg.get_androidversion_code()) + if actual != code: + raise AppError( + f"{apk.name}: tag says versionCode {code} but the APK's manifest " + f"says {actual}; the tag and the build do not match" + ) + + +def extract_apk_icon(apk: Path, dest: Path): + """Last resort: pull the launcher icon out of the APK itself. + + An app whose fastlane tree carries no images/icon.png would otherwise be + listed with no icon at all, because fdroidserver does not fall back to the + APK. Two of these four apps are in exactly that state, and an iconless row + in the client reads as a broken repo rather than as missing metadata. + + Shipping an icon in fastlane is still the better answer — it can be the + square store icon rather than whatever density the launcher happens to + use — so this only fills the gap, it does not replace it. + """ + pkg = open_apk(apk) + if pkg is None: + log(" androguard unavailable; cannot recover an icon from the APK") + return False + # Descending, because the default pick is whatever the manifest points at, + # and on every modern Android app that is an adaptive-icon XML rather than + # a bitmap. Asking for a bounded density returns a real PNG. + for dpi in (640, 480, 320, 240, 160): + name = pkg.get_app_icon(max_dpi=dpi) + if name and name.lower().endswith(".png"): + data = pkg.get_file(name) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(data) + log(f" recovered icon from the APK ({name}, <={dpi}dpi, {len(data)} bytes)") + return True + log(" no bitmap launcher icon in the APK; listing will have no icon") + return False + + +_listing_done = {} + + +def sync_listing(app, cfg, tag, apk: Path, locale): + """Refresh metadata// from the app repo. True if it changed. + + Refetched on every run rather than only when the release moves, so a + description or screenshot corrected upstream without a new release still + lands here. Within a run it is done once per tag: the later passes exist to + notice a *new* release, and repeating the fetch and the APK icon parse for + a tag already handled would be pure waste. + """ + if _listing_done.get(app["id"]) == tag: + return False + root = Path(cfg["root"]) + live = root / "metadata" / app["packageId"] + with tempfile.TemporaryDirectory(dir=root / "tmp") as td: + staged = Path(td) / "metadata" + staged.mkdir() + extract_fastlane(cfg["forge"], app["repo"], tag, app.get("fastlane", "fastlane"), staged) + icon = staged / locale / "images" / "icon.png" + if not icon.exists(): + extract_apk_icon(apk, icon) + _listing_done[app["id"]] = tag + if live.exists() and tree_manifest(live) == tree_manifest(staged): + return False + # Replace rather than merge, so a file deleted upstream disappears here + # too. The swap is two renames in the same filesystem; `fdroid update` + # is not running concurrently (the unit is a oneshot) so there is no + # reader to tear. + if live.exists(): + shutil.rmtree(live.with_name(live.name + ".old"), ignore_errors=True) + live.rename(live.with_name(live.name + ".old")) + staged.rename(live) + shutil.rmtree(live.with_name(live.name + ".old"), ignore_errors=True) + return True + + +def app_metadata(app, code, version_name): + """The non-localized half of the listing: metadata/.yml. + + Name, Summary and Description are deliberately not set. fdroidserver reads + those from the localized files this script just unpacked, and a value here + would win over them — pinning the listing to whatever was true when this + file was written. + + Two fields here are load-bearing for the changelog, and both have to be + right or the client shows an empty "What's New": + + CurrentVersionCode is what fdroidserver resolves changelogs/.txt + against, so without it the changelog file is never read at all. + + Builds is what gets the text into the index once read. index-v2 only + carries whatsNew for a version matching the *last* Builds entry, and a repo + of prebuilt APKs has no build recipes, so the entry has to be declared. It + is metadata only — nothing here builds anything, `fdroid update` just + matches versionCode against it. + """ + forge_repo = app["forgeUrl"] + lines = [ + "# Generated by fdroid-sync. Edit apps.nix, not this file.", + f"AuthorName: {app.get('authorName', 'randogoth')}", + f"License: {app['license']}", + "Categories:", + ] + lines += [f" - {c}" for c in app["categories"]] + lines += [ + f"SourceCode: {forge_repo}", + f"IssueTracker: {forge_repo}/issues", + f"Changelog: {forge_repo}/releases", + ] + if app.get("webSite"): + lines.append(f"WebSite: {app['webSite']}") + lines += [ + f"CurrentVersion: {version_name}", + f"CurrentVersionCode: {code}", + "Builds:", + f" - versionName: {version_name}", + f" versionCode: {code}", + "", + ] + return "\n".join(lines) + + +# -------------------------------------------------------------------------- +# per-app reconcile + + +def prune_apks(root: Path, package_id, keep): + """Drop all but the newest `keep` versions of one app. + + F-Droid's own answer is archive_older, which moves old APKs into archive/ + and publishes a second index for them. That is a feature for a repo with + users who pin old versions; here it would be a second index to sign and + serve so that nobody can reach four-releases-ago builds of a launcher. + Deleting is the honest version of the same decision, and because the index + is regenerated from whatever APKs are present, it needs no bookkeeping. + """ + found = [] + for apk in (root / "repo").glob(f"{package_id}_*.apk"): + code = apk.stem[len(package_id) + 1:] + if code.isdigit(): + found.append((int(code), apk)) + dropped = False + for _code, apk in sorted(found, reverse=True)[keep:]: + log(f" pruning {apk.name} (keeping newest {keep})") + apk.unlink() + dropped = True + return dropped + + +def reconcile(app, cfg): + """Bring one app's APK and listing in line with its latest release.""" + root = Path(cfg["root"]) + locale = cfg.get("locale", DEFAULT_LOCALE) + release = latest_release(cfg["forge"], app["repo"]) + tag = release["tag_name"] + name, code = version_code(tag) + apk_path = root / "repo" / f"{app['packageId']}_{code}.apk" + + changed = False + if apk_path.exists(): + log(f" {app['id']}: {tag} already published") + else: + asset = await_apk_asset(cfg["forge"], app["repo"], release) + log(f" {app['id']}: fetching {tag} ({asset['name']}, {asset['size']} bytes)") + download(asset["browser_download_url"], apk_path, expect_size=asset["size"]) + try: + check_version_code(apk_path, code) + except AppError: + # Unlink first: left in place it would be picked up by the next + # run as "already published" and never checked again. + apk_path.unlink() + raise + changed = True + + if sync_listing(app, cfg, tag, apk_path, locale): + log(f" {app['id']}: listing updated from {tag}") + changed = True + + meta = root / "metadata" / f"{app['packageId']}.yml" + want = app_metadata(app, code, name) + if not meta.exists() or meta.read_text() != want: + meta.write_text(want) + log(f" {app['id']}: metadata set to version {name} ({code})") + changed = True + + if prune_apks(root, app["packageId"], cfg.get("keep", 3)): + changed = True + return changed + + +# -------------------------------------------------------------------------- + + +def build_index(cfg): + """Sign a fresh index over whatever is now in repo/. + + Deliberately without --create-metadata: an APK with no metadata/.yml + means apps.nix and the repo disagree, and the useful outcome is a loud + failure, not a listing auto-filled with the working directory's name as its + category (which is what the skeleton generator actually does). + """ + log("building and signing the index") + env = dict(os.environ) + env["FDROID_KEYSTORE_PASS"] = Path(cfg["keystorePassFile"]).read_text().strip() + # fdroidserver reads keypass separately even when it is the same secret. + env["FDROID_KEY_PASS"] = env["FDROID_KEYSTORE_PASS"] + subprocess.run([cfg["fdroid"], "update"], cwd=cfg["root"], env=env, check=True) + + +def main(argv): + if len(argv) < 2: + print(__doc__, file=sys.stderr) + return 2 + cfg = json.loads(Path(argv[1]).read_text()) + root = Path(cfg["root"]) + for sub in ("repo", "metadata", "tmp"): + (root / sub).mkdir(parents=True, exist_ok=True) + + wanted = set(argv[2:]) + apps = [a for a in cfg["apps"] if not wanted or a["id"] in wanted] + unknown = wanted - {a["id"] for a in cfg["apps"]} + if unknown: + print(f"unknown app(s): {', '.join(sorted(unknown))}", file=sys.stderr) + return 2 + for app in apps: + app["forgeUrl"] = f"{cfg['forge']}/{app['repo']}" + + # An index that was never built is a change in itself, otherwise a first + # run on a fresh server that finds every APK already in place (restored + # from a backup, say) would serve no index at all. + changed = not (root / "repo" / "index-v2.json").exists() + failed = [] + for attempt in range(1, MAX_PASSES + 1): + log(f"checking {len(apps)} app(s) against {cfg['forge']} (pass {attempt})") + moved = False + failed = [] + for app in apps: + try: + if reconcile(app, cfg): + moved = True + except AppError as e: + log(f" {app['id']}: {e}") + failed.append(app["id"]) + # Broad on purpose: one app's surprise must not sink the other three. + except Exception as e: + log(f" {app['id']}: unexpected {type(e).__name__}: {e}") + failed.append(app["id"]) + changed |= moved + if not moved: + break + + if changed: + build_index(cfg) + else: + log("nothing changed; index left alone") + + if failed: + # Loud, but only after everything that could be published has been. + log(f"FAILED: {', '.join(failed)}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/flake.lock b/flake.lock index 915f6ef..227b212 100644 --- a/flake.lock +++ b/flake.lock @@ -2,16 +2,16 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1791594108, - "narHash": "sha256-XK4KceonkwD8LuMAt/vNrz982mXJM5jeHAxMwsUXXAQ=", + "lastModified": 1782847189, + "narHash": "sha256-twXPFqFsrrY5r28Zh7Homgcp2gUMBgQ6WDS98Q/3xFI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6510408d147d8c5e6d76043e899e9251756f6a75", + "rev": "b6018f87da91d19d0ab4cf979885689b469cdd41", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixos-25.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 7dfdeca..4a4b0fe 100644 --- a/flake.nix +++ b/flake.nix @@ -1,16 +1,33 @@ { - description = "Self-hosted F-Droid repo tooling"; + description = "Self-hosted F-Droid repo fed from Forgejo releases"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; outputs = { self, nixpkgs }: let - pkgs = nixpkgs.legacyPackages.x86_64-linux; - in { - devShells.x86_64-linux.default = pkgs.mkShell { - buildInputs = [ pkgs.fdroidserver pkgs.rsync pkgs.jdk ]; - }; + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + in + { + # Everything the repo needs server-side; see module.nix for the options + # and README.md for how bucur consumes this. + nixosModules.default = import ./module.nix; - nixosModules.fdroid-repo = import ./nixos-module.nix; + # The app list, exposed so the serving host can read or extend it + # without importing the whole module. + lib.apps = import ./apps.nix; + + devShells.${system}.default = pkgs.mkShell { + # jdk: fdroidserver bundles apksigner but shells out to keytool, jar + # and jarsigner to build and sign the index. + # androguard: the sync script reads a launcher icon out of an APK when + # the app ships no fastlane icon. + packages = [ + pkgs.fdroidserver + pkgs.jdk + (pkgs.python3.withPackages (ps: [ ps.androguard ])) + pkgs.jq + ]; + }; }; } diff --git a/module.nix b/module.nix new file mode 100644 index 0000000..cf6866d --- /dev/null +++ b/module.nix @@ -0,0 +1,309 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.fdroidRepo; + + # fdroidserver bundles its own apksigner but no JDK, and `fdroid update` + # shells out to keytool, jar and jarsigner to build and sign the index. On a + # host with no JDK on PATH that failure arrives late — after every APK has + # been downloaded and parsed — so the JDK is wired in here rather than left + # to the ambient environment. + # + # androguard comes from the same package set so the two agree on it: the sync + # script imports it to recover a launcher icon from an APK whose fastlane + # tree ships none. + pythonEnv = pkgs.python3.withPackages (ps: [ ps.androguard ]); + + syncConfig = pkgs.writeText "fdroid-sync.json" (builtins.toJSON { + forge = cfg.forgeUrl; + root = cfg.stateDir; + keep = cfg.keepVersions; + locale = cfg.locale; + keystorePassFile = cfg.keystorePassFile; + fdroid = "${pkgs.fdroidserver}/bin/fdroid"; + apps = cfg.apps; + }); + + # No secrets in here, which is the point of the {env: ...} indirection: + # fdroidserver resolves those from the environment at run time, so the file + # can live in the world-readable Nix store while the keystore password stays + # in sops. Writing the password in literally — which is what `fdroid init` + # does — would publish it to every user on the host. + repoConfig = pkgs.writeText "fdroid-config.yml" '' + repo_url: https://${cfg.domain}/fdroid/repo + repo_name: ${cfg.repoName} + repo_description: ${cfg.repoDescription} + repo_keyalias: ${cfg.keyAlias} + keystore: ${cfg.keystoreFile} + keystorepass: {env: FDROID_KEYSTORE_PASS} + keypass: {env: FDROID_KEY_PASS} + # Pruning is done by the sync script, which deletes old APKs outright. The + # alternative is this knob, which moves them to archive/ and expects a + # second signed index served alongside the first — a lot of moving parts so + # that nobody can install four-releases-ago builds. + archive_older: 0 + # Would drop a symlink named after the app next to the repo on every + # update. Harmless, but it clutters the directory Caddy serves. + make_current_version_link: false + ''; + + syncScript = pkgs.writeShellScript "fdroid-sync" '' + set -euo pipefail + # Set outright rather than prepended to systemd's default PATH, because + # what fdroidserver needs here is not obvious and inheriting it hides a + # missing entry until the thing that needs it runs: + # + # jdk — `fdroid update` calls jar, keytool and jarsigner by bare + # name to build and sign the index + # findutils — the bundled apksigner is a Gradle-style launcher that + # dies with "xargs is not available" without it. The + # failure is not a clean one: fdroidserver reads it as + # every APK having an invalid signature and quarantines + # the lot into archive/ before giving up on the index. + # coreutils — the same launcher runs `uname`; install(1) below + export PATH=${ + lib.makeBinPath [ pkgs.jdk pkgs.coreutils pkgs.findutils pkgs.gnused pkgs.gnugrep ] + } + install -d -m 0755 ${cfg.stateDir}/repo ${cfg.stateDir}/metadata ${cfg.stateDir}/tmp + # 0600, because fdroidserver warns about a group-readable config.yml on + # every run. There is nothing secret in ours, but a warning that is always + # printed is a warning nobody reads when it finally matters. + install -m 0600 ${repoConfig} ${cfg.stateDir}/config.yml + exec ${pythonEnv}/bin/python3 ${./fdroid-sync.py} ${syncConfig} "$@" + ''; + + # Templated with a placeholder; the real HMAC secret is substituted at + # service start from the sops-managed file, so it never enters the store. + hooksTemplate = pkgs.writeText "fdroid-hooks.json" (builtins.toJSON [{ + id = "rebuild-fdroid"; + execute-command = "/run/current-system/sw/bin/systemctl"; + pass-arguments-to-command = [ + { source = "string"; name = "start"; } + { source = "string"; name = "--no-block"; } + { source = "string"; name = "fdroid-sync.service"; } + ]; + response-message = "Queued an F-Droid repo rebuild"; + trigger-rule.match = { + type = "payload-hmac-sha256"; + secret = "placeholder"; + # Forgejo signs with the webhook's secret under Gitea's header name. + parameter = { source = "header"; name = "X-Gitea-Signature"; }; + }; + }]); + + makeHooks = pkgs.writeShellScript "fdroid-make-hooks" '' + set -euo pipefail + ${pkgs.jq}/bin/jq \ + --arg s "$(cat ${cfg.webhookSecretFile})" \ + 'map(.["trigger-rule"].match.secret = $s)' \ + ${hooksTemplate} > /run/fdroid-webhook/hooks.json + ''; +in +{ + options.services.fdroidRepo = { + enable = lib.mkEnableOption "self-hosted F-Droid repo fed from Forgejo releases"; + + domain = lib.mkOption { + type = lib.types.str; + example = "apps.randogoth.com"; + description = '' + Host the repo is served from. Baked into the signed index as the + repo's address, so changing it after people have subscribed points + their clients at the old one until they re-add the repo. + ''; + }; + + forgeUrl = lib.mkOption { + type = lib.types.str; + default = "https://code.randogoth.com"; + description = "Base URL of the Forgejo instance holding the releases."; + }; + + apps = lib.mkOption { + type = lib.types.listOf (lib.types.attrsOf lib.types.anything); + default = import ./apps.nix; + description = "Apps to publish. See apps.nix for the field reference."; + }; + + stateDir = lib.mkOption { + type = lib.types.str; + default = "/var/lib/fdroid"; + description = '' + fdroidserver working directory: config.yml, metadata/ and the repo/ + tree that gets served. Holds every published APK, so size it for + `keepVersions` x the number of apps x APK size. + ''; + }; + + keepVersions = lib.mkOption { + type = lib.types.ints.positive; + default = 3; + description = '' + How many versions of each app to keep. Older APKs are deleted and drop + out of the index on the next update. + ''; + }; + + locale = lib.mkOption { + type = lib.types.str; + default = "en-US"; + description = '' + Locale treated as the default when filling gaps — currently only for + deciding where a recovered APK icon goes. + ''; + }; + + repoName = lib.mkOption { + type = lib.types.str; + default = "randogoth apps"; + description = "Repo name shown in the F-Droid client."; + }; + + repoDescription = lib.mkOption { + type = lib.types.str; + default = "Apps by randogoth, built and signed by their own CI on code.randogoth.com."; + description = "Repo description shown in the F-Droid client."; + }; + + keyAlias = lib.mkOption { + type = lib.types.str; + default = "nanuqsaurus"; + description = "Alias of the index signing key inside the keystore."; + }; + + keystoreFile = lib.mkOption { + type = lib.types.str; + example = "/run/secrets/fdroid_keystore"; + description = '' + Path to the PKCS#12 keystore holding the repo's index signing key. + This key *is* the repo's identity: clients pin its fingerprint on + subscribe, so replacing it orphans every existing subscriber. Back it + up somewhere that is not this server. + ''; + }; + + keystorePassFile = lib.mkOption { + type = lib.types.str; + example = "/run/secrets/fdroid_keystore_pass"; + description = "File containing the keystore password (used for both store and key)."; + }; + + webhookSecretFile = lib.mkOption { + type = lib.types.str; + example = "/run/secrets/fdroid_webhook_secret"; + description = "File containing the shared secret Forgejo signs webhook payloads with."; + }; + + webhookPort = lib.mkOption { + type = lib.types.port; + default = 9001; + description = '' + Localhost port for the rebuild webhook, reverse-proxied at + /hooks/ on `domain`. Separate from bucur's site-deploy webhook on + 9000 so that neither can take the other down. + ''; + }; + + serveCaddy = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Write a Caddyfile fragment to /etc/caddy/Caddyfile.d/, which bucur's + Caddyfile glob-imports. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ (pkgs.writeShellScriptBin "fdroid-sync" "exec ${syncScript} \"$@\"") ]; + + systemd.services.fdroid-sync = { + description = "Publish the latest ${toString (builtins.length cfg.apps)} app releases to the F-Droid repo"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + + # A system activation must not kill a run in flight: it is a oneshot + # triggered by a webhook, so there is nothing for an activation to + # restart, and interrupting one mid-`fdroid update` would leave a + # half-written index being served. + restartIfChanged = false; + stopIfChanged = false; + + serviceConfig = { + Type = "oneshot"; + ExecStart = syncScript; + StateDirectory = "fdroid"; + StateDirectoryMode = "0755"; # Caddy reads repo/ out of here + # Generous, because a run that arrives via webhook may sit waiting for + # CI to finish uploading an 80 MB APK before it can do anything. + TimeoutStartSec = "45min"; + # One CPU on this host, shared with Forgejo and the runner that just + # built the APK. Signing an index is not worth starving them for. + Nice = 10; + IOSchedulingClass = "idle"; + }; + }; + + systemd.timers.fdroid-sync = { + description = "Reconcile the F-Droid repo with the forge daily"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "daily"; + RandomizedDelaySec = "2h"; + # The webhook is the fast path and this is the floor under it: a + # delivery that failed or was never sent is never retried by Forgejo, + # so without a timer a missed release sits unpublished until the next + # one happens to carry it along. + Persistent = true; + }; + }; + + systemd.services.fdroid-webhook = { + description = "Webhook receiver for F-Droid repo rebuilds"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "sops-nix.service" ]; + serviceConfig = { + Type = "simple"; + RuntimeDirectory = "fdroid-webhook"; + ExecStartPre = "+${makeHooks}"; + ExecStart = "${pkgs.webhook}/bin/webhook -hooks /run/fdroid-webhook/hooks.json -port ${toString cfg.webhookPort}"; + Restart = "on-failure"; + }; + }; + + environment.etc = lib.mkIf cfg.serveCaddy { + "caddy/Caddyfile.d/fdroid.caddyfile".text = '' + ${cfg.domain} { + handle /hooks/* { + reverse_proxy localhost:${toString cfg.webhookPort} + } + + # F-Droid's convention is /fdroid/repo, and the fingerprint + # clients pin is tied to that exact address via the signed index, + # so it is not a path that can be tidied up later. + # + # Rooted at repo/ rather than at the state directory with /fdroid + # stripped, which would have served the whole fdroidserver working + # tree: config.yml and metadata/ are not secret, but they are not + # part of the repo and nothing should be able to fetch them. + redir /fdroid/repo /fdroid/repo/ permanent + handle_path /fdroid/repo/* { + root * ${cfg.stateDir}/repo + # No `browse`: repo/index.html is the landing page, and a + # directory listing for everything else only advertises paths + # that no client needs to discover. + file_server + } + + # `fdroid update` writes repo/index.html: the repo's own landing + # page, listing every app and carrying the QR code that adds the + # repo to a phone. Better than anything hand-written here, and it + # can never fall out of step with the index. + redir / /fdroid/repo/ permanent + + encode zstd gzip + } + ''; + }; + }; +} diff --git a/nixos-module.nix b/nixos-module.nix deleted file mode 100644 index 4ef5572..0000000 --- a/nixos-module.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; - -let - cfg = config.services.fdroidRepo; -in { - options.services.fdroidRepo = { - enable = mkEnableOption "static F-Droid repo hosting via nginx"; - - domain = mkOption { - type = types.str; - description = "Domain the repo is served from, e.g. apps.example.com"; - }; - - repoPath = mkOption { - type = types.path; - default = "/var/lib/fdroid-repo"; - description = "Directory containing the synced repo/ output (index-v1.json, index-v2.json, apks, icons)."; - }; - }; - - config = mkIf cfg.enable { - services.nginx = { - enable = true; - virtualHosts.${cfg.domain} = { - enableACME = true; - forceSSLRedirect = true; - root = cfg.repoPath; - }; - }; - - # Requires security.acme.defaults.email and security.acme.acceptTerms = true - # to be set in your top-level NixOS config. - - systemd.tmpfiles.rules = [ - "d ${cfg.repoPath} 0755 nginx nginx -" - ]; - }; -} diff --git a/sync.sh b/sync.sh deleted file mode 100755 index e9063f2..0000000 --- a/sync.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Usage: ./sync.sh user@host:/var/lib/fdroid-repo -# Regenerates the index, then rsyncs repo/ to the serving host. - -cd "$(dirname "$0")" - -fdroid update -c - -if [ $# -lt 1 ]; then - echo "Usage: $0 user@host:/var/lib/fdroid-repo" >&2 - exit 1 -fi - -rsync -av --delete repo/ "$1/"