diff --git a/.gitignore b/.gitignore index 2c837ac..c26a20c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,15 @@ +# The repo's index signing key and the config that holds its password. The key +# is the repo's identity to every subscriber: back it up off this machine. +# Server-side it lives in bucur's sops store, not here. keystore.p12 config.yml + +# Local scratch from running fdroid by hand; the real tree lives on the server. repo/ -unsigned/ +metadata/ archive/ +unsigned/ tmp/ stats/ +result +__pycache__/ diff --git a/apps.nix b/apps.nix new file mode 100644 index 0000000..ab24d36 --- /dev/null +++ b/apps.nix @@ -0,0 +1,56 @@ +# The apps this repo serves. One entry per app; adding an app is adding an +# entry here and pushing. +# +# Everything in this file is the part of a listing that does not change when a +# release is cut: the package identity, where its releases live, its licence +# and where it belongs in F-Droid's category list. Everything that *does* +# change per release — name, summary, description, changelog, icon, +# screenshots — is deliberately absent, because it is read from the app repo's +# own fastlane/ tree at the released tag. Writing a description here would +# create a second copy of it that silently drifts from the one the app ships. +# +# Fields: +# id short slug; names log lines and the --app filter, nothing else +# packageId the Android applicationId. This is the app's identity to +# F-Droid and to every phone that has it installed, so it has to +# match `applicationId` in android/app/build.gradle{,.kts} +# exactly — a typo here publishes a second, unrelated app rather +# than an update to the existing one. +# repo owner/name on the forge +# license SPDX identifier, shown in the client +# categories F-Droid's fixed category vocabulary, not free text +# fastlane where the fastlane tree sits inside the repo (default +# "fastlane"; the-mews keeps its Flutter app under app/) +# webSite optional; omitted means the client shows only the source link +[ + { + id = "muzzlevelocity"; + packageId = "vision.flux.muzzlevelocity"; + repo = "randogoth/muzzle-velocity"; + license = "Apache-2.0"; + categories = [ "Writing" ]; + } + { + id = "plasmoid"; + packageId = "com.randogoth.plasmoid"; + repo = "randogoth/plasmoid"; + license = "Apache-2.0"; + categories = [ "System" ]; + } + { + id = "themews"; + packageId = "page.mews.mews"; + repo = "randogoth/the-mews"; + license = "Apache-2.0"; + categories = [ "Internet" ]; + fastlane = "app/fastlane"; + webSite = "https://mews.page"; + } + { + id = "kirakira"; + packageId = "com.app.smol_mail"; + repo = "randogoth/kirakira"; + license = "MIT"; + categories = [ "Internet" ]; + } +] diff --git a/bin/register-webhooks.sh b/bin/register-webhooks.sh new file mode 100755 index 0000000..2d7e84d --- /dev/null +++ b/bin/register-webhooks.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Point every app repo's release webhook at the F-Droid rebuild endpoint. +# +# Forgejo webhooks are instance state, not repo content, so they cannot be +# declared in a flake and have to be registered once per repo. This does that +# from the single source of truth — apps.nix — so adding an app means editing +# that file and re-running this, not remembering four web forms. +# +# Idempotent: a hook already pointing at the same URL is updated in place +# rather than duplicated, so re-running after a secret rotation is the +# supported way to roll the secret. +# +# The secret must be the same value the serving host reads from +# webhookSecretFile; a mismatch makes every delivery fail the HMAC check and +# the repo silently stops updating. Pipe it straight out of sops so it reaches +# neither the terminal nor the shell history: +# +# +# FORGEJO_TOKEN= \ +# FDROID_WEBHOOK_SECRET=$(= \ +# nix run nixpkgs#sops -- -d --extract '["fdroid_webhook_secret"]' \ +# ) \ +# ../fdroid-repo/bin/register-webhooks.sh +# +# Usage: register-webhooks.sh [base-url] +set -euo pipefail + +cd "$(dirname "$0")/.." + +: "${FORGEJO_TOKEN:?set FORGEJO_TOKEN to a Forgejo access token with repo write access}" +# Env, not an argument: an argv is visible to every other process on the box +# for as long as the script runs. +SECRET="${FDROID_WEBHOOK_SECRET:?set FDROID_WEBHOOK_SECRET to the shared webhook secret}" +BASE="${1:-https://apps.randogoth.com}" +FORGE="${FORGE:-https://code.randogoth.com}" +URL="$BASE/hooks/rebuild-fdroid" + +api() { + local method=$1 path=$2 + shift 2 + curl -sS -X "$method" \ + -H "Authorization: token $FORGEJO_TOKEN" \ + -H "Content-Type: application/json" \ + "$FORGE/api/v1$path" "$@" +} + +for repo in $(nix eval --json --file ./apps.nix | jq -r '.[].repo'); do + # Forgejo returns the full hook list; match on the target URL because the + # hook id is assigned by the server and is not knowable in advance. + existing=$(api GET "/repos/$repo/hooks" | jq -r --arg u "$URL" \ + '.[] | select(.config.url == $u) | .id' | head -1) + + payload=$(jq -n --arg url "$URL" --arg secret "$SECRET" '{ + type: "gitea", + active: true, + # "release" alone: a tag push fires before CI has built anything, so a + # push-triggered rebuild would always find the old release and do nothing. + events: ["release"], + config: { url: $url, content_type: "json", secret: $secret } + }') + + if [ -n "$existing" ]; then + api PATCH "/repos/$repo/hooks/$existing" -d "$payload" >/dev/null + echo "updated $repo (hook $existing) -> $URL" + else + api POST "/repos/$repo/hooks" -d "$payload" >/dev/null + echo "created $repo -> $URL" + fi +done + +echo +echo "Note: the release webhook fires when the release object is created," +echo "which in these workflows is one API call before the APK is attached." +echo "fdroid-sync waits for the upload to land, so that race is handled." diff --git a/fdroid-sync.py b/fdroid-sync.py new file mode 100755 index 0000000..fab33d6 --- /dev/null +++ b/fdroid-sync.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 +"""Reconcile a self-hosted F-Droid repo with the latest Forgejo releases. + +Nothing is built here. Each app's APK is the one its own CI built and signed, +and its listing text comes from the fastlane/ tree in the same repo at the same +tag, so the store entry and the binary can never describe different releases. + +A run is a pure function of what the forge currently holds: it works out what +should be in the repo, makes it so, and rebuilds the index only if something +actually moved. That is what makes it safe to drive from a webhook, which has +no delivery guarantee and no retry — a missed delivery costs latency, not +correctness, because the next run (or the timer) converges anyway. + +Usage: fdroid-sync [app-id ...] +""" + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +# Both halves of a release filename come from the git tag, which the app +# workflows write as v+ (Flutter's own pubspec +# version syntax). The code is what F-Droid orders versions by and what names +# the changelog file, so a tag without one is an error rather than a guess. +TAG_SEP = "+" + +# How long to keep waiting for an APK that is still uploading. The release +# webhook fires when the release object is created, which in these workflows is +# one API call *before* the APK is attached to it, so a webhook-triggered run +# routinely arrives at a release with no asset yet. That is a race to wait out, +# not an error: a 50-80 MB upload off a CI runner takes a while. +ASSET_WAIT_SECONDS = 600 +ASSET_POLL_SECONDS = 20 + +# A run can be triggered while an earlier one is still going, and systemd +# merges the second `start` into the first rather than queueing it, so the +# release that triggered it would be missed until the daily timer. Re-checking +# the forge after a pass that changed something closes that window; a pass that +# changes nothing ends the loop, so the steady state is still a single pass. +MAX_PASSES = 3 + +DEFAULT_LOCALE = "en-US" + +# Per-read, not per-transfer, so this is not a cap on how long an 80 MB APK may +# take — only on how long the forge may go silent mid-stream. +HTTP_TIMEOUT = 60 + + +def log(*args): + print(*args, flush=True) + + +class AppError(Exception): + """A failure confined to one app, so the other three still get a chance.""" + + +# -------------------------------------------------------------------------- +# forge API + + +def api_get(url): + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp: + return json.load(resp) + + +def download(url, dest: Path, expect_size=None): + """Fetch to a sibling temp file, then rename. + + Never streams straight onto the destination: a half-written file under + repo/ is one `fdroid update` away from being published as a real release, + and a phone that downloads it gets a signature failure it cannot explain. + """ + tmp = dest.with_name(dest.name + ".part") + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp, open(tmp, "wb") as out: + shutil.copyfileobj(resp, out, 1 << 20) + size = tmp.stat().st_size + if expect_size is not None and size != expect_size: + tmp.unlink() + raise AppError(f"{dest.name}: got {size} bytes, forge advertised {expect_size}") + tmp.rename(dest) + return size + + +def latest_release(forge, repo): + """The newest published, non-draft, non-prerelease release.""" + url = f"{forge}/api/v1/repos/{repo}/releases/latest" + try: + return api_get(url) + except urllib.error.HTTPError as e: + if e.code == 404: + raise AppError("no published release yet") from None + raise + + +def version_code(tag): + name, sep, code = tag.lstrip("v").partition(TAG_SEP) + if not sep or not code.isdigit(): + raise AppError( + f"tag {tag!r} is not v{TAG_SEP}; " + "cannot tell which version this is" + ) + return name, int(code) + + +def apk_asset(release): + apks = [a for a in release.get("assets", []) if a["name"].endswith(".apk")] + if len(apks) > 1: + raise AppError( + f"release {release['tag_name']} has {len(apks)} APKs " + f"({', '.join(a['name'] for a in apks)}); expected exactly one" + ) + return apks[0] if apks else None + + +def await_apk_asset(forge, repo, release): + """Poll until the APK shows up, or give up loudly.""" + asset = apk_asset(release) + if asset: + return asset + deadline = time.monotonic() + ASSET_WAIT_SECONDS + log(f" {release['tag_name']}: no APK attached yet, waiting for the upload") + while time.monotonic() < deadline: + time.sleep(ASSET_POLL_SECONDS) + asset = apk_asset(latest_release(forge, repo)) + if asset: + return asset + raise AppError( + f"release {release['tag_name']} still has no APK after " + f"{ASSET_WAIT_SECONDS}s; did the build fail after creating the release?" + ) + + +# -------------------------------------------------------------------------- +# listing metadata, straight out of the app's own fastlane tree + + +def tree_manifest(root: Path): + """(relpath, sha256) for every file under root, sorted. + + Used to decide whether a freshly fetched listing differs from the one + already in place. The alternative — always replacing it — would mark every + run as changed and rebuild the index daily for nothing, which rewrites + every index timestamp and makes every client re-download it. + """ + out = [] + for path in sorted(root.rglob("*")): + if path.is_file(): + out.append((str(path.relative_to(root)), hashlib.sha256(path.read_bytes()).hexdigest())) + return out + + +def extract_fastlane(forge, repo, tag, fastlane_dir, dest: Path): + """Unpack /metadata/android/ from the source tag into dest. + + Copied verbatim, subdirectories and all: fdroidserver reads the fastlane + layout natively — title.txt, short_description.txt, full_description.txt, + changelogs/.txt and images/{icon,featureGraphic}.png plus + images/*Screenshots/ — so there is nothing to translate and no mapping + table to keep in step with upstream. + """ + url = f"{forge}/{repo}/archive/{urllib.parse.quote(tag, safe='')}.tar.gz" + prefix = f"{fastlane_dir.strip('/')}/metadata/android/" + with tempfile.TemporaryDirectory() as td: + tarball = Path(td) / "src.tar.gz" + download(url, tarball) + found = 0 + with tarfile.open(tarball) as tf: + for member in tf: + if not member.isfile(): + continue + # Drop the archive's top-level directory, which Forgejo names + # after the repo rather than after anything we control. + _, _, rel = member.name.partition("/") + if not rel.startswith(prefix): + continue + rel = rel[len(prefix):] + target = (dest / rel).resolve() + # A tarball is attacker-controlled input in principle, and this + # one is unpacked as root. Refuse any member that resolves + # outside dest rather than trusting the forge to never serve a + # `../` path. + if not str(target).startswith(str(dest.resolve()) + os.sep): + raise AppError(f"archive member escapes the destination: {member.name}") + target.parent.mkdir(parents=True, exist_ok=True) + src = tf.extractfile(member) + if src is None: + continue + with open(target, "wb") as out: + shutil.copyfileobj(src, out) + found += 1 + if not found: + raise AppError(f"{fastlane_dir}/metadata/android/ is empty or missing at tag {tag}") + return found + + +def open_apk(apk: Path): + """androguard's APK reader, or None if androguard is not installed.""" + try: + from androguard.core.apk import APK + except ImportError: + return None + # androguard logs through loguru, not the stdlib, and at DEBUG it prints a + # line per resource-table oddity — hundreds of them per Flutter APK, which + # buries the one line we actually emit here. + try: + from loguru import logger + + logger.remove() + except ImportError: + pass + return APK(str(apk)) + + +def check_version_code(apk: Path, code): + """Fail if the tag and the APK disagree about which version this is. + + Everything downstream keys off the version code taken from the git tag: + it names the APK in repo/, it is what pruning sorts on, and it is what + resolves the changelog. F-Droid, meanwhile, reads the code out of the APK's + own manifest. Normally both come from the same pubspec line and agree, but + nothing enforces that — a hand-moved tag is enough to split them, and then + the repo publishes a file whose name claims one version while the index + claims another, and pruning starts ordering by a number that means nothing. + """ + pkg = open_apk(apk) + if pkg is None: + return + actual = int(pkg.get_androidversion_code()) + if actual != code: + raise AppError( + f"{apk.name}: tag says versionCode {code} but the APK's manifest " + f"says {actual}; the tag and the build do not match" + ) + + +def extract_apk_icon(apk: Path, dest: Path): + """Last resort: pull the launcher icon out of the APK itself. + + An app whose fastlane tree carries no images/icon.png would otherwise be + listed with no icon at all, because fdroidserver does not fall back to the + APK. Two of these four apps are in exactly that state, and an iconless row + in the client reads as a broken repo rather than as missing metadata. + + Shipping an icon in fastlane is still the better answer — it can be the + square store icon rather than whatever density the launcher happens to + use — so this only fills the gap, it does not replace it. + """ + pkg = open_apk(apk) + if pkg is None: + log(" androguard unavailable; cannot recover an icon from the APK") + return False + # Descending, because the default pick is whatever the manifest points at, + # and on every modern Android app that is an adaptive-icon XML rather than + # a bitmap. Asking for a bounded density returns a real PNG. + for dpi in (640, 480, 320, 240, 160): + name = pkg.get_app_icon(max_dpi=dpi) + if name and name.lower().endswith(".png"): + data = pkg.get_file(name) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(data) + log(f" recovered icon from the APK ({name}, <={dpi}dpi, {len(data)} bytes)") + return True + log(" no bitmap launcher icon in the APK; listing will have no icon") + return False + + +_listing_done = {} + + +def sync_listing(app, cfg, tag, apk: Path, locale): + """Refresh metadata// from the app repo. True if it changed. + + Refetched on every run rather than only when the release moves, so a + description or screenshot corrected upstream without a new release still + lands here. Within a run it is done once per tag: the later passes exist to + notice a *new* release, and repeating the fetch and the APK icon parse for + a tag already handled would be pure waste. + """ + if _listing_done.get(app["id"]) == tag: + return False + root = Path(cfg["root"]) + live = root / "metadata" / app["packageId"] + with tempfile.TemporaryDirectory(dir=root / "tmp") as td: + staged = Path(td) / "metadata" + staged.mkdir() + extract_fastlane(cfg["forge"], app["repo"], tag, app.get("fastlane", "fastlane"), staged) + icon = staged / locale / "images" / "icon.png" + if not icon.exists(): + extract_apk_icon(apk, icon) + _listing_done[app["id"]] = tag + if live.exists() and tree_manifest(live) == tree_manifest(staged): + return False + # Replace rather than merge, so a file deleted upstream disappears here + # too. The swap is two renames in the same filesystem; `fdroid update` + # is not running concurrently (the unit is a oneshot) so there is no + # reader to tear. + if live.exists(): + shutil.rmtree(live.with_name(live.name + ".old"), ignore_errors=True) + live.rename(live.with_name(live.name + ".old")) + staged.rename(live) + shutil.rmtree(live.with_name(live.name + ".old"), ignore_errors=True) + return True + + +def app_metadata(app, code, version_name): + """The non-localized half of the listing: metadata/.yml. + + Name, Summary and Description are deliberately not set. fdroidserver reads + those from the localized files this script just unpacked, and a value here + would win over them — pinning the listing to whatever was true when this + file was written. + + Two fields here are load-bearing for the changelog, and both have to be + right or the client shows an empty "What's New": + + CurrentVersionCode is what fdroidserver resolves changelogs/.txt + against, so without it the changelog file is never read at all. + + Builds is what gets the text into the index once read. index-v2 only + carries whatsNew for a version matching the *last* Builds entry, and a repo + of prebuilt APKs has no build recipes, so the entry has to be declared. It + is metadata only — nothing here builds anything, `fdroid update` just + matches versionCode against it. + """ + forge_repo = app["forgeUrl"] + lines = [ + "# Generated by fdroid-sync. Edit apps.nix, not this file.", + f"AuthorName: {app.get('authorName', 'randogoth')}", + f"License: {app['license']}", + "Categories:", + ] + lines += [f" - {c}" for c in app["categories"]] + lines += [ + f"SourceCode: {forge_repo}", + f"IssueTracker: {forge_repo}/issues", + f"Changelog: {forge_repo}/releases", + ] + if app.get("webSite"): + lines.append(f"WebSite: {app['webSite']}") + lines += [ + f"CurrentVersion: {version_name}", + f"CurrentVersionCode: {code}", + "Builds:", + f" - versionName: {version_name}", + f" versionCode: {code}", + "", + ] + return "\n".join(lines) + + +# -------------------------------------------------------------------------- +# per-app reconcile + + +def prune_apks(root: Path, package_id, keep): + """Drop all but the newest `keep` versions of one app. + + F-Droid's own answer is archive_older, which moves old APKs into archive/ + and publishes a second index for them. That is a feature for a repo with + users who pin old versions; here it would be a second index to sign and + serve so that nobody can reach four-releases-ago builds of a launcher. + Deleting is the honest version of the same decision, and because the index + is regenerated from whatever APKs are present, it needs no bookkeeping. + """ + found = [] + for apk in (root / "repo").glob(f"{package_id}_*.apk"): + code = apk.stem[len(package_id) + 1:] + if code.isdigit(): + found.append((int(code), apk)) + dropped = False + for _code, apk in sorted(found, reverse=True)[keep:]: + log(f" pruning {apk.name} (keeping newest {keep})") + apk.unlink() + dropped = True + return dropped + + +def reconcile(app, cfg): + """Bring one app's APK and listing in line with its latest release.""" + root = Path(cfg["root"]) + locale = cfg.get("locale", DEFAULT_LOCALE) + release = latest_release(cfg["forge"], app["repo"]) + tag = release["tag_name"] + name, code = version_code(tag) + apk_path = root / "repo" / f"{app['packageId']}_{code}.apk" + + changed = False + if apk_path.exists(): + log(f" {app['id']}: {tag} already published") + else: + asset = await_apk_asset(cfg["forge"], app["repo"], release) + log(f" {app['id']}: fetching {tag} ({asset['name']}, {asset['size']} bytes)") + download(asset["browser_download_url"], apk_path, expect_size=asset["size"]) + try: + check_version_code(apk_path, code) + except AppError: + # Unlink first: left in place it would be picked up by the next + # run as "already published" and never checked again. + apk_path.unlink() + raise + changed = True + + if sync_listing(app, cfg, tag, apk_path, locale): + log(f" {app['id']}: listing updated from {tag}") + changed = True + + meta = root / "metadata" / f"{app['packageId']}.yml" + want = app_metadata(app, code, name) + if not meta.exists() or meta.read_text() != want: + meta.write_text(want) + log(f" {app['id']}: metadata set to version {name} ({code})") + changed = True + + if prune_apks(root, app["packageId"], cfg.get("keep", 3)): + changed = True + return changed + + +# -------------------------------------------------------------------------- + + +def build_index(cfg): + """Sign a fresh index over whatever is now in repo/. + + Deliberately without --create-metadata: an APK with no metadata/.yml + means apps.nix and the repo disagree, and the useful outcome is a loud + failure, not a listing auto-filled with the working directory's name as its + category (which is what the skeleton generator actually does). + """ + log("building and signing the index") + env = dict(os.environ) + env["FDROID_KEYSTORE_PASS"] = Path(cfg["keystorePassFile"]).read_text().strip() + # fdroidserver reads keypass separately even when it is the same secret. + env["FDROID_KEY_PASS"] = env["FDROID_KEYSTORE_PASS"] + subprocess.run([cfg["fdroid"], "update"], cwd=cfg["root"], env=env, check=True) + + +def main(argv): + if len(argv) < 2: + print(__doc__, file=sys.stderr) + return 2 + cfg = json.loads(Path(argv[1]).read_text()) + root = Path(cfg["root"]) + for sub in ("repo", "metadata", "tmp"): + (root / sub).mkdir(parents=True, exist_ok=True) + + wanted = set(argv[2:]) + apps = [a for a in cfg["apps"] if not wanted or a["id"] in wanted] + unknown = wanted - {a["id"] for a in cfg["apps"]} + if unknown: + print(f"unknown app(s): {', '.join(sorted(unknown))}", file=sys.stderr) + return 2 + for app in apps: + app["forgeUrl"] = f"{cfg['forge']}/{app['repo']}" + + # An index that was never built is a change in itself, otherwise a first + # run on a fresh server that finds every APK already in place (restored + # from a backup, say) would serve no index at all. + changed = not (root / "repo" / "index-v2.json").exists() + failed = [] + for attempt in range(1, MAX_PASSES + 1): + log(f"checking {len(apps)} app(s) against {cfg['forge']} (pass {attempt})") + moved = False + failed = [] + for app in apps: + try: + if reconcile(app, cfg): + moved = True + except AppError as e: + log(f" {app['id']}: {e}") + failed.append(app["id"]) + # Broad on purpose: one app's surprise must not sink the other three. + except Exception as e: + log(f" {app['id']}: unexpected {type(e).__name__}: {e}") + failed.append(app["id"]) + changed |= moved + if not moved: + break + + if changed: + build_index(cfg) + else: + log("nothing changed; index left alone") + + if failed: + # Loud, but only after everything that could be published has been. + log(f"FAILED: {', '.join(failed)}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/flake.lock b/flake.lock index 915f6ef..227b212 100644 --- a/flake.lock +++ b/flake.lock @@ -2,16 +2,16 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1791594108, - "narHash": "sha256-XK4KceonkwD8LuMAt/vNrz982mXJM5jeHAxMwsUXXAQ=", + "lastModified": 1782847189, + "narHash": "sha256-twXPFqFsrrY5r28Zh7Homgcp2gUMBgQ6WDS98Q/3xFI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6510408d147d8c5e6d76043e899e9251756f6a75", + "rev": "b6018f87da91d19d0ab4cf979885689b469cdd41", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixos-25.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 7dfdeca..4a4b0fe 100644 --- a/flake.nix +++ b/flake.nix @@ -1,16 +1,33 @@ { - description = "Self-hosted F-Droid repo tooling"; + description = "Self-hosted F-Droid repo fed from Forgejo releases"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; outputs = { self, nixpkgs }: let - pkgs = nixpkgs.legacyPackages.x86_64-linux; - in { - devShells.x86_64-linux.default = pkgs.mkShell { - buildInputs = [ pkgs.fdroidserver pkgs.rsync pkgs.jdk ]; - }; + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + in + { + # Everything the repo needs server-side; see module.nix for the options + # and README.md for how bucur consumes this. + nixosModules.default = import ./module.nix; - nixosModules.fdroid-repo = import ./nixos-module.nix; + # The app list, exposed so the serving host can read or extend it + # without importing the whole module. + lib.apps = import ./apps.nix; + + devShells.${system}.default = pkgs.mkShell { + # jdk: fdroidserver bundles apksigner but shells out to keytool, jar + # and jarsigner to build and sign the index. + # androguard: the sync script reads a launcher icon out of an APK when + # the app ships no fastlane icon. + packages = [ + pkgs.fdroidserver + pkgs.jdk + (pkgs.python3.withPackages (ps: [ ps.androguard ])) + pkgs.jq + ]; + }; }; } diff --git a/module.nix b/module.nix new file mode 100644 index 0000000..cf6866d --- /dev/null +++ b/module.nix @@ -0,0 +1,309 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.fdroidRepo; + + # fdroidserver bundles its own apksigner but no JDK, and `fdroid update` + # shells out to keytool, jar and jarsigner to build and sign the index. On a + # host with no JDK on PATH that failure arrives late — after every APK has + # been downloaded and parsed — so the JDK is wired in here rather than left + # to the ambient environment. + # + # androguard comes from the same package set so the two agree on it: the sync + # script imports it to recover a launcher icon from an APK whose fastlane + # tree ships none. + pythonEnv = pkgs.python3.withPackages (ps: [ ps.androguard ]); + + syncConfig = pkgs.writeText "fdroid-sync.json" (builtins.toJSON { + forge = cfg.forgeUrl; + root = cfg.stateDir; + keep = cfg.keepVersions; + locale = cfg.locale; + keystorePassFile = cfg.keystorePassFile; + fdroid = "${pkgs.fdroidserver}/bin/fdroid"; + apps = cfg.apps; + }); + + # No secrets in here, which is the point of the {env: ...} indirection: + # fdroidserver resolves those from the environment at run time, so the file + # can live in the world-readable Nix store while the keystore password stays + # in sops. Writing the password in literally — which is what `fdroid init` + # does — would publish it to every user on the host. + repoConfig = pkgs.writeText "fdroid-config.yml" '' + repo_url: https://${cfg.domain}/fdroid/repo + repo_name: ${cfg.repoName} + repo_description: ${cfg.repoDescription} + repo_keyalias: ${cfg.keyAlias} + keystore: ${cfg.keystoreFile} + keystorepass: {env: FDROID_KEYSTORE_PASS} + keypass: {env: FDROID_KEY_PASS} + # Pruning is done by the sync script, which deletes old APKs outright. The + # alternative is this knob, which moves them to archive/ and expects a + # second signed index served alongside the first — a lot of moving parts so + # that nobody can install four-releases-ago builds. + archive_older: 0 + # Would drop a symlink named after the app next to the repo on every + # update. Harmless, but it clutters the directory Caddy serves. + make_current_version_link: false + ''; + + syncScript = pkgs.writeShellScript "fdroid-sync" '' + set -euo pipefail + # Set outright rather than prepended to systemd's default PATH, because + # what fdroidserver needs here is not obvious and inheriting it hides a + # missing entry until the thing that needs it runs: + # + # jdk — `fdroid update` calls jar, keytool and jarsigner by bare + # name to build and sign the index + # findutils — the bundled apksigner is a Gradle-style launcher that + # dies with "xargs is not available" without it. The + # failure is not a clean one: fdroidserver reads it as + # every APK having an invalid signature and quarantines + # the lot into archive/ before giving up on the index. + # coreutils — the same launcher runs `uname`; install(1) below + export PATH=${ + lib.makeBinPath [ pkgs.jdk pkgs.coreutils pkgs.findutils pkgs.gnused pkgs.gnugrep ] + } + install -d -m 0755 ${cfg.stateDir}/repo ${cfg.stateDir}/metadata ${cfg.stateDir}/tmp + # 0600, because fdroidserver warns about a group-readable config.yml on + # every run. There is nothing secret in ours, but a warning that is always + # printed is a warning nobody reads when it finally matters. + install -m 0600 ${repoConfig} ${cfg.stateDir}/config.yml + exec ${pythonEnv}/bin/python3 ${./fdroid-sync.py} ${syncConfig} "$@" + ''; + + # Templated with a placeholder; the real HMAC secret is substituted at + # service start from the sops-managed file, so it never enters the store. + hooksTemplate = pkgs.writeText "fdroid-hooks.json" (builtins.toJSON [{ + id = "rebuild-fdroid"; + execute-command = "/run/current-system/sw/bin/systemctl"; + pass-arguments-to-command = [ + { source = "string"; name = "start"; } + { source = "string"; name = "--no-block"; } + { source = "string"; name = "fdroid-sync.service"; } + ]; + response-message = "Queued an F-Droid repo rebuild"; + trigger-rule.match = { + type = "payload-hmac-sha256"; + secret = "placeholder"; + # Forgejo signs with the webhook's secret under Gitea's header name. + parameter = { source = "header"; name = "X-Gitea-Signature"; }; + }; + }]); + + makeHooks = pkgs.writeShellScript "fdroid-make-hooks" '' + set -euo pipefail + ${pkgs.jq}/bin/jq \ + --arg s "$(cat ${cfg.webhookSecretFile})" \ + 'map(.["trigger-rule"].match.secret = $s)' \ + ${hooksTemplate} > /run/fdroid-webhook/hooks.json + ''; +in +{ + options.services.fdroidRepo = { + enable = lib.mkEnableOption "self-hosted F-Droid repo fed from Forgejo releases"; + + domain = lib.mkOption { + type = lib.types.str; + example = "apps.randogoth.com"; + description = '' + Host the repo is served from. Baked into the signed index as the + repo's address, so changing it after people have subscribed points + their clients at the old one until they re-add the repo. + ''; + }; + + forgeUrl = lib.mkOption { + type = lib.types.str; + default = "https://code.randogoth.com"; + description = "Base URL of the Forgejo instance holding the releases."; + }; + + apps = lib.mkOption { + type = lib.types.listOf (lib.types.attrsOf lib.types.anything); + default = import ./apps.nix; + description = "Apps to publish. See apps.nix for the field reference."; + }; + + stateDir = lib.mkOption { + type = lib.types.str; + default = "/var/lib/fdroid"; + description = '' + fdroidserver working directory: config.yml, metadata/ and the repo/ + tree that gets served. Holds every published APK, so size it for + `keepVersions` x the number of apps x APK size. + ''; + }; + + keepVersions = lib.mkOption { + type = lib.types.ints.positive; + default = 3; + description = '' + How many versions of each app to keep. Older APKs are deleted and drop + out of the index on the next update. + ''; + }; + + locale = lib.mkOption { + type = lib.types.str; + default = "en-US"; + description = '' + Locale treated as the default when filling gaps — currently only for + deciding where a recovered APK icon goes. + ''; + }; + + repoName = lib.mkOption { + type = lib.types.str; + default = "randogoth apps"; + description = "Repo name shown in the F-Droid client."; + }; + + repoDescription = lib.mkOption { + type = lib.types.str; + default = "Apps by randogoth, built and signed by their own CI on code.randogoth.com."; + description = "Repo description shown in the F-Droid client."; + }; + + keyAlias = lib.mkOption { + type = lib.types.str; + default = "nanuqsaurus"; + description = "Alias of the index signing key inside the keystore."; + }; + + keystoreFile = lib.mkOption { + type = lib.types.str; + example = "/run/secrets/fdroid_keystore"; + description = '' + Path to the PKCS#12 keystore holding the repo's index signing key. + This key *is* the repo's identity: clients pin its fingerprint on + subscribe, so replacing it orphans every existing subscriber. Back it + up somewhere that is not this server. + ''; + }; + + keystorePassFile = lib.mkOption { + type = lib.types.str; + example = "/run/secrets/fdroid_keystore_pass"; + description = "File containing the keystore password (used for both store and key)."; + }; + + webhookSecretFile = lib.mkOption { + type = lib.types.str; + example = "/run/secrets/fdroid_webhook_secret"; + description = "File containing the shared secret Forgejo signs webhook payloads with."; + }; + + webhookPort = lib.mkOption { + type = lib.types.port; + default = 9001; + description = '' + Localhost port for the rebuild webhook, reverse-proxied at + /hooks/ on `domain`. Separate from bucur's site-deploy webhook on + 9000 so that neither can take the other down. + ''; + }; + + serveCaddy = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Write a Caddyfile fragment to /etc/caddy/Caddyfile.d/, which bucur's + Caddyfile glob-imports. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ (pkgs.writeShellScriptBin "fdroid-sync" "exec ${syncScript} \"$@\"") ]; + + systemd.services.fdroid-sync = { + description = "Publish the latest ${toString (builtins.length cfg.apps)} app releases to the F-Droid repo"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + + # A system activation must not kill a run in flight: it is a oneshot + # triggered by a webhook, so there is nothing for an activation to + # restart, and interrupting one mid-`fdroid update` would leave a + # half-written index being served. + restartIfChanged = false; + stopIfChanged = false; + + serviceConfig = { + Type = "oneshot"; + ExecStart = syncScript; + StateDirectory = "fdroid"; + StateDirectoryMode = "0755"; # Caddy reads repo/ out of here + # Generous, because a run that arrives via webhook may sit waiting for + # CI to finish uploading an 80 MB APK before it can do anything. + TimeoutStartSec = "45min"; + # One CPU on this host, shared with Forgejo and the runner that just + # built the APK. Signing an index is not worth starving them for. + Nice = 10; + IOSchedulingClass = "idle"; + }; + }; + + systemd.timers.fdroid-sync = { + description = "Reconcile the F-Droid repo with the forge daily"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "daily"; + RandomizedDelaySec = "2h"; + # The webhook is the fast path and this is the floor under it: a + # delivery that failed or was never sent is never retried by Forgejo, + # so without a timer a missed release sits unpublished until the next + # one happens to carry it along. + Persistent = true; + }; + }; + + systemd.services.fdroid-webhook = { + description = "Webhook receiver for F-Droid repo rebuilds"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "sops-nix.service" ]; + serviceConfig = { + Type = "simple"; + RuntimeDirectory = "fdroid-webhook"; + ExecStartPre = "+${makeHooks}"; + ExecStart = "${pkgs.webhook}/bin/webhook -hooks /run/fdroid-webhook/hooks.json -port ${toString cfg.webhookPort}"; + Restart = "on-failure"; + }; + }; + + environment.etc = lib.mkIf cfg.serveCaddy { + "caddy/Caddyfile.d/fdroid.caddyfile".text = '' + ${cfg.domain} { + handle /hooks/* { + reverse_proxy localhost:${toString cfg.webhookPort} + } + + # F-Droid's convention is /fdroid/repo, and the fingerprint + # clients pin is tied to that exact address via the signed index, + # so it is not a path that can be tidied up later. + # + # Rooted at repo/ rather than at the state directory with /fdroid + # stripped, which would have served the whole fdroidserver working + # tree: config.yml and metadata/ are not secret, but they are not + # part of the repo and nothing should be able to fetch them. + redir /fdroid/repo /fdroid/repo/ permanent + handle_path /fdroid/repo/* { + root * ${cfg.stateDir}/repo + # No `browse`: repo/index.html is the landing page, and a + # directory listing for everything else only advertises paths + # that no client needs to discover. + file_server + } + + # `fdroid update` writes repo/index.html: the repo's own landing + # page, listing every app and carrying the QR code that adds the + # repo to a phone. Better than anything hand-written here, and it + # can never fall out of step with the index. + redir / /fdroid/repo/ permanent + + encode zstd gzip + } + ''; + }; + }; +} diff --git a/nixos-module.nix b/nixos-module.nix deleted file mode 100644 index 4ef5572..0000000 --- a/nixos-module.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; - -let - cfg = config.services.fdroidRepo; -in { - options.services.fdroidRepo = { - enable = mkEnableOption "static F-Droid repo hosting via nginx"; - - domain = mkOption { - type = types.str; - description = "Domain the repo is served from, e.g. apps.example.com"; - }; - - repoPath = mkOption { - type = types.path; - default = "/var/lib/fdroid-repo"; - description = "Directory containing the synced repo/ output (index-v1.json, index-v2.json, apks, icons)."; - }; - }; - - config = mkIf cfg.enable { - services.nginx = { - enable = true; - virtualHosts.${cfg.domain} = { - enableACME = true; - forceSSLRedirect = true; - root = cfg.repoPath; - }; - }; - - # Requires security.acme.defaults.email and security.acme.acceptTerms = true - # to be set in your top-level NixOS config. - - systemd.tmpfiles.rules = [ - "d ${cfg.repoPath} 0755 nginx nginx -" - ]; - }; -} diff --git a/sync.sh b/sync.sh deleted file mode 100755 index e9063f2..0000000 --- a/sync.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Usage: ./sync.sh user@host:/var/lib/fdroid-repo -# Regenerates the index, then rsyncs repo/ to the serving host. - -cd "$(dirname "$0")" - -fdroid update -c - -if [ $# -lt 1 ]; then - echo "Usage: $0 user@host:/var/lib/fdroid-repo" >&2 - exit 1 -fi - -rsync -av --delete repo/ "$1/"