diff --git a/README.md b/README.md index 45b01c7..f5c2311 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,14 @@ The USB, HTTP and BLE API is documented in [arduino/strix/API.md](arduino/strix/ `gcp` prints one line per second: UTC time, second number, the four 200-bit sums, Stouffer Z, Z², and the cumulative Z² − 1 with its p = 0.05 envelope (`*` when outside). `--full` tracks the full-rate trials instead. It reconnects after USB dropouts and fills missed seconds from the device's 60-second history. `--log` appends GCP-style CSV rows, `13,,,,,,` (empty for unhealthy generators), after a `12,gmtime,Date/Time,g0,g1,g2,g3` header. +## HTTP server for a USB-connected STRIX + +`tools/strix_serve.py` (`devbox run serve`) serves the device's WiFi HTTP API routes for a board on USB, so the same clients work with either. Options: `--listen ADDRESS:PORT` (default `127.0.0.1:8080`, use `0.0.0.0:8080` for the network), `--port` for the serial port, `--gcp-log FILE` to append every per-second trial to a JSON-lines file, and `--set-token-file FILE` to require that file's token for `/set` (header `Authorization: Bearer `), for servers reachable by others. + +Since only one program can use the USB port, the server follows the GCP trials itself: `/gcp` serves the last hour instead of the device's 60 seconds, and every record gets a `time` field (Unix time at the start of the second, from the host clock). `/entropy` accepts up to 1 MiB. The server reconnects after USB dropouts; requests in the meantime get status 503. + +The API is described in OpenAPI 3 at `/openapi.json` (`tools/strix_openapi.json`), with an interactive Swagger UI at `/docs` (its scripts load from a CDN). + ## TODO - WiFi credentials stored on the device instead of compile-time defaults diff --git a/arduino/strix/API.md b/arduino/strix/API.md index eccb81f..6296450 100644 --- a/arduino/strix/API.md +++ b/arduino/strix/API.md @@ -83,7 +83,7 @@ Until WiFi is started, the FPGA holds the WiFi module in reset. | `/config` | `get` | | `/set?key=&value=` | `set ` | -Replies have status 200 (errors are JSON `{"error":...}`), or 404 for unknown paths, and allow cross-origin requests. `watch` works over USB and BLE only. +Replies have status 200 (errors are JSON `{"error":...}`), or 404 for unknown paths, and allow cross-origin requests. `tools/strix_serve.py` serves the same routes for a board connected over USB (see the README). `watch` works over USB and BLE only. ## Bluetooth LE diff --git a/devbox.json b/devbox.json index a05756f..631d8b4 100644 --- a/devbox.json +++ b/devbox.json @@ -53,6 +53,9 @@ ], "strix": [ "uv run tools/strix.py \"$@\"" + ], + "serve": [ + "uv run tools/strix_serve.py \"$@\"" ] } } diff --git a/tools/strix_openapi.json b/tools/strix_openapi.json new file mode 100644 index 0000000..0d3273e --- /dev/null +++ b/tools/strix_openapi.json @@ -0,0 +1,675 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "STRIX HTTP API", + "version": "0.4.0", + "description": "HTTP API of a STRIX true random number generator connected over USB, served by tools/strix_serve.py. The routes match the device's own WiFi HTTP API (arduino/strix/API.md); this server adds the `time` field to GCP records, keeps an hour of them, accepts larger entropy requests and answers 503 while the device is unreachable. Device errors are JSON objects with an `error` field and status 200." + }, + "servers": [ + { + "url": "/" + } + ], + "tags": [ + { + "name": "entropy", + "description": "Random bytes from the entropy stream" + }, + { + "name": "device", + "description": "Identity, health, statistics and the bias amplifier" + }, + { + "name": "settings", + "description": "Runtime settings" + }, + { + "name": "gcp", + "description": "Per-second trials in Global Consciousness Project format" + } + ], + "paths": { + "/entropy": { + "get": { + "tags": [ + "entropy" + ], + "summary": "Entropy bytes", + "description": "Bytes from the stream source (setting `stream_source`, default: the XOR of all healthy generators). Refused while that source is not healthy.", + "parameters": [ + { + "name": "bytes", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 1048576 + } + }, + { + "name": "format", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "hex", + "bin" + ], + "default": "hex" + } + } + ], + "responses": { + "200": { + "description": "Uppercase hex, raw bytes, or an error", + "content": { + "text/plain": { + "schema": { + "type": "string" + }, + "example": "3F09A2C47E11D05B" + }, + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + }, + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + } + } + }, + "/": { + "get": { + "tags": [ + "entropy" + ], + "summary": "Entropy bytes as hex (compatibility route)", + "parameters": [ + { + "name": "length", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 1048576 + } + } + ], + "responses": { + "200": { + "description": "Uppercase hex", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + } + } + }, + "/id": { + "get": { + "tags": [ + "device" + ], + "summary": "Device identity", + "responses": { + "200": { + "description": "Identity", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Id" + } + } + } + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + } + } + }, + "/health": { + "get": { + "tags": [ + "device" + ], + "summary": "Generator health and stream state", + "responses": { + "200": { + "description": "Health", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Health" + } + } + } + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + } + } + }, + "/stats": { + "get": { + "tags": [ + "device" + ], + "summary": "Live statistics of the latest trial", + "description": "Per channel (`g0`-`g3`, `combined`): bits, ones, bias z and lag-1 autocorrelation z for the trial, the rolling window and the cumulative total (with random walk position).", + "responses": { + "200": { + "description": "Statistics", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + } + } + }, + "/amp": { + "get": { + "tags": [ + "device" + ], + "summary": "Latest bias amplifier trial", + "description": "Per path (`bias`, `autocorr`): amplified bits, ones, z, sub-trial votes (`1`, `0`, `.` for no vote) and the trial result (null on a tie).", + "responses": { + "200": { + "description": "Amplifier trial", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + } + } + }, + "/wifi": { + "get": { + "tags": [ + "device" + ], + "summary": "WiFi module status", + "responses": { + "200": { + "description": "WiFi status", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + } + } + }, + "/config": { + "get": { + "tags": [ + "settings" + ], + "summary": "All settings", + "responses": { + "200": { + "description": "Settings", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Config" + } + } + } + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + } + } + }, + "/set": { + "get": { + "tags": [ + "settings" + ], + "summary": "Change a setting", + "description": "Value ranges are listed in arduino/strix/API.md. Changing `trial_ms` or `stats_vn` restarts the statistics, `amp_*` the amplifier; `stream_*` flushes the stream. When the server runs with --set-token-file, requests need that token as a bearer token.", + "parameters": [ + { + "name": "key", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": [ + "temporal", + "generators", + "radio_generators", + "trial_ms", + "stats_vn", + "rolling", + "stream_source", + "stream_vn", + "amp_sources", + "amp_method", + "amp_stages", + "amp_sub_ms", + "amp_subtrials", + "amp_vn" + ] + } + }, + { + "name": "value", + "in": "query", + "required": true, + "schema": { + "type": "string" + }, + "example": "5" + } + ], + "responses": { + "200": { + "description": "Confirmation or error", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/Ok" + }, + { + "$ref": "#/components/schemas/Error" + } + ] + } + } + } + }, + "401": { + "description": "Missing or wrong token", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "503": { + "$ref": "#/components/responses/Unavailable" + } + }, + "security": [ + { + "setToken": [] + } + ] + } + }, + "/gcp": { + "get": { + "tags": [ + "gcp" + ], + "summary": "Per-second GCP-format trials", + "description": "Without `since`: the latest second. With `since`: every kept second after it, oldest first (the last hour is kept). Polling with `since` set to the last second received loses no seconds. Second numbers restart when the device restarts.", + "parameters": [ + { + "name": "since", + "in": "query", + "schema": { + "type": "integer", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "One record, a list of records, or an error before the first second", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/GcpRecord" + }, + { + "type": "object", + "required": [ + "records" + ], + "properties": { + "records": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GcpRecord" + } + } + } + }, + { + "$ref": "#/components/schemas/Error" + } + ] + } + } + } + } + } + } + }, + "/openapi.json": { + "get": { + "summary": "This specification", + "responses": { + "200": { + "description": "OpenAPI document", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + } + } + } + }, + "components": { + "responses": { + "Unavailable": { + "description": "The device is not connected", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "NotFound": { + "description": "Unknown path", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + }, + "schemas": { + "Error": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + }, + "Ok": { + "type": "object", + "required": [ + "ok" + ], + "properties": { + "ok": { + "type": "boolean" + } + } + }, + "Id": { + "type": "object", + "properties": { + "device": { + "type": "string", + "example": "STRIX" + }, + "firmware": { + "type": "string", + "example": "0.4.0" + }, + "fpga_ready": { + "type": "boolean" + }, + "fpga_version": { + "type": "integer", + "example": 10 + }, + "generators": { + "type": "integer", + "example": 4 + }, + "rings_per_generator": { + "type": "integer", + "example": 90 + }, + "reset_cause": { + "type": "integer", + "description": "SAMD21 PM->RCAUSE: 1 power-on, 4 brown-out, 16 reset pin, 64 software" + }, + "external_power": { + "type": "boolean", + "description": "Charger or USB power present at boot" + } + } + }, + "Health": { + "type": "object", + "properties": { + "healthy": { + "type": "integer", + "description": "Bit mask of healthy generators" + }, + "generators": { + "type": "array", + "items": { + "type": "object", + "properties": { + "healthy": { + "type": "boolean" + }, + "active": { + "type": "boolean" + }, + "swept": { + "type": "boolean", + "description": "A dead-ring sweep has completed" + }, + "rct_failed": { + "type": "boolean" + }, + "apt_failed": { + "type": "boolean" + }, + "rct_events": { + "type": "integer" + }, + "apt_events": { + "type": "integer" + }, + "dead_rings": { + "type": "integer" + } + } + } + }, + "stream": { + "type": "object", + "properties": { + "level": { + "type": "integer" + }, + "dropped": { + "type": "integer" + } + } + } + } + }, + "Config": { + "type": "object", + "properties": { + "temporal": { + "type": "integer" + }, + "generators": { + "type": "integer" + }, + "trial_ms": { + "type": "integer" + }, + "stats_vn": { + "type": "integer" + }, + "rolling": { + "type": "integer" + }, + "stream_source": { + "type": "integer" + }, + "stream_vn": { + "type": "integer" + }, + "amp_sources": { + "type": "integer" + }, + "amp_method": { + "type": "string", + "enum": [ + "walk", + "vote" + ] + }, + "amp_stages": { + "type": "string", + "example": "8,8,4" + }, + "amp_sub_ms": { + "type": "integer" + }, + "amp_subtrials": { + "type": "integer" + }, + "amp_vn": { + "type": "integer" + }, + "radio_generators": { + "type": "integer" + } + } + }, + "TrialSet": { + "type": "object", + "description": "Per-generator sums of XOR-masked bits; null for generators not healthy for the whole second", + "properties": { + "bits": { + "type": "integer", + "description": "Bits per generator" + }, + "sums": { + "type": "array", + "items": { + "type": "integer", + "nullable": true + } + }, + "z": { + "type": "array", + "items": { + "type": "number", + "nullable": true + }, + "description": "(2 sum - bits) / sqrt(bits)" + }, + "stouffer": { + "type": "number", + "nullable": true, + "description": "Sum of z over sqrt(healthy generators)" + }, + "netvar": { + "type": "number", + "nullable": true, + "description": "stouffer squared, chi-square with one degree of freedom" + } + } + }, + "GcpRecord": { + "type": "object", + "properties": { + "time": { + "type": "integer", + "description": "Unix time at the start of the second, from the host clock" + }, + "second": { + "type": "integer", + "description": "Seconds since the FPGA started" + }, + "healthy": { + "type": "integer", + "description": "Bit mask of generators healthy for the whole second" + }, + "sample": { + "$ref": "#/components/schemas/TrialSet" + }, + "full": { + "$ref": "#/components/schemas/TrialSet" + } + } + } + }, + "securitySchemes": { + "setToken": { + "type": "http", + "scheme": "bearer", + "description": "Token from the server's --set-token-file" + } + } + } +} diff --git a/tools/strix_serve.py b/tools/strix_serve.py new file mode 100644 index 0000000..2d33453 --- /dev/null +++ b/tools/strix_serve.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""HTTP server for a STRIX connected over USB, with the same routes as the device's own WiFi +HTTP API (arduino/strix/API.md), so clients work with either. + +The USB port can only be opened by one program, so this server also follows the per-second +GCP-format trials itself: it polls the device, keeps the last hour in memory for /gcp, adds a +host timestamp to every record (`time`, Unix time at the start of the second) and can append +them to a JSON-lines file (--gcp-log). It reconnects after USB dropouts; requests meanwhile +get status 503. + +With --set-token-file, /set requires the header `Authorization: Bearer `, so the +server can be exposed publicly without letting anyone change settings. + +/openapi.json serves the OpenAPI description (strix_openapi.json) and /docs a Swagger UI page, +which loads its scripts from a CDN. +""" + +import argparse +import hmac +import json +import math +import threading +import time +from collections import deque +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import serial + +from strix import MAX_ENTROPY_REQUEST, Device, find_port + +MAX_HTTP_ENTROPY = 1 << 20 +GCP_KEPT = 3600 # seconds of trials served by /gcp +GCP_POLL_S = 0.5 # also bounds the timestamp error +RECONNECT_S = 2 +OPENAPI = (Path(__file__).parent / "strix_openapi.json").read_bytes() +SWAGGER_UI = b""" + + + +STRIX HTTP API + + + +
+ + + + +""" +PASSTHROUGH = {"/id": "id", "/health": "health", "/stats": "stats", "/amp": "amp", "/wifi": "wifi", + "/config": "get"} + + +class Unavailable(Exception): + pass + + +class Strix: + """The USB connection, shared by request threads and the GCP poller under one lock.""" + + def __init__(self, port: str | None, gcp_log: str | None): + self.port = port + self.lock = threading.Lock() + self.device = None + self.records = deque(maxlen=GCP_KEPT) + self.last_second = None + self.reconnected = False + self.offset = None # host time at the end of second 0 + self.log = open(gcp_log, "a") if gcp_log else None + + def _connected(self) -> Device: + if self.device is None: + try: + device = Device(self.port or find_port()) + # A client that exited without `watch off` leaves the device pushing lines that + # would be taken for replies. + device.send("watch off") + while device.read_line() != '{"ok":true}': + pass + except (serial.SerialException, OSError, TimeoutError) as error: + raise Unavailable(str(error)) + self.device = device + return self.device + + def call(self, action): + """Runs action(device) under the lock; drops the connection on USB errors.""" + with self.lock: + try: + return action(self._connected()) + except (serial.SerialException, OSError, TimeoutError) as error: + if self.device: + self.device.serial.close() + self.device = None + raise Unavailable(str(error)) + + def command(self, line: str) -> str: + def run(device): + device.send(line) + return device.read_line() + return self.call(run) + + def entropy(self, count: int) -> bytes: + def run(device): + chunks = [] + while count > sum(map(len, chunks)): + chunks.append(device.entropy(min(count - sum(map(len, chunks)), MAX_ENTROPY_REQUEST))) + return b"".join(chunks) + return self.call(run) + + def poll_gcp(self) -> None: + reply = json.loads(self.command("gcp" if self.last_second is None else f"gcp {self.last_second}")) + records = [reply] if "second" in reply else reply.get("records", []) + if not records and self.reconnected: + # A board reset restarts its second count, so nothing would ever follow last_second. + latest = json.loads(self.command("gcp")) + if latest.get("second", self.last_second) < self.last_second: + self.last_second = None + self.reconnected = False + if not records: + return + newest = records[-1]["second"] + now = time.time() + # The newest record ended at most one poll interval ago; re-anchor after a device restart. + if self.offset is None or abs(now - (self.offset + newest)) > 2: + self.offset = math.floor(now) - newest + for record in records: + record = {"time": self.offset + record["second"] - 1, **record} + self.records.append(record) + if self.log: + self.log.write(json.dumps(record) + "\n") + self.log.flush() + self.last_second = newest + self.reconnected = False + + def run_poller(self) -> None: + while True: + try: + self.poll_gcp() + time.sleep(GCP_POLL_S) + except Unavailable: + self.reconnected = self.last_second is not None + time.sleep(RECONNECT_S) + except (ValueError, KeyError): + time.sleep(GCP_POLL_S) # no second completed yet, or a garbled line + + def gcp(self, since: str) -> dict: + records = list(self.records) + if not records: + return {"error": "no second completed yet"} + if not since: + return records[-1] + after = int(since) + return {"records": [r for r in records if r["second"] > after]} + + +class Handler(BaseHTTPRequestHandler): + strix: Strix + set_token: str | None = None + + def may_set(self) -> bool: + if self.set_token is None: + return True + given = self.headers.get("Authorization", "") + return hmac.compare_digest(given.encode(), f"Bearer {self.set_token}".encode()) + + def send(self, status: int, body: bytes, kind: str = "application/json") -> None: + self.send_response(status) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(body) + + def send_json(self, value, status: int = 200) -> None: + self.send(status, (json.dumps(value) + "\n").encode()) + + def do_GET(self) -> None: + url = urlparse(self.path) + query = {key: values[0] for key, values in parse_qs(url.query).items()} + try: + if url.path == "/entropy" or (url.path == "/" and "length" in query): + self.serve_entropy(query.get("bytes", query.get("length", "")), query.get("format", "hex")) + elif url.path in PASSTHROUGH: + self.send(200, (self.strix.command(PASSTHROUGH[url.path]) + "\n").encode()) + elif url.path == "/set" and not self.may_set(): + self.send_json({"error": "unauthorized"}, 401) + elif url.path == "/set": + line = f"set {query.get('key', '')} {query.get('value', '')}" + self.send(200, (self.strix.command(line) + "\n").encode()) + elif url.path == "/gcp": + self.send_json(self.strix.gcp(query.get("since", ""))) + elif url.path == "/openapi.json": + self.send(200, OPENAPI) + elif url.path == "/docs": + self.send(200, SWAGGER_UI, "text/html; charset=utf-8") + else: + self.send_json({"error": "not found"}, 404) + except Unavailable as error: + self.send_json({"error": f"device unavailable: {error}"}, 503) + except ValueError: + self.send_json({"error": "invalid parameter"}) + + def serve_entropy(self, count_text: str, form: str) -> None: + count = int(count_text) + if not 1 <= count <= MAX_HTTP_ENTROPY or form not in ("hex", "bin"): + self.send_json({"error": f"usage: bytes=<1-{MAX_HTTP_ENTROPY}>&format=hex|bin"}) + return + try: + data = self.strix.entropy(count) + except RuntimeError as error: # the device's error reply, e.g. an unhealthy source + self.send(200, (str(error) + "\n").encode()) + return + if form == "bin": + self.send(200, data, "application/octet-stream") + else: + self.send(200, (data.hex().upper() + "\n").encode(), "text/plain") + + def log_message(self, format, *args) -> None: + pass + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--listen", default="127.0.0.1:8080", help="address:port (default 127.0.0.1:8080)") + parser.add_argument("--port", help="serial port (default: the STRIX board, or STRIX_PORT)") + parser.add_argument("--gcp-log", help="append every per-second trial to this JSON-lines file") + parser.add_argument("--set-token-file", help="require this file's token as a bearer token for /set") + args = parser.parse_args() + + if args.set_token_file: + with open(args.set_token_file) as token: + Handler.set_token = token.read().strip() + if not Handler.set_token: + parser.error("the set token file is empty") + + Handler.strix = Strix(args.port, args.gcp_log) + threading.Thread(target=Handler.strix.run_poller, daemon=True).start() + host, port = args.listen.rsplit(":", 1) + server = ThreadingHTTPServer((host, int(port)), Handler) + print(f"serving on http://{host}:{port}", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main()