import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; import 'package:actors/actors.dart'; import 'package:http/http.dart' as http; import '../../mews/atom.dart'; import '../../mews/document.dart'; import '../../mews/gemtext.dart'; import '../../mews/html.dart'; import '../../mews/parser.dart'; import '../../protocols/gemini.dart' as gemini; import '../../protocols/spartan.dart' as spartan; import '../../protocols/tofu.dart'; /// Commands accepted by the fetch actor. Network I/O, XML parsing and the /// protocol transports all happen in the actor's isolate; results return by /// message only. sealed class FetchCommand { const FetchCommand(); } /// Fetch [url], sending validators when the page is cached. A 304 parses /// [cachedSource] in place of a body, and a network failure falls back to it, /// so the reader stays online-transparent and works offline. /// /// [pins] are the reader's trusted hosts by `host:port`, which the Gemini /// transport checks every certificate against. The validators mean nothing to /// Gemini and Spartan: a cached protocol page is only the offline fallback. /// /// [clientCert] and [clientKey] are the PEM of the identity the reader bound /// to this host. Both are passed because the actor's isolate cannot read the /// database or the key files; the identity is worn only where the reader /// chose to wear it. class FetchPage extends FetchCommand { const FetchPage({ required this.url, this.etag, this.lastModified, this.cachedSource, this.pins = const {}, this.clientCert, this.clientKey, }); final String url; final String? etag; final String? lastModified; final String? cachedSource; final Map pins; final String? clientCert; final String? clientKey; } /// Submit a form's fields to its action URL (spec 4.2). An action on a /// protocol page carries the fields as a query string, the same way the /// protocol's own input requests do. class SubmitForm extends FetchCommand { const SubmitForm({required this.url, required this.method, required this.fields}); final String url; final String method; final Map fields; } /// Fetch and read a subscribed page's Atom feed (spec 6.2). [url] is absolute /// and on the page's own site, which the parser has already checked. class FetchFeed extends FetchCommand { const FetchFeed({required this.url}); final String url; } /// Parses a page's source without touching the network; used to prepare the /// neighboring pages that swipes reveal. class ParseSource extends FetchCommand { const ParseSource({required this.url, required this.source}); final String url; final String source; } /// Fetches the raw bytes of [url] over its protocol, for a body that is not a /// page to read: an image on a capsule. The pins and the identity travel with /// the command the way they do for a page. class FetchBytes extends FetchCommand { const FetchBytes({ required this.url, this.pins = const {}, this.clientCert, this.clientKey, }); final String url; final Map pins; final String? clientCert; final String? clientKey; } sealed class FetchResult { const FetchResult(); } /// The posts an Atom feed lists. class FetchFeedOk extends FetchResult { const FetchFeedOk({required this.url, required this.entries}); final String url; final List entries; } class FetchPageOk extends FetchResult { const FetchPageOk({ required this.page, required this.source, this.etag, this.lastModified, this.notModified = false, this.pinUpdates = const [], }); final MewsPage page; /// The markup the page was parsed from, for the cache. Empty when there is /// nothing text worth keeping, such as an image. final String source; final String? etag; final String? lastModified; /// True when the page came from the local cache (304 or offline). final bool notModified; /// Certificates the transports pinned on the way, to record on the main /// isolate: a host's first contact, or a renewal the old pin had expired /// for. final List pinUpdates; } class FetchFormOk extends FetchResult { const FetchFormOk(this.page); final MewsPage page; } class FetchNetworkError extends FetchResult { const FetchNetworkError(); } class FetchHttpStatusError extends FetchResult { const FetchHttpStatusError(this.status); final int status; } class FetchParseError extends FetchResult { const FetchParseError(this.message, {this.simplified, this.declaresMews = false}); final String message; /// The page as the app can show it, for the simplified view, or null when /// nothing legible came out of the markup. final MewsPage? simplified; /// Whether the markup declared the Mews conformance marker. A page that /// claims to be Mews and still will not parse has a real markup error; one /// that never claimed it is just an ordinary web page. final bool declaresMews; } /// A Gemini or Spartan status carrying a message for the reader: a temporary /// or permanent failure, a client or server error. class FetchProtocolError extends FetchResult { const FetchProtocolError(this.message); final String message; } /// Redirects went around more times than the transport follows. class FetchRedirectLoop extends FetchResult { const FetchRedirectLoop(); } /// A body the reader cannot be shown, of a type the client does not present. class FetchUnsupportedType extends FetchResult { const FetchUnsupportedType(this.mime); final String mime; } /// The server's certificate is not the one pinned for it, and the pin has not /// expired. The reader can still choose to trust what was presented. class FetchCertificateChanged extends FetchResult { const FetchCertificateChanged({ required this.host, required this.port, required this.presented, }); final String host; final int port; final TofuPin presented; } /// The raw body of a protocol request, for the renderer to show as an image. class FetchBytesOk extends FetchResult { const FetchBytesOk(this.data); final Uint8List data; } /// The site asked for a client certificate (Gemini 6x). [meta] says why; 61 /// means an identity was offered and refused. class FetchClientCertificateRequired extends FetchResult { const FetchClientCertificateRequired({required this.status, required this.meta}); final int status; final String meta; } class FetchHandler with Handler { // Identifies the client to servers that vary behavior by user agent, like // the http-to-https redirects some Mews sites run (spec 7.2). static const _userAgent = 'Mews/0.1 (Mews client)'; final http.Client _client = http.Client(); final gemini.GeminiClient _gemini = gemini.GeminiClient(); final spartan.SpartanClient _spartan = spartan.SpartanClient(); @override Future handle(FetchCommand message) async { return switch (message) { FetchPage( :final url, :final etag, :final lastModified, :final cachedSource, :final pins, :final clientCert, :final clientKey, ) => await _fetchPage(url, etag, lastModified, cachedSource, pins, clientCert, clientKey), SubmitForm(:final url, :final method, :final fields) => await _submitForm(url, method, fields), FetchFeed(:final url) => await _fetchFeed(url), ParseSource(:final url, :final source) => _parse(source, url), FetchBytes(:final url, :final pins, :final clientCert, :final clientKey) => await _fetchBytes(url, pins, clientCert, clientKey), }; } /// Fetches bytes over a protocol, sharing the page fetch's trust and /// identity, and answers with the body alone: a body the reader cannot show /// as a page is not a failure to explain, just an image to place. Future _fetchBytes( String url, Map pins, String? clientCert, String? clientKey) async { final uri = Uri.tryParse(url); if (uri == null) return const FetchNetworkError(); try { if (uri.scheme == 'gemini') { final response = await _gemini.fetch(uri, pins: pins, clientCert: _identity(clientCert, clientKey)); if (response.isSuccess) return FetchBytesOk(response.body ?? Uint8List(0)); return const FetchNetworkError(); } if (uri.scheme == 'spartan') { final response = await _spartan.fetch(uri); if (response.isSuccess) return FetchBytesOk(response.body ?? Uint8List(0)); return const FetchNetworkError(); } } on gemini.GeminiException { return const FetchNetworkError(); } on spartan.SpartanException { return const FetchNetworkError(); } return const FetchNetworkError(); } Future _fetchPage(String url, String? etag, String? lastModified, String? cachedSource, Map pins, [String? clientCert, String? clientKey]) async { switch (Uri.tryParse(url)?.scheme) { case 'gemini': return _fetchProtocol(url, cachedSource, pins, clientCert, clientKey); case 'spartan': return _fetchProtocol(url, cachedSource, pins, null, null); default: return _fetchWeb(url, etag, lastModified, cachedSource); } } Future _fetchWeb( String url, String? etag, String? lastModified, String? cachedSource) async { final headers = {'User-Agent': _userAgent}; if (etag != null) headers['If-None-Match'] = etag; if (lastModified != null) headers['If-Modified-Since'] = lastModified; http.Response response; try { response = await _client .get(Uri.parse(url), headers: headers) .timeout(const Duration(seconds: 20)); } catch (_) { return _fromCacheOrError(url, cachedSource, etag, lastModified); } if (response.statusCode == 304) { return _fromCacheOrError(url, cachedSource, etag, lastModified); } if (response.statusCode != 200) return FetchHttpStatusError(response.statusCode); final newEtag = response.headers['etag']; final newLastModified = response.headers['last-modified']; return _parse(response.body, url, bytes: response.bodyBytes, etag: newEtag, lastModified: newLastModified); } /// The page could not be revalidated; use the cached copy when there is one. FetchResult _fromCacheOrError( String url, String? cachedSource, String? etag, String? lastModified) { if (cachedSource == null) return const FetchNetworkError(); return _parse(cachedSource, url, etag: etag, lastModified: lastModified, notModified: true); } /// [bytes] are the undecoded body when there is one. The HTML conversion /// reads the document's own charset from them, which an already-decoded /// string has lost. FetchResult _parse(String source, String url, {List? bytes, String? etag, String? lastModified, bool notModified = false}) { try { final page = parseMewsPage(source: source, url: url); return FetchPageOk( page: page, source: source, etag: etag, lastModified: lastModified, notModified: notModified); } on MewsParseException catch (e) { return _parseFailure(e, source, bytes, url); } } /// Markup the strict parse refused is converted for the simplified view, so /// the reader is offered the structure Mews supports rather than nothing. /// Converting here keeps the work off the main isolate. FetchResult _parseFailure( MewsParseException error, String source, List? bytes, String url) { try { final converted = convertHtmlPage(url: url, source: source, bytes: bytes); return FetchParseError(error.message, simplified: converted.page, declaresMews: converted.declaresMews); } on Exception { // Nothing legible in it; the error page stands on its own. return FetchParseError(error.message); } } /// Fetches a Gemini or Spartan page. A protocol fetch has no validators, so /// the cache is only the offline fallback: the source is reparsed as /// gemtext, the format it was cached in. Future _fetchProtocol(String url, String? cachedSource, Map pins, String? clientCert, String? clientKey) async { FetchResult result; try { if (Uri.parse(url).scheme == 'gemini') { result = await _fetchGemini(Uri.parse(url), pins, identity: _identity(clientCert, clientKey)); } else { result = await _fetchSpartan(Uri.parse(url)); } } on gemini.GeminiCertificateChanged catch (e) { return FetchCertificateChanged( host: e.host, port: e.port, presented: e.presented); } on gemini.GeminiTooManyRedirects { return const FetchRedirectLoop(); } on spartan.SpartanTooManyRedirects { return const FetchRedirectLoop(); } on gemini.GeminiException { return _fromProtocolCacheOrError(url, cachedSource); } on spartan.SpartanException { return _fromProtocolCacheOrError(url, cachedSource); } // The transport may have renewed a pin even on a failed request; a pin // that never got recorded is only pinned again on the next visit, which // is the same outcome, so only successes carry updates. return result; } /// The reader's identity as the transport can wear it: a certificate and /// its key together, or nothing. One without the other is not worn at all. SecurityContext? _identity(String? certPem, String? keyPem) { if (certPem == null || keyPem == null) return null; return SecurityContext() ..useCertificateChainBytes(utf8.encode(certPem)) ..usePrivateKeyBytes(utf8.encode(keyPem)); } Future _fetchGemini(Uri url, Map pins, {SecurityContext? identity}) async { final response = await _gemini.fetch(url, pins: pins, clientCert: identity); if (response.isSuccess) { return _protocolPage(response.url, response.meta, response.body, response.pinUpdates); } if (response.isRedirect) { // A redirect the client would not follow: somewhere other than // gemini, which the reader opens as its own navigation. return FetchProtocolError(response.meta); } if (response.status == 10 || response.status == 11) { return _inputPage(url, response.status, response.meta); } if (response.wantsClientCertificate) { return FetchClientCertificateRequired(status: response.status, meta: response.meta); } return FetchProtocolError(response.meta); } Future _fetchSpartan(Uri url) async { final response = await _spartan.fetch(url); if (response.isSuccess) { return _protocolPage(response.url, response.meta.isEmpty ? 'text/gemini' : response.meta, response.body, const []); } if (response.isRedirect) { return FetchProtocolError(response.meta); } return FetchProtocolError(response.meta); } /// Builds a page from a protocol body by its MIME type: gemtext through the /// document model, other text plainly, images as an image block, and /// anything else refused rather than shown as garbage. FetchResult _protocolPage(Uri url, String meta, List? body, List pinUpdates) { final type = meta.split(';').first.trim().toLowerCase(); if (type == 'text/gemini' || type.isEmpty) { final text = _decode(body); return FetchPageOk( page: parseGemtextPage(source: text, url: url.toString()), source: text, pinUpdates: pinUpdates); } if (type.startsWith('text/')) { final text = _decode(body); return FetchPageOk( page: MewsPage( url: url.toString(), specVersion: gemtextSpecVersion, lang: 'en', direction: TextDirection.ltr, variant: MewsVariant.plain, entries: const [], body: [PreBlock(text)], ), source: text); } if (type.startsWith('image/')) { return FetchPageOk( page: MewsPage( url: url.toString(), specVersion: gemtextSpecVersion, lang: 'en', direction: TextDirection.ltr, variant: MewsVariant.plain, entries: const [], body: [ImageBlock(src: url.toString(), alt: '', allowed: true)], ), // Binary bodies are not cached: the cache holds source text. source: '', pinUpdates: pinUpdates); } return FetchUnsupportedType(type); } /// An input request (Gemini 1x) becomes a one-field form whose answer is /// sent back as the address's query string, so the reader answers within /// the page the same way a form submits. FetchResult _inputPage(Uri url, int status, String meta) { return FetchPageOk( page: MewsPage( url: url.toString(), specVersion: gemtextSpecVersion, lang: 'en', direction: TextDirection.ltr, variant: MewsVariant.plain, entries: const [], body: [ FormBlock( action: url.toString(), method: 'get', fields: [MewsTextField(name: 'query', password: status == 11, label: meta)], submitLabel: null, ), ], ), source: ''); } FetchResult _fromProtocolCacheOrError(String url, String? cachedSource) { if (cachedSource == null || cachedSource.isEmpty) return const FetchNetworkError(); return FetchPageOk( page: parseGemtextPage(source: cachedSource, url: url), source: cachedSource, notModified: true, ); } static String _decode(List? bytes) => bytes == null ? '' : utf8.decode(bytes, allowMalformed: true); Future _fetchFeed(String url) async { final http.Response response; try { response = await _client .get(Uri.parse(url), headers: {'User-Agent': _userAgent}) .timeout(const Duration(seconds: 20)); } catch (_) { return const FetchNetworkError(); } if (response.statusCode != 200) { return FetchHttpStatusError(response.statusCode); } try { return FetchFeedOk( url: url, entries: parseAtomEntries(source: response.body, feedUrl: url), ); } on MewsParseException catch (e) { return FetchParseError(e.message); } } Future _submitForm(String url, String method, Map fields) async { // A form on a protocol page answers as a query string, the protocols' // own way of carrying input. final uri = Uri.tryParse(url); if (uri != null && (uri.scheme == 'gemini' || uri.scheme == 'spartan')) { return _fetchProtocol( uri.replace(queryParameters: {...uri.queryParameters, ...fields}).toString(), null, const {}, null, null); } final headers = {'User-Agent': _userAgent}; final http.Response response; try { if (method == 'post') { response = await _client .post(Uri.parse(url), headers: headers, body: fields) .timeout(const Duration(seconds: 20)); } else { response = await _client .get(Uri.parse(url).replace(queryParameters: fields), headers: headers) .timeout(const Duration(seconds: 20)); } } catch (_) { // A refused connection, a timeout or an address the client cannot speak // are all one failed submit to the reader. return const FetchNetworkError(); } if (response.statusCode != 200) return FetchHttpStatusError(response.statusCode); return _parse(response.body, url, bytes: response.bodyBytes); } @override Future close() async => _client.close(); }