From fa51d5793d561cf5a589623f8c35aa8f61700759 Mon Sep 17 00:00:00 2001 From: randogoth Date: Sun, 11 Oct 2026 13:35:19 +0300 Subject: [PATCH] feat: read subscriptions from an Atom feed when a page lists no dated links --- SPEC.md | 1 + app/lib/application/actors/fetch_actor.dart | 40 +++++++ .../controllers/subscriptions_controller.dart | 26 +++- app/lib/mews/atom.dart | 85 +++++++++++++ app/lib/mews/parser.dart | 14 ++- app/test/atom_test.dart | 113 ++++++++++++++++++ app/test/controllers_test.dart | 85 +++++++++++++ app/test/fake_fetch.dart | 11 +- app/test/parser_test.dart | 40 +++++++ 9 files changed, 411 insertions(+), 4 deletions(-) create mode 100644 app/lib/mews/atom.dart create mode 100644 app/test/atom_test.dart diff --git a/SPEC.md b/SPEC.md index dc1280c..557f370 100644 --- a/SPEC.md +++ b/SPEC.md @@ -67,6 +67,7 @@ The v1 client presents one reader screen: - A swipe right goes back and a swipe left goes forward through the history stack. The whole empty margin beside the text column accepts the swipe, with a 40 px floor, so the gesture stays clear of selecting text. Touch devices navigate this way; pointer platforms get the address bar's buttons as well. - The client's own pages live under the `mews:` scheme. `mews:hub` names them and says what each one holds; `mews:posts`, `mews:bookmarks`, `mews:subscriptions`, `mews:history` and `mews:settings` each carry a strip linking to the others. Bare `mews:` is the hub, and the hub is where a session with no history to return to starts. Each page is built as an ordinary Mews document and rendered by the same renderer as a fetched page, so it joins the history stack and the swipe transitions, and none of them is recorded in history or offered for subscription. - `mews:posts` is every post from every subscribed site in one list. The reader can order it by date, by site, or unread first, and limit it to unread posts; those choices ride in the address as a query string, so going back undoes one and a swipe preview shows the page the reader will land on. Unread posts are shown bold, and opening a post marks it read. +- A site that publishes through an Atom feed instead of dated links is read through the feed (spec 6.2). Dated links are the normative mechanism, so the feed is only asked for when the page itself listed nothing, and a feed on another site is refused. Posts are keyed by address, so a post listed both ways is stored once, under the title the page gave it. - Subscribed sites are checked once in the background at launch, and whenever the reader asks. Sites are checked one after another rather than at once, and a site that cannot be reached keeps the posts already held and is still recorded as checked, so one site being down never stops the rest. - The reader can tag bookmarks and sites. A tag exists only while something carries it, so adding a tag and making one are the same act; `mews:tags` lists every tag in use and what carries it, and following a tag narrows the list it belongs to. There is no renaming and no empty tag. - Each library page carries a search box that narrows it by title or address, and each row carries its own actions: remove a bookmark, leave a site or check it now, forget a page. History lists one row per address, headed by the most recent visit, rather than repeating a page once per visit. Clearing history takes a second step on a page the reader can simply leave, since it cannot be undone. diff --git a/app/lib/application/actors/fetch_actor.dart b/app/lib/application/actors/fetch_actor.dart index 5a54abb..0f27581 100644 --- a/app/lib/application/actors/fetch_actor.dart +++ b/app/lib/application/actors/fetch_actor.dart @@ -1,6 +1,7 @@ import 'package:actors/actors.dart'; import 'package:http/http.dart' as http; +import '../../mews/atom.dart'; import '../../mews/document.dart'; import '../../mews/parser.dart'; @@ -31,6 +32,14 @@ class SubmitForm extends FetchCommand { final Map fields; } +/// Fetch and read a subscribed page's Atom feed (spec 6.2). [url] is absolute +/// and on the page's own site, which the parser has already checked. +class FetchFeed extends FetchCommand { + const FetchFeed({required this.url}); + + final String url; +} + /// Parses a page's source without touching the network; used to prepare the /// neighboring pages that swipes reveal. class ParseSource extends FetchCommand { @@ -44,6 +53,14 @@ sealed class FetchResult { const FetchResult(); } +/// The posts an Atom feed lists. +class FetchFeedOk extends FetchResult { + const FetchFeedOk({required this.url, required this.entries}); + + final String url; + final List entries; +} + class FetchPageOk extends FetchResult { const FetchPageOk({ required this.page, @@ -103,6 +120,7 @@ class FetchHandler with Handler { await _fetchPage(url, etag, lastModified, cachedSource), SubmitForm(:final url, :final method, :final fields) => await _submitForm(url, method, fields), + FetchFeed(:final url) => await _fetchFeed(url), ParseSource(:final url, :final source) => _parse(source, url), }; } @@ -149,6 +167,28 @@ class FetchHandler with Handler { } } + Future _fetchFeed(String url) async { + final http.Response response; + try { + response = await _client + .get(Uri.parse(url), headers: {'User-Agent': _userAgent}) + .timeout(const Duration(seconds: 20)); + } catch (_) { + return const FetchNetworkError(); + } + if (response.statusCode != 200) { + return FetchHttpStatusError(response.statusCode); + } + try { + return FetchFeedOk( + url: url, + entries: parseAtomEntries(source: response.body, feedUrl: url), + ); + } on MewsParseException catch (e) { + return FetchParseError(e.message, response.body); + } + } + Future _submitForm(String url, String method, Map fields) async { final headers = {'User-Agent': _userAgent}; final http.Response response; diff --git a/app/lib/application/controllers/subscriptions_controller.dart b/app/lib/application/controllers/subscriptions_controller.dart index d1c43d9..ff29cfb 100644 --- a/app/lib/application/controllers/subscriptions_controller.dart +++ b/app/lib/application/controllers/subscriptions_controller.dart @@ -43,9 +43,14 @@ class SubscriptionsController { } await _db.addSubscription(page.url, page.title, page.atomFeedHref); subscribedUrls.value = {...subscribedUrls.value, page.url}; - // The posts already on the page are the site's back catalogue, not news, - // so they are stored read and only later posts are flagged. + // Whatever the site has already published is its back catalogue, not + // news, so it is stored read and only later posts are flagged. A site + // that publishes only through a feed is seeded the same way. await _db.addEntries(page.url, page.entries, read: true); + final feed = page.atomFeedHref; + if (page.entries.isEmpty && feed != null) { + await _checkFeed(page.url, feed, read: true); + } } Future remove(String siteUrl) async { @@ -127,6 +132,13 @@ class SubscriptionsController { await _db.describeSubscription(siteUrl, title: page.title, atomHref: page.atomFeedHref); await _db.addEntries(siteUrl, page.entries); + // Dated links are the normative way a site publishes (spec 6.1), so + // the feed is only read when the page itself listed nothing. A site + // where both agree is then fetched once, not twice. + final feed = page.atomFeedHref; + if (page.entries.isEmpty && feed != null) { + await _checkFeed(siteUrl, feed); + } if (!notModified) { // Keep the cache current so the site also reads offline, but do not // record a visit: the reader did not open it. @@ -144,4 +156,14 @@ class SubscriptionsController { } await _db.touchSubscription(siteUrl, DateTime.now()); } + + /// Reads a site's Atom feed (spec 6.2). Entries are keyed by address, so a + /// post listed by both the page and the feed is stored once. + Future _checkFeed(String siteUrl, String feedUrl, + {bool read = false}) async { + final result = await _fetch.send(FetchFeed(url: feedUrl)); + if (result case FetchFeedOk(:final entries)) { + await _db.addEntries(siteUrl, entries, read: read); + } + } } diff --git a/app/lib/mews/atom.dart b/app/lib/mews/atom.dart new file mode 100644 index 0000000..c766c44 --- /dev/null +++ b/app/lib/mews/atom.dart @@ -0,0 +1,85 @@ +/// Reads an Atom feed (spec 6.2) into the same shape dated links produce, so a +/// site's posts come from one store however they were published. Pure data: no +/// Flutter imports, so it can run inside the fetch actor's isolate. +library; + +import 'package:xml/xml.dart'; + +import 'document.dart'; +import 'parser.dart' show MewsParseException; + +/// The entries of an Atom feed, newest first. Entries without a usable address +/// or date are skipped rather than guessed at. +/// +/// Element names are matched without their namespace, so a feed using the +/// default Atom namespace and one using an `atom:` prefix both read the same. +List parseAtomEntries({ + required String source, + required String feedUrl, +}) { + final XmlDocument doc; + try { + doc = XmlDocument.parse(source); + } on XmlException catch (e) { + throw MewsParseException(e.toString()); + } + + final base = Uri.tryParse(feedUrl); + final entries = []; + for (final element in doc.rootElement.children.whereType()) { + if (element.name.local != 'entry') continue; + final url = _entryUrl(element, base); + final date = _entryDate(element); + if (url == null || date == null) continue; + entries.add(MewsEntry( + url: url, + date: date, + title: _childText(element, 'title') ?? url, + )); + } + + entries.sort((a, b) => b.date.compareTo(a.date)); + return entries; +} + +/// An entry's address: its alternate link, or its id when that is a web +/// address. The id is a fallback because many feeds use the post's URL for it. +String? _entryUrl(XmlElement entry, Uri? base) { + for (final child in entry.children.whereType()) { + if (child.name.local != 'link') continue; + final rel = child.getAttribute('rel'); + if (rel != null && rel != 'alternate') continue; + final href = child.getAttribute('href'); + if (href == null || href.isEmpty) continue; + final resolved = base?.resolve(href) ?? Uri.tryParse(href); + if (resolved != null) return resolved.toString(); + } + + final id = _childText(entry, 'id'); + if (id == null) return null; + final asUri = Uri.tryParse(id); + return asUri != null && (asUri.scheme == 'http' || asUri.scheme == 'https') + ? asUri.toString() + : null; +} + +/// When an entry was published. `updated` is required by Atom, so it comes +/// first, with `published` as the fallback. +DateTime? _entryDate(XmlElement entry) { + for (final name in const ['updated', 'published']) { + final text = _childText(entry, name); + if (text == null) continue; + final date = DateTime.tryParse(text); + if (date != null) return date; + } + return null; +} + +String? _childText(XmlElement parent, String name) { + for (final child in parent.children.whereType()) { + if (child.name.local != name) continue; + final text = child.innerText.trim(); + if (text.isNotEmpty) return text; + } + return null; +} diff --git a/app/lib/mews/parser.dart b/app/lib/mews/parser.dart index 9863dcc..496ac26 100644 --- a/app/lib/mews/parser.dart +++ b/app/lib/mews/parser.dart @@ -51,7 +51,7 @@ MewsPage parseMewsPage({required String source, required String url}) { title: headData.title, description: headData.description, author: headData.author, - atomFeedHref: headData.atomFeedHref, + atomFeedHref: _feedHref(headData.atomFeedHref, url), lang: html.getAttribute('xml:lang') ?? html.getAttribute('lang') ?? bodyElement.getAttribute('lang') ?? @@ -294,6 +294,18 @@ ImageBlock _imageBlock(XmlElement element, String pageUrl) { /// Rough registrable-domain match: the last two labels. Fine for the small /// web; a public-suffix list is out of scope for v1. +/// The page's Atom feed as an absolute address (spec 6.2), or null when the +/// page declares none or points at another site: a client must not load +/// another site's resources without the reader acting (specs 7.4 and 8.1). +String? _feedHref(String? href, String pageUrl) { + if (href == null || href.isEmpty) return null; + final page = Uri.tryParse(pageUrl); + if (page == null) return null; + final resolved = page.resolve(href); + if (resolved.scheme != 'http' && resolved.scheme != 'https') return null; + return _sameSite(resolved, page) ? resolved.toString() : null; +} + bool _sameSite(Uri image, Uri page) { String? domain(Uri uri) { final host = uri.host; diff --git a/app/test/atom_test.dart b/app/test/atom_test.dart new file mode 100644 index 0000000..fbefd2b --- /dev/null +++ b/app/test/atom_test.dart @@ -0,0 +1,113 @@ +import 'package:flutter_test/flutter_test.dart'; + +import 'package:mews/mews/atom.dart'; +import 'package:mews/mews/parser.dart'; + +const defaultNamespaced = ''' + + + Signal Bars + + Collecting retro phones + + tag:example.com,2026:retro + 2026-10-08T09:30:00Z + + + First post + + 2026-09-21T12:00:00Z + + +'''; + +const prefixed = ''' + + + + Reading the bars + + 2026-10-09T08:00:00Z + + +'''; + +void main() { + test('reads entries from a feed in the default namespace', () { + final entries = parseAtomEntries( + source: defaultNamespaced, feedUrl: 'https://example.com/feed.xml'); + + expect(entries, hasLength(2)); + // Newest first, like the dated links a page lists. + expect(entries.first.title, 'Collecting retro phones'); + expect(entries.first.date, DateTime.utc(2026, 10, 8, 9, 30)); + // Relative links resolve against the feed's own address. + expect(entries.first.url, 'https://example.com/2026-10-08-retro.html'); + // A link without a rel is the alternate one. + expect(entries.last.url, 'https://example.com/2026-09-21-first.html'); + // published stands in when updated is absent. + expect(entries.last.date, DateTime.utc(2026, 9, 21, 12)); + }); + + test('reads a feed using an atom prefix', () { + final entries = + parseAtomEntries(source: prefixed, feedUrl: 'https://example.com/feed.xml'); + expect(entries.single.title, 'Reading the bars'); + expect(entries.single.url, 'https://example.com/bars.html'); + }); + + test('falls back to the id when it is a web address', () { + const source = ''' + + + No link here + https://example.com/no-link.html + 2026-10-08T09:30:00Z + + +'''; + final entries = + parseAtomEntries(source: source, feedUrl: 'https://example.com/feed.xml'); + expect(entries.single.url, 'https://example.com/no-link.html'); + }); + + test('skips entries with nothing to identify or date them', () { + const source = ''' + + + No date + + + + No address + tag:example.com,2026:nope + 2026-10-08T09:30:00Z + + + Unreadable date + + last Tuesday + + +'''; + expect( + parseAtomEntries(source: source, feedUrl: 'https://example.com/feed.xml'), + isEmpty); + }); + + test('a malformed feed is an error, not a guess', () { + expect( + () => parseAtomEntries( + source: 'Half', feedUrl: 'https://example.com/feed.xml'), + throwsA(isA<MewsParseException>()), + ); + }); + + test('an empty feed reads as no entries', () { + expect( + parseAtomEntries( + source: '<feed xmlns="http://www.w3.org/2005/Atom"></feed>', + feedUrl: 'https://example.com/feed.xml'), + isEmpty); + }); +} diff --git a/app/test/controllers_test.dart b/app/test/controllers_test.dart index 7e6da65..8999ec5 100644 --- a/app/test/controllers_test.dart +++ b/app/test/controllers_test.dart @@ -7,6 +7,7 @@ import 'package:mews/application/controllers/reader_settings_controller.dart'; import 'package:mews/application/controllers/subscriptions_controller.dart'; import 'package:mews/database/database.dart'; import 'package:mews/mews/document.dart'; +import 'package:mews/mews/parser.dart'; import 'fake_fetch.dart'; @@ -207,6 +208,90 @@ void main() { expect(redraws, 2); }); + test('a site publishing only through a feed still reports posts', () async { + const site = 'https://feedonly.example/'; + const feed = 'https://feedonly.example/feed.xml'; + // A page with no dated links, but it declares a feed (spec 6.2). + const page = ''' +<?xml version="1.0" encoding="UTF-8"?> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <title>Feed Only + + + +

Feed Only

Nothing listed here.

+ +'''; + String feedWith(List<(String, String)> entries) => ''' + +${entries.map((e) => '${e.$2}2026-10-08T09:30:00Z').join()} + +'''; + + final feeds = {feed: feedWith(const [('old.html', 'An older post')])}; + final subscriptions = SubscriptionsController( + db: db, + fetch: FakeFetch({site: page}, feeds: feeds), + ); + await subscriptions.load(); + await subscriptions.toggle(parseMewsPage(source: page, url: site)); + + // What the feed already listed is the back catalogue, not news. + expect(await db.unreadCount(), 0); + expect((await db.allSubscriptions()).single.atomHref, feed); + + // A post added to the feed later is new. + feeds[feed] = feedWith(const [ + ('new.html', 'A newer post'), + ('old.html', 'An older post'), + ]); + await subscriptions.refreshAll(); + expect(await db.unreadCount(), 1); + final entries = await db.riverEntries(unreadOnly: true); + expect(entries.single.title, 'A newer post'); + + // Checking again reports nothing, because an address identifies a post. + await subscriptions.refreshAll(); + expect(await db.unreadCount(), 1); + }); + + test('a page listing dated links is not fetched twice', () async { + const site = 'https://both.example/'; + const page = ''' + + + + Both + + + +

Both

+ +'''; + // The feed would answer, but a page that lists its own posts is enough, + // so nothing asks for it. + final subscriptions = SubscriptionsController( + db: db, + fetch: FakeFetch({site: page}, feeds: const { + 'https://both.example/feed.xml': + 'Duplicate' + '2026-10-08T09:30:00Z' + '', + }), + ); + await subscriptions.load(); + await subscriptions.toggle(parseMewsPage(source: page, url: site)); + await subscriptions.refreshAll(); + + // One post, under the title the page gave it. + final entries = await db.riverEntries(); + expect(entries, hasLength(1)); + expect(entries.single.title, 'The only post'); + }); + test('caches pages for offline reopening', () async { await db.saveCachedPage(CachedPagesCompanion.insert( url: 'https://example.com/page.html', diff --git a/app/test/fake_fetch.dart b/app/test/fake_fetch.dart index ba7c1c6..26429de 100644 --- a/app/test/fake_fetch.dart +++ b/app/test/fake_fetch.dart @@ -1,6 +1,7 @@ import 'package:actors/actors.dart'; import 'package:mews/application/actors/fetch_actor.dart'; +import 'package:mews/mews/atom.dart'; import 'package:mews/mews/parser.dart'; /// Serves pages from an in-memory map without network or isolates, so @@ -9,10 +10,13 @@ import 'package:mews/mews/parser.dart'; /// what a site publishes between two fetches. An address with no source is /// unreachable, and the cached copy stands in for it if there is one. class FakeFetch implements Actor { - FakeFetch(this.sources); + FakeFetch(this.sources, {this.feeds = const {}}); final Map sources; + /// Atom feeds by address, for sites that publish through one. + final Map feeds; + @override Future send(FetchCommand message) async { return switch (message) { @@ -25,6 +29,11 @@ class FakeFetch implements Actor { (null, final String cached) => _parse(url, cached), _ => const FetchNetworkError(), }, + FetchFeed(:final url) => switch (feeds[url]) { + final String source => + FetchFeedOk(url: url, entries: parseAtomEntries(source: source, feedUrl: url)), + null => const FetchNetworkError(), + }, _ => const FetchNetworkError(), }; } diff --git a/app/test/parser_test.dart b/app/test/parser_test.dart index 630ab8e..0d3bc8c 100644 --- a/app/test/parser_test.dart +++ b/app/test/parser_test.dart @@ -240,4 +240,44 @@ void main() { expect(quoteText, hasLength(3)); expect((quoteText[0] as MewsText).text, 'Quoted words '); }); + + group('the atom feed link', () { + String pageWith(String link) => ''' + + + + Signal Bars + + $link + +

Signal Bars

+ +'''; + + test('resolves against the page, so a relative href is usable', () { + final page = parseMewsPage( + source: pageWith( + ''), + url: 'https://example.com/posts/index.html', + ); + expect(page.atomFeedHref, 'https://example.com/posts/feed.xml'); + }); + + test('is refused when it points at another site', () { + // A client must not load another site's resources on its own + // (specs 7.4 and 8.1). + final page = parseMewsPage( + source: pageWith( + ''), + url: 'https://example.com/index.html', + ); + expect(page.atomFeedHref, isNull); + }); + + test('is null when the page declares none', () { + final page = parseMewsPage( + source: pageWith(''), url: 'https://example.com/index.html'); + expect(page.atomFeedHref, isNull); + }); + }); }