randogoth/the-mews
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat: read subscriptions from an Atom feed when a page lists no dated links

Browse source
This commit is contained in:
randogoth 2026-10-11 13:35:19 +03:00
parent b0b86ef5b1
commit fa51d5793d
9 changed files with 411 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

40
app/lib/application/actors/fetch_actor.dart
View file

@ -1,6 +1,7 @@
import 'package:actors/actors.dart';
import 'package:http/http.dart' as http;
import '../../mews/atom.dart';
import '../../mews/document.dart';
import '../../mews/parser.dart';
@ -31,6 +32,14 @@ class SubmitForm extends FetchCommand {
final Map<String, String> fields;
}
/// Fetch and read a subscribed page's Atom feed (spec 6.2). [url] is absolute
/// and on the page's own site, which the parser has already checked.
class FetchFeed extends FetchCommand {
const FetchFeed({required this.url});
final String url;
}
/// Parses a page's source without touching the network; used to prepare the
/// neighboring pages that swipes reveal.
class ParseSource extends FetchCommand {
@ -44,6 +53,14 @@ sealed class FetchResult {
const FetchResult();
}
/// The posts an Atom feed lists.
class FetchFeedOk extends FetchResult {
const FetchFeedOk({required this.url, required this.entries});
final String url;
final List<MewsEntry> entries;
}
class FetchPageOk extends FetchResult {
const FetchPageOk({
required this.page,
@ -103,6 +120,7 @@ class FetchHandler with Handler<FetchCommand, FetchResult> {
await _fetchPage(url, etag, lastModified, cachedSource),
SubmitForm(:final url, :final method, :final fields) =>
await _submitForm(url, method, fields),
FetchFeed(:final url) => await _fetchFeed(url),
ParseSource(:final url, :final source) => _parse(source, url),
};
}
@ -149,6 +167,28 @@ class FetchHandler with Handler<FetchCommand, FetchResult> {
}
}
Future<FetchResult> _fetchFeed(String url) async {
final http.Response response;
try {
response = await _client
.get(Uri.parse(url), headers: {'User-Agent': _userAgent})
.timeout(const Duration(seconds: 20));
} catch (_) {
return const FetchNetworkError();
}
if (response.statusCode != 200) {
return FetchHttpStatusError(response.statusCode);
}
try {
return FetchFeedOk(
url: url,
entries: parseAtomEntries(source: response.body, feedUrl: url),
);
} on MewsParseException catch (e) {
return FetchParseError(e.message, response.body);
}
}
Future<FetchResult> _submitForm(String url, String method, Map<String, String> fields) async {
final headers = {'User-Agent': _userAgent};
final http.Response response;

26
app/lib/application/controllers/subscriptions_controller.dart
View file

@ -43,9 +43,14 @@ class SubscriptionsController {
}
await _db.addSubscription(page.url, page.title, page.atomFeedHref);
subscribedUrls.value = {...subscribedUrls.value, page.url};
// The posts already on the page are the site's back catalogue, not news,
// so they are stored read and only later posts are flagged.
// Whatever the site has already published is its back catalogue, not
// news, so it is stored read and only later posts are flagged. A site
// that publishes only through a feed is seeded the same way.
await _db.addEntries(page.url, page.entries, read: true);
final feed = page.atomFeedHref;
if (page.entries.isEmpty && feed != null) {
await _checkFeed(page.url, feed, read: true);
}
}
Future<void> remove(String siteUrl) async {
@ -127,6 +132,13 @@ class SubscriptionsController {
await _db.describeSubscription(siteUrl,
title: page.title, atomHref: page.atomFeedHref);
await _db.addEntries(siteUrl, page.entries);
// Dated links are the normative way a site publishes (spec 6.1), so
// the feed is only read when the page itself listed nothing. A site
// where both agree is then fetched once, not twice.
final feed = page.atomFeedHref;
if (page.entries.isEmpty && feed != null) {
await _checkFeed(siteUrl, feed);
}
if (!notModified) {
// Keep the cache current so the site also reads offline, but do not
// record a visit: the reader did not open it.
@ -144,4 +156,14 @@ class SubscriptionsController {
}
await _db.touchSubscription(siteUrl, DateTime.now());
}
/// Reads a site's Atom feed (spec 6.2). Entries are keyed by address, so a
/// post listed by both the page and the feed is stored once.
Future<void> _checkFeed(String siteUrl, String feedUrl,
{bool read = false}) async {
final result = await _fetch.send(FetchFeed(url: feedUrl));
if (result case FetchFeedOk(:final entries)) {
await _db.addEntries(siteUrl, entries, read: read);
}
}
}

85
app/lib/mews/atom.dart Normal file
View file

@ -0,0 +1,85 @@
/// Reads an Atom feed (spec 6.2) into the same shape dated links produce, so a
/// site's posts come from one store however they were published. Pure data: no
/// Flutter imports, so it can run inside the fetch actor's isolate.
library;
import 'package:xml/xml.dart';
import 'document.dart';
import 'parser.dart' show MewsParseException;
/// The entries of an Atom feed, newest first. Entries without a usable address
/// or date are skipped rather than guessed at.
///
/// Element names are matched without their namespace, so a feed using the
/// default Atom namespace and one using an `atom:` prefix both read the same.
List<MewsEntry> parseAtomEntries({
required String source,
required String feedUrl,
}) {
final XmlDocument doc;
try {
doc = XmlDocument.parse(source);
} on XmlException catch (e) {
throw MewsParseException(e.toString());
}
final base = Uri.tryParse(feedUrl);
final entries = <MewsEntry>[];
for (final element in doc.rootElement.children.whereType<XmlElement>()) {
if (element.name.local != 'entry') continue;
final url = _entryUrl(element, base);
final date = _entryDate(element);
if (url == null || date == null) continue;
entries.add(MewsEntry(
url: url,
date: date,
title: _childText(element, 'title') ?? url,
));
}
entries.sort((a, b) => b.date.compareTo(a.date));
return entries;
}
/// An entry's address: its alternate link, or its id when that is a web
/// address. The id is a fallback because many feeds use the post's URL for it.
String? _entryUrl(XmlElement entry, Uri? base) {
for (final child in entry.children.whereType<XmlElement>()) {
if (child.name.local != 'link') continue;
final rel = child.getAttribute('rel');
if (rel != null && rel != 'alternate') continue;
final href = child.getAttribute('href');
if (href == null || href.isEmpty) continue;
final resolved = base?.resolve(href) ?? Uri.tryParse(href);
if (resolved != null) return resolved.toString();
}
final id = _childText(entry, 'id');
if (id == null) return null;
final asUri = Uri.tryParse(id);
return asUri != null && (asUri.scheme == 'http' || asUri.scheme == 'https')
? asUri.toString()
: null;
}
/// When an entry was published. `updated` is required by Atom, so it comes
/// first, with `published` as the fallback.
DateTime? _entryDate(XmlElement entry) {
for (final name in const ['updated', 'published']) {
final text = _childText(entry, name);
if (text == null) continue;
final date = DateTime.tryParse(text);
if (date != null) return date;
}
return null;
}
String? _childText(XmlElement parent, String name) {
for (final child in parent.children.whereType<XmlElement>()) {
if (child.name.local != name) continue;
final text = child.innerText.trim();
if (text.isNotEmpty) return text;
}
return null;
}

14
app/lib/mews/parser.dart
View file

@ -51,7 +51,7 @@ MewsPage parseMewsPage({required String source, required String url}) {
title: headData.title,
description: headData.description,
author: headData.author,
atomFeedHref: headData.atomFeedHref,
atomFeedHref: _feedHref(headData.atomFeedHref, url),
lang: html.getAttribute('xml:lang') ??
html.getAttribute('lang') ??
bodyElement.getAttribute('lang') ??
@ -294,6 +294,18 @@ ImageBlock _imageBlock(XmlElement element, String pageUrl) {
/// Rough registrable-domain match: the last two labels. Fine for the small
/// web; a public-suffix list is out of scope for v1.
/// The page's Atom feed as an absolute address (spec 6.2), or null when the
/// page declares none or points at another site: a client must not load
/// another site's resources without the reader acting (specs 7.4 and 8.1).
String? _feedHref(String? href, String pageUrl) {
if (href == null || href.isEmpty) return null;
final page = Uri.tryParse(pageUrl);
if (page == null) return null;
final resolved = page.resolve(href);
if (resolved.scheme != 'http' && resolved.scheme != 'https') return null;
return _sameSite(resolved, page) ? resolved.toString() : null;
}
bool _sameSite(Uri image, Uri page) {
String? domain(Uri uri) {
final host = uri.host;

113
app/test/atom_test.dart Normal file
View file

@ -0,0 +1,113 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:mews/mews/atom.dart';
import 'package:mews/mews/parser.dart';
const defaultNamespaced = '''
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Signal Bars</title>
<entry>
<title>Collecting retro phones</title>
<link rel="alternate" href="2026-10-08-retro.html" />
<id>tag:example.com,2026:retro</id>
<updated>2026-10-08T09:30:00Z</updated>
</entry>
<entry>
<title>First post</title>
<link href="2026-09-21-first.html" />
<published>2026-09-21T12:00:00Z</published>
</entry>
</feed>
''';
const prefixed = '''
<?xml version="1.0" encoding="utf-8"?>
<atom:feed xmlns:atom="http://www.w3.org/2005/Atom">
<atom:entry>
<atom:title>Reading the bars</atom:title>
<atom:link rel="alternate" href="https://example.com/bars.html" />
<atom:updated>2026-10-09T08:00:00Z</atom:updated>
</atom:entry>
</atom:feed>
''';
void main() {
test('reads entries from a feed in the default namespace', () {
final entries = parseAtomEntries(
source: defaultNamespaced, feedUrl: 'https://example.com/feed.xml');
expect(entries, hasLength(2));
// Newest first, like the dated links a page lists.
expect(entries.first.title, 'Collecting retro phones');
expect(entries.first.date, DateTime.utc(2026, 10, 8, 9, 30));
// Relative links resolve against the feed's own address.
expect(entries.first.url, 'https://example.com/2026-10-08-retro.html');
// A link without a rel is the alternate one.
expect(entries.last.url, 'https://example.com/2026-09-21-first.html');
// published stands in when updated is absent.
expect(entries.last.date, DateTime.utc(2026, 9, 21, 12));
});
test('reads a feed using an atom prefix', () {
final entries =
parseAtomEntries(source: prefixed, feedUrl: 'https://example.com/feed.xml');
expect(entries.single.title, 'Reading the bars');
expect(entries.single.url, 'https://example.com/bars.html');
});
test('falls back to the id when it is a web address', () {
const source = '''
<feed xmlns="http://www.w3.org/2005/Atom">
<entry>
<title>No link here</title>
<id>https://example.com/no-link.html</id>
<updated>2026-10-08T09:30:00Z</updated>
</entry>
</feed>
''';
final entries =
parseAtomEntries(source: source, feedUrl: 'https://example.com/feed.xml');
expect(entries.single.url, 'https://example.com/no-link.html');
});
test('skips entries with nothing to identify or date them', () {
const source = '''
<feed xmlns="http://www.w3.org/2005/Atom">
<entry>
<title>No date</title>
<link rel="alternate" href="a.html" />
</entry>
<entry>
<title>No address</title>
<id>tag:example.com,2026:nope</id>
<updated>2026-10-08T09:30:00Z</updated>
</entry>
<entry>
<title>Unreadable date</title>
<link rel="alternate" href="b.html" />
<updated>last Tuesday</updated>
</entry>
</feed>
''';
expect(
parseAtomEntries(source: source, feedUrl: 'https://example.com/feed.xml'),
isEmpty);
});
test('a malformed feed is an error, not a guess', () {
expect(
() => parseAtomEntries(
source: '<feed><entry><title>Half', feedUrl: 'https://example.com/feed.xml'),
throwsA(isA<MewsParseException>()),
);
});
test('an empty feed reads as no entries', () {
expect(
parseAtomEntries(
source: '<feed xmlns="http://www.w3.org/2005/Atom"></feed>',
feedUrl: 'https://example.com/feed.xml'),
isEmpty);
});
}

85
app/test/controllers_test.dart
View file

@ -7,6 +7,7 @@ import 'package:mews/application/controllers/reader_settings_controller.dart';
import 'package:mews/application/controllers/subscriptions_controller.dart';
import 'package:mews/database/database.dart';
import 'package:mews/mews/document.dart';
import 'package:mews/mews/parser.dart';
import 'fake_fetch.dart';
@ -207,6 +208,90 @@ void main() {
expect(redraws, 2);
});
test('a site publishing only through a feed still reports posts', () async {
const site = 'https://feedonly.example/';
const feed = 'https://feedonly.example/feed.xml';
// A page with no dated links, but it declares a feed (spec 6.2).
const page = '''
<?xml version="1.0" encoding="UTF-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Feed Only</title>
<meta name="mews-profile" content="0.1" />
<link rel="alternate" type="application/atom+xml" href="feed.xml" />
</head>
<body><h1>Feed Only</h1><p>Nothing listed here.</p></body>
</html>
''';
String feedWith(List<(String, String)> entries) => '''
<feed xmlns="http://www.w3.org/2005/Atom">
${entries.map((e) => '<entry><title>${e.$2}</title><link rel="alternate" href="${e.$1}" /><updated>2026-10-08T09:30:00Z</updated></entry>').join()}
</feed>
''';
final feeds = {feed: feedWith(const [('old.html', 'An older post')])};
final subscriptions = SubscriptionsController(
db: db,
fetch: FakeFetch({site: page}, feeds: feeds),
);
await subscriptions.load();
await subscriptions.toggle(parseMewsPage(source: page, url: site));
// What the feed already listed is the back catalogue, not news.
expect(await db.unreadCount(), 0);
expect((await db.allSubscriptions()).single.atomHref, feed);
// A post added to the feed later is new.
feeds[feed] = feedWith(const [
('new.html', 'A newer post'),
('old.html', 'An older post'),
]);
await subscriptions.refreshAll();
expect(await db.unreadCount(), 1);
final entries = await db.riverEntries(unreadOnly: true);
expect(entries.single.title, 'A newer post');
// Checking again reports nothing, because an address identifies a post.
await subscriptions.refreshAll();
expect(await db.unreadCount(), 1);
});
test('a page listing dated links is not fetched twice', () async {
const site = 'https://both.example/';
const page = '''
<?xml version="1.0" encoding="UTF-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Both</title>
<meta name="mews-profile" content="0.1" />
<link rel="alternate" type="application/atom+xml" href="feed.xml" />
</head>
<body><h1>Both</h1><ul>
<li><a href="one.html">2026-10-08 The only post</a></li>
</ul></body>
</html>
''';
// The feed would answer, but a page that lists its own posts is enough,
// so nothing asks for it.
final subscriptions = SubscriptionsController(
db: db,
fetch: FakeFetch({site: page}, feeds: const {
'https://both.example/feed.xml':
'<feed xmlns="http://www.w3.org/2005/Atom"><entry><title>Duplicate</title>'
'<link rel="alternate" href="one.html" /><updated>2026-10-08T09:30:00Z</updated>'
'</entry></feed>',
}),
);
await subscriptions.load();
await subscriptions.toggle(parseMewsPage(source: page, url: site));
await subscriptions.refreshAll();
// One post, under the title the page gave it.
final entries = await db.riverEntries();
expect(entries, hasLength(1));
expect(entries.single.title, 'The only post');
});
test('caches pages for offline reopening', () async {
await db.saveCachedPage(CachedPagesCompanion.insert(
url: 'https://example.com/page.html',

11
app/test/fake_fetch.dart
View file

@ -1,6 +1,7 @@
import 'package:actors/actors.dart';
import 'package:mews/application/actors/fetch_actor.dart';
import 'package:mews/mews/atom.dart';
import 'package:mews/mews/parser.dart';
/// Serves pages from an in-memory map without network or isolates, so
@ -9,10 +10,13 @@ import 'package:mews/mews/parser.dart';
/// what a site publishes between two fetches. An address with no source is
/// unreachable, and the cached copy stands in for it if there is one.
class FakeFetch implements Actor<FetchCommand, FetchResult> {
FakeFetch(this.sources);
FakeFetch(this.sources, {this.feeds = const {}});
final Map<String, String> sources;
/// Atom feeds by address, for sites that publish through one.
final Map<String, String> feeds;
@override
Future<FetchResult> send(FetchCommand message) async {
return switch (message) {
@ -25,6 +29,11 @@ class FakeFetch implements Actor<FetchCommand, FetchResult> {
(null, final String cached) => _parse(url, cached),
_ => const FetchNetworkError(),
},
FetchFeed(:final url) => switch (feeds[url]) {
final String source =>
FetchFeedOk(url: url, entries: parseAtomEntries(source: source, feedUrl: url)),
null => const FetchNetworkError(),
},
_ => const FetchNetworkError(),
};
}

40
app/test/parser_test.dart
View file

@ -240,4 +240,44 @@ void main() {
expect(quoteText, hasLength(3));
expect((quoteText[0] as MewsText).text, 'Quoted words ');
});
group('the atom feed link', () {
String pageWith(String link) => '''
<?xml version="1.0" encoding="UTF-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Signal Bars</title>
<meta name="mews-profile" content="0.1" />
$link
</head>
<body><h1>Signal Bars</h1></body>
</html>
''';
test('resolves against the page, so a relative href is usable', () {
final page = parseMewsPage(
source: pageWith(
'<link rel="alternate" type="application/atom+xml" href="feed.xml" />'),
url: 'https://example.com/posts/index.html',
);
expect(page.atomFeedHref, 'https://example.com/posts/feed.xml');
});
test('is refused when it points at another site', () {
// A client must not load another site's resources on its own
// (specs 7.4 and 8.1).
final page = parseMewsPage(
source: pageWith(
'<link rel="alternate" type="application/atom+xml" href="https://elsewhere.test/feed.xml" />'),
url: 'https://example.com/index.html',
);
expect(page.atomFeedHref, isNull);
});
test('is null when the page declares none', () {
final page = parseMewsPage(
source: pageWith(''), url: 'https://example.com/index.html');
expect(page.atomFeedHref, isNull);
});
});
}