feat: check every subscription in the background and add the posts page

app/lib/application/controllers/browser_controller.dart

@@ -277,6 +277,9 @@ class BrowserController {
         currentPage.value = page;
         error.value = null;
         visitedUrls.value = {...visitedUrls.value, url};
+        // Opening a post is reading it. Nothing happens when the address is
+        // not one, so this needs no knowledge of the reader's subscriptions.
+        await _db.markEntryRead(url);
         if (!notModified) {
           await _db.saveCachedPage(CachedPagesCompanion.insert(
             url: url,

app/lib/application/controllers/subscriptions_controller.dart

@@ -1,18 +1,34 @@
+import 'package:actors/actors.dart';
+import 'package:drift/drift.dart' show Value;
 import 'package:signals/signals.dart';
 
 import '../../database/database.dart';
 import '../../mews/document.dart';
+import '../actors/fetch_actor.dart';
 
 /// Which sites the reader subscribes to (spec 6). The posts themselves live in
 /// the database, so an unread post survives a restart; only the set of
 /// addresses is held here, because that is what the chrome watches.
 class SubscriptionsController {
-  SubscriptionsController(this._db);
+  SubscriptionsController({
+    required MewsDatabase db,
+    required Actor<FetchCommand, FetchResult> fetch,
+  })  : _db = db,
+        _fetch = fetch;
 
   final MewsDatabase _db;
+  final Actor<FetchCommand, FetchResult> _fetch;
 
   final subscribedUrls = signal(<String>{});
 
+  /// Whether a check is running, so a page can say so instead of looking idle.
+  final refreshing = signal(false);
+
+  /// Called after a check changes something a page would show. Set by the
+  /// dependency container to the navigator's rerender; holding the navigator
+  /// directly would make the wiring circular.
+  void Function()? onChanged;
+
   Future<void> load() async {
     final rows = await _db.allSubscriptions();
     subscribedUrls.value = rows.map((row) => row.url).toSet();
@@ -47,4 +63,85 @@ class SubscriptionsController {
     await _db.touchSubscription(page.url, DateTime.now());
     return fresh;
   }
+
+  /// Checks every subscribed site for posts. Sites are checked one after
+  /// another on purpose: the fetch actor answers concurrent sends, so a
+  /// parallel pass would open one request per site at once against small
+  /// servers, for a wait nobody is sitting through.
+  Future<void> refreshAll() async {
+    if (refreshing.value) return;
+    refreshing.value = true;
+    onChanged?.call();
+    try {
+      for (final site in await _db.allSubscriptions()) {
+        await _check(site.url);
+      }
+    } finally {
+      refreshing.value = false;
+      onChanged?.call();
+    }
+  }
+
+  /// Checks one site, on the reader's request.
+  Future<void> refreshOne(String siteUrl) async {
+    if (refreshing.value || !isSubscribed(siteUrl)) return;
+    refreshing.value = true;
+    onChanged?.call();
+    try {
+      await _check(siteUrl);
+    } finally {
+      refreshing.value = false;
+      onChanged?.call();
+    }
+  }
+
+  Future<void> markRead(String url, {String? siteUrl}) async {
+    await _db.markEntryRead(url, siteUrl: siteUrl);
+    onChanged?.call();
+  }
+
+  Future<void> markUnread(String url, {String? siteUrl}) async {
+    await _db.markEntryUnread(url, siteUrl: siteUrl);
+    onChanged?.call();
+  }
+
+  /// Fetches one site and stores whatever it publishes. A site that cannot be
+  /// reached is still recorded as checked and keeps the posts already held, so
+  /// one site being down never stops the rest of the pass.
+  Future<void> _check(String siteUrl) async {
+    try {
+      final cached = await _db.cachedPage(siteUrl);
+      final result = await _fetch.send(FetchPage(
+        url: siteUrl,
+        etag: cached?.etag,
+        lastModified: cached?.lastModified,
+        cachedSource: cached?.source,
+      ));
+      if (result case FetchPageOk(
+        :final page,
+        :final source,
+        :final etag,
+        :final lastModified,
+        :final notModified,
+      )) {
+        await _db.describeSubscription(siteUrl,
+            title: page.title, atomHref: page.atomFeedHref);
+        await _db.addEntries(siteUrl, page.entries);
+        if (!notModified) {
+          // Keep the cache current so the site also reads offline, but do not
+          // record a visit: the reader did not open it.
+          await _db.saveCachedPage(CachedPagesCompanion.insert(
+            url: siteUrl,
+            source: source,
+            title: Value(page.title),
+            etag: Value(etag),
+            lastModified: Value(lastModified),
+          ));
+        }
+      }
+    } catch (_) {
+      // Nothing to tell the reader: the site's "last checked" date says it.
+    }
+    await _db.touchSubscription(siteUrl, DateTime.now());
+  }
 }

app/lib/application/internal_actions.dart

@@ -1,5 +1,6 @@
 import 'controllers/reader_settings_controller.dart'
     show ReaderSettingsController;
+import 'controllers/subscriptions_controller.dart';
 import 'pages/mews_urls.dart';
 
 /// Applies the change an action address encodes, e.g. mews:do/set-theme.
@@ -10,10 +11,14 @@ import 'pages/mews_urls.dart';
 /// only links carrying these addresses are the ones the client builds, so a
 /// miss is a bug rather than something to explain to the reader.
 class InternalActions {
-  InternalActions({required ReaderSettingsController settings})
-      : _settings = settings;
+  InternalActions({
+    required ReaderSettingsController settings,
+    required SubscriptionsController subscriptions,
+  })  : _settings = settings,
+        _subscriptions = subscriptions;
 
   final ReaderSettingsController _settings;
+  final SubscriptionsController _subscriptions;
 
   Future<bool> apply(String url) async {
     final verb = MewsUrls.verbOf(url);
@@ -24,6 +29,10 @@ class InternalActions {
       'set-size' => _setSize(args['step']),
       'set-theme' => _settings.setThemeMode(args['value'] ?? ''),
       'set-colors' => _settings.setVariant(args['value'] ?? ''),
+      'mark-read' => _markRead(args, read: true),
+      'mark-unread' => _markRead(args, read: false),
+      'refresh-subscriptions' => _refreshAll(),
+      'refresh-subscription' => _refreshOne(args['url']),
       _ => false,
     };
   }
@@ -33,4 +42,28 @@ class InternalActions {
     'down' => _settings.decreaseSize(),
     _ => Future.value(false),
   };
+
+  Future<bool> _markRead(Map<String, String> args, {required bool read}) async {
+    final url = args['url'];
+    if (url == null || url.isEmpty) return false;
+    final site = args['site'];
+    if (read) {
+      await _subscriptions.markRead(url, siteUrl: site);
+    } else {
+      await _subscriptions.markUnread(url, siteUrl: site);
+    }
+    return true;
+  }
+
+  /// The check redraws the page itself as it runs, so the caller need not.
+  Future<bool> _refreshAll() async {
+    await _subscriptions.refreshAll();
+    return false;
+  }
+
+  Future<bool> _refreshOne(String? siteUrl) async {
+    if (siteUrl == null || siteUrl.isEmpty) return false;
+    await _subscriptions.refreshOne(siteUrl);
+    return false;
+  }
 }

app/lib/application/internal_pages.dart

@@ -3,10 +3,12 @@ import '../l10n/strings.dart';
 import '../mews/document.dart';
 import 'controllers/reader_settings_controller.dart'
     show ReaderSettingsController;
+import 'controllers/subscriptions_controller.dart';
 import 'pages/bookmarks_page.dart';
 import 'pages/history_page.dart';
 import 'pages/hub_page.dart';
 import 'pages/mews_urls.dart';
+import 'pages/posts_page.dart';
 import 'pages/settings_page.dart';
 import 'pages/subscriptions_page.dart';
 
@@ -17,13 +19,16 @@ class InternalPages {
   InternalPages({
     required MewsDatabase db,
     required ReaderSettingsController settings,
+    required SubscriptionsController subscriptions,
     required Strings strings,
   })  : _db = db,
         _settings = settings,
+        _subscriptions = subscriptions,
         _strings = strings;
 
   final MewsDatabase _db;
   final ReaderSettingsController _settings;
+  final SubscriptionsController _subscriptions;
   final Strings _strings;
 
   /// Whether [url] is one of the internal pages.
@@ -34,7 +39,12 @@ class InternalPages {
     // Internal addresses are opaque, so the page name is the path, not a host.
     final uri = Uri.parse(MewsUrls.canonical(url));
     return switch (uri.path) {
-      'hub' => buildHubPage(uri, db: _db, strings: _strings),
+      'hub' => buildHubPage(uri,
+          db: _db, subscriptions: _subscriptions, strings: _strings),
+      'posts' => buildPostsPage(uri,
+          db: _db,
+          strings: _strings,
+          refreshing: _subscriptions.refreshing.value),
       'bookmarks' => buildBookmarksPage(uri, db: _db, strings: _strings),
       'subscriptions' => buildSubscriptionsPage(uri, db: _db, strings: _strings),
       'history' => buildHistoryPage(uri, db: _db, strings: _strings),

app/lib/application/pages/hub_page.dart

@@ -1,6 +1,7 @@
 import '../../database/database.dart';
 import '../../l10n/strings.dart';
 import '../../mews/document.dart';
+import '../controllers/subscriptions_controller.dart';
 import 'mews_urls.dart';
 import 'page_blocks.dart';
 
@@ -10,11 +11,13 @@ import 'page_blocks.dart';
 Future<MewsPage> buildHubPage(
   Uri url, {
   required MewsDatabase db,
+  required SubscriptionsController subscriptions,
   required Strings strings,
 }) async {
   final bookmarks = await db.countBookmarks();
   final sites = await db.countSubscriptions();
   final visits = await db.countHistory();
+  final unread = await db.unreadCount();
 
   return internalPage(
     url: url,
@@ -22,6 +25,14 @@ Future<MewsPage> buildHubPage(
     body: [
       HeadingBlock(1, [MewsText(strings.t('hub-title'))]),
       itemList([
+        itemRow(
+          href: MewsUrls.posts,
+          label: strings.t('nav-posts'),
+          note: countText(strings, unread,
+              none: 'count-unread-none',
+              one: 'count-unread-one',
+              many: 'count-unread-many'),
+        ),
         itemRow(
           href: MewsUrls.bookmarks,
           label: strings.t('nav-bookmarks'),
@@ -48,6 +59,14 @@ Future<MewsPage> buildHubPage(
         ),
         itemRow(href: MewsUrls.settings, label: strings.t('nav-settings')),
       ]),
+      ParagraphBlock([
+        if (subscriptions.refreshing.value)
+          MewsEmphasis(EmphasisStyle.emphasis,
+              [MewsText(strings.t('posts-checking'))])
+        else
+          MewsLink(MewsUrls.action('refresh-subscriptions'),
+              [MewsText(strings.t('action-refresh'))]),
+      ]),
     ],
   );
 }

app/lib/application/pages/mews_urls.dart

@@ -13,6 +13,7 @@ abstract final class MewsUrls {
   static const scheme = 'mews';
 
   static const hub = 'mews:hub';
+  static const posts = 'mews:posts';
   static const bookmarks = 'mews:bookmarks';
   static const subscriptions = 'mews:subscriptions';
   static const history = 'mews:history';

app/lib/application/pages/page_blocks.dart

@@ -11,9 +11,14 @@ import 'mews_urls.dart';
 /// constant here rather than a message.
 const actionSeparator = '  ';
 
-/// One choice in a group of mutually exclusive values.
+/// A link with its label: a choice in a group, or an action on a row. For an
+/// action [current] is always false.
 typedef Choice = ({String href, String label, bool current});
 
+/// An action link on a row.
+Choice action(String href, String label) =>
+    (href: href, label: label, current: false);
+
 /// One entry in the strip of internal pages.
 typedef NavItem = ({String url, String labelId});
 
@@ -21,6 +26,7 @@ typedef NavItem = ({String url, String labelId});
 /// never shifts under the reader.
 const navItems = <NavItem>[
   (url: MewsUrls.hub, labelId: 'nav-hub'),
+  (url: MewsUrls.posts, labelId: 'nav-posts'),
   (url: MewsUrls.bookmarks, labelId: 'nav-bookmarks'),
   (url: MewsUrls.subscriptions, labelId: 'nav-subscriptions'),
   (url: MewsUrls.history, labelId: 'nav-history'),
@@ -61,18 +67,59 @@ MewsListItem itemRow({
   required String label,
   String? date,
   String? note,
+  bool strong = false,
+  List<Choice> actions = const [],
 }) {
+  final text = date == null ? label : '$date $label';
   return MewsListItem([
     ParagraphBlock([
-      MewsLink(href, [MewsText(date == null ? label : '$date $label')]),
+      MewsLink(href, [
+        if (strong)
+          MewsEmphasis(EmphasisStyle.strong, [MewsText(text)])
+        else
+          MewsText(text),
+      ]),
       if (note != null) ...[
         const MewsLineBreak(),
         MewsEmphasis(EmphasisStyle.emphasis, [MewsText(note)]),
       ],
+      if (actions.isNotEmpty) ...[
+        const MewsLineBreak(),
+        ..._separated(
+            [for (final action in actions) MewsLink(action.href, [MewsText(action.label)])]),
+      ],
     ]),
   ]);
 }
 
+/// Joins runs with the separator between them, never around them.
+List<MewsInline> _separated(List<MewsInline> runs) {
+  final out = <MewsInline>[];
+  for (final run in runs) {
+    if (out.isNotEmpty) out.add(const MewsText(actionSeparator));
+    out.add(run);
+  }
+  return out;
+}
+
+/// A group of mutually exclusive values on one line, for a short group that
+/// would waste a list. The chosen value is plain text, the rest are links.
+List<MewsBlock> choiceLine({required String label, required List<Choice> options}) {
+  return [
+    ParagraphBlock([
+      MewsEmphasis(EmphasisStyle.emphasis, [MewsText(label)]),
+      const MewsText(actionSeparator),
+      ..._separated([
+        for (final option in options)
+          if (option.current)
+            MewsEmphasis(EmphasisStyle.strong, [MewsText(option.label)])
+          else
+            MewsLink(option.href, [MewsText(option.label)]),
+      ]),
+    ]),
+  ];
+}
+
 /// A group of mutually exclusive values: the current one plain, the rest links.
 ListBlock choiceRow(List<Choice> options) {
   return ListBlock(

app/lib/application/pages/posts_page.dart

@@ -0,0 +1,104 @@
+import '../../database/database.dart';
+import '../../l10n/strings.dart';
+import '../../mews/document.dart';
+import 'mews_urls.dart';
+import 'page_blocks.dart';
+
+/// Every post from every subscribed site in one list (spec 6.1), newest first
+/// unless the reader orders it otherwise. Unread posts stand out, and opening
+/// one marks it read.
+Future<MewsPage> buildPostsPage(
+  Uri url, {
+  required MewsDatabase db,
+  required Strings strings,
+  required bool refreshing,
+}) async {
+  final sort = EntrySort.parse(url.queryParameters['sort']);
+  final unreadOnly = url.queryParameters['show'] == 'unread';
+  final entries =
+      await db.riverEntries(sort: sort, unreadOnly: unreadOnly);
+  final unread = await db.unreadCount();
+  final title = strings.t('posts-title');
+
+  return internalPage(
+    url: url,
+    title: title,
+    body: [
+      ...chrome(title: title, current: url, strings: strings),
+      ParagraphBlock([
+        MewsEmphasis(EmphasisStyle.emphasis, [
+          MewsText(countText(strings, unread,
+              none: 'count-unread-none',
+              one: 'count-unread-one',
+              many: 'count-unread-many')),
+        ]),
+        const MewsText(actionSeparator),
+        if (refreshing)
+          MewsEmphasis(EmphasisStyle.emphasis,
+              [MewsText(strings.t('posts-checking'))])
+        else
+          MewsLink(MewsUrls.action('refresh-subscriptions'),
+              [MewsText(strings.t('action-refresh'))]),
+      ]),
+      ...choiceLine(
+        label: strings.t('posts-sort-label'),
+        options: [
+          for (final option in const [
+            (EntrySort.date, 'posts-sort-date'),
+            (EntrySort.source, 'posts-sort-source'),
+            (EntrySort.unread, 'posts-sort-unread'),
+          ])
+            (
+              href: MewsUrls.page(url, {'sort': option.$1.name}),
+              label: strings.t(option.$2),
+              current: option.$1 == sort,
+            ),
+        ],
+      ),
+      ...choiceLine(
+        label: strings.t('posts-show-label'),
+        options: [
+          (
+            href: MewsUrls.page(url, {'show': null}),
+            label: strings.t('posts-show-all'),
+            current: !unreadOnly,
+          ),
+          (
+            href: MewsUrls.page(url, {'show': 'unread'}),
+            label: strings.t('posts-show-unread'),
+            current: unreadOnly,
+          ),
+        ],
+      ),
+      const RuleBlock(),
+      if (entries.isEmpty)
+        ParagraphBlock([
+          MewsText(strings
+              .t(unreadOnly ? 'posts-none-unread' : 'posts-empty')),
+        ])
+      else
+        itemList([
+          for (final entry in entries)
+            itemRow(
+              href: entry.url,
+              label: entry.title,
+              date: dayText(entry.date),
+              note: entry.siteName,
+              strong: entry.unread,
+              actions: [
+                if (entry.unread)
+                  action(
+                      MewsUrls.action('mark-read',
+                          {'site': entry.siteUrl, 'url': entry.url}),
+                      strings.t('posts-mark-read'))
+                else
+                  action(
+                      MewsUrls.action('mark-unread',
+                          {'site': entry.siteUrl, 'url': entry.url}),
+                      strings.t('posts-mark-unread')),
+              ],
+            ),
+        ]),
+    ],
+  );
+}

app/lib/database/database.dart

@@ -286,6 +286,51 @@ class MewsDatabase extends _$MewsDatabase {
     return statement.write(const FeedEntriesCompanion(readAt: Value(null)));
   }
 
+  /// Every stored post with the site it came from, for the posts page.
+  Future<List<RiverEntry>> riverEntries({
+    EntrySort sort = EntrySort.date,
+    bool unreadOnly = false,
+    int limit = 200,
+  }) async {
+    final site = alias(subscriptions, 'site');
+    final query = select(feedEntries).join([
+      leftOuterJoin(site, site.url.equalsExp(feedEntries.siteUrl)),
+    ]);
+    if (unreadOnly) query.where(feedEntries.readAt.isNull());
+
+    // A site without a title sorts under its address, so the order never
+    // depends on a null.
+    final sourceName = coalesce([site.title, feedEntries.siteUrl]);
+    query.orderBy(switch (sort) {
+      EntrySort.date => [
+          OrderingTerm.desc(feedEntries.date),
+          OrderingTerm.asc(feedEntries.url),
+        ],
+      EntrySort.source => [
+          OrderingTerm.asc(sourceName),
+          OrderingTerm.desc(feedEntries.date),
+        ],
+      EntrySort.unread => [
+          // Unread first, whichever way the database orders nulls.
+          OrderingTerm.desc(feedEntries.readAt.isNull()),
+          OrderingTerm.desc(feedEntries.date),
+        ],
+    });
+    query.limit(limit);
+
+    return (await query.get()).map((row) {
+      final entry = row.readTable(feedEntries);
+      return RiverEntry(
+        siteUrl: entry.siteUrl,
+        siteTitle: row.readTableOrNull(site)?.title,
+        url: entry.url,
+        title: entry.title,
+        date: entry.date,
+        readAt: entry.readAt,
+      );
+    }).toList();
+  }
+
   /// How many posts are unread, for one site or across all of them.
   Future<int> unreadCount({String? siteUrl}) async {
     final total = feedEntries.url.count();
@@ -371,3 +416,43 @@ class MewsDatabase extends _$MewsDatabase {
   Future<void> saveCachedPage(CachedPagesCompanion entry) =>
       into(cachedPages).insert(entry, mode: InsertMode.insertOrReplace);
 }
+
+/// How the posts page is ordered.
+enum EntrySort {
+  date,
+  source,
+  unread;
+
+  /// Reads the order from an address, falling back to the default rather than
+  /// refusing a value the reader may have typed.
+  static EntrySort parse(String? name) => switch (name) {
+    'source' => EntrySort.source,
+    'unread' => EntrySort.unread,
+    _ => EntrySort.date,
+  };
+}
+
+/// One post together with the site that published it.
+class RiverEntry {
+  const RiverEntry({
+    required this.siteUrl,
+    required this.siteTitle,
+    required this.url,
+    required this.title,
+    required this.date,
+    required this.readAt,
+  });
+
+  final String siteUrl;
+  final String? siteTitle;
+  final String url;
+  final String title;
+  final DateTime date;
+  final DateTime? readAt;
+
+  bool get unread => readAt == null;
+
+  /// The site as the reader knows it: its title, or its address until one is
+  /// recorded.
+  String get siteName => siteTitle ?? siteUrl;
+}

app/lib/presentation/app.dart

@@ -1,3 +1,5 @@
+import 'dart:async' show unawaited;
+
 import 'package:actors/actors.dart';
 import 'package:flutter/material.dart';
 import 'package:signals/signals_flutter.dart' show SignalBuilder;
@@ -27,16 +29,23 @@ class AppDependencies {
         strings = strings ?? Strings() {
     settings = ReaderSettingsController(db);
     bookmarks = BookmarksController(db);
-    subscriptions = SubscriptionsController(db);
-    internalPages =
-        InternalPages(db: db, settings: settings, strings: this.strings);
-    internalActions = InternalActions(settings: settings);
+    subscriptions = SubscriptionsController(db: db, fetch: this.fetch);
+    internalPages = InternalPages(
+        db: db,
+        settings: settings,
+        subscriptions: subscriptions,
+        strings: this.strings);
+    internalActions =
+        InternalActions(settings: settings, subscriptions: subscriptions);
     browser = BrowserController(
       db: db,
       fetch: this.fetch,
       internalPages: internalPages,
       internalActions: internalActions,
     );
+    // A check running in the background redraws whichever of our pages is
+    // open, so a new post shows up without the reader navigating.
+    subscriptions.onChanged = browser.refreshInternalPage;
   }
 
   /// Localized strings, also used to title the internal pages.
@@ -56,6 +65,9 @@ class AppDependencies {
     await bookmarks.load();
     await subscriptions.load();
     await browser.restoreLastSession();
+    // Subscriptions are checked in the background: the reader never waits on
+    // the network to see their last page.
+    unawaited(subscriptions.refreshAll());
   }
 }