feat: offer a non-Mews web page as a simplified view or in the browser

app/test/app_widget_test.dart

@@ -8,7 +8,7 @@ import 'package:flutter/rendering.dart' show RenderParagraph;
 import 'package:flutter_test/flutter_test.dart';
 
 import 'package:mews/application/controllers/browser_controller.dart'
-    show BrokenPage, LoadFailed;
+    show LoadFailed;
 import 'package:mews/database/database.dart';
 import 'package:mews/l10n/strings.dart';
 import 'package:mews/mews/document.dart';
@@ -116,16 +116,67 @@ void main() {
     expect(find.textContaining('Sign guestbook'), findsOneWidget);
   });
 
-  testWidgets('shows the broken-page error and its text view', (tester) async {
-    await pumpApp(tester);
-    deps.browser.currentPage.value = null;
-    deps.browser.error.value = const BrokenPage('<p>Half a page &amp; more');
+  testWidgets('offers a web page simplified or in the browser', (tester) async {
+    // Ordinary HTML: unclosed tags, text loose in a div, a script to ignore.
+    // Nothing here is a markup error, it is simply not a Mews page.
+    const webPage = '''
+<!DOCTYPE html>
+<html><head><title>A Journal</title></head>
+<body>
+  <script>ignore.me()</script>
+  <div>Loose text in a div.
+  <h2>A heading</h2>
+  <ul><li>First<li>Second</ul>
+  <p>Never closed
+</body></html>
+''';
+    final stubDeps = AppDependencies(
+      database: MewsDatabase.test(NativeDatabase.memory()),
+      fetch: FakeFetch(const {'https://example.com/journal': webPage}),
+      strings: strings,
+    );
+    await tester.pumpWidget(MewsApp(deps: stubDeps, strings: strings));
+    await tester.pumpAndSettle();
+
+    await stubDeps.browser.open('https://example.com/journal');
+    await tester.pumpAndSettle();
+
+    expect(find.textContaining('not a Mews page'), findsOneWidget);
+    expect(find.textContaining('markup error'), findsNothing);
+    expect(find.textContaining('Open in a web browser'), findsOneWidget);
+
+    await tester.tap(find.textContaining('Show a simplified version'));
+    await tester.pumpAndSettle();
+
+    // The structure survives, and what Mews cannot show is left out.
+    expect(find.textContaining('A heading'), findsOneWidget);
+    expect(find.textContaining('Loose text in a div'), findsOneWidget);
+    expect(find.textContaining('Never closed'), findsOneWidget);
+    expect(find.textContaining('Second'), findsWidgets);
+    expect(find.textContaining('ignore.me'), findsNothing);
+    expect(find.textContaining('simplified version'), findsWidgets);
+  });
+
+  testWidgets('calls a page that claims Mews and will not parse broken',
+      (tester) async {
+    const claimed = '''
+<html><head><title>Claimed</title>
+<meta name="mews-profile" content="0.1"></head>
+<body><p>Never closed</body></html>
+''';
+    final stubDeps = AppDependencies(
+      database: MewsDatabase.test(NativeDatabase.memory()),
+      fetch: FakeFetch(const {'https://example.com/claimed': claimed}),
+      strings: strings,
+    );
+    await tester.pumpWidget(MewsApp(deps: stubDeps, strings: strings));
+    await tester.pumpAndSettle();
+
+    await stubDeps.browser.open('https://example.com/claimed');
     await tester.pumpAndSettle();
 
     expect(find.textContaining('markup error'), findsOneWidget);
-    await tester.tap(find.textContaining('Show page text'));
-    await tester.pumpAndSettle();
-    expect(find.textContaining('Half a page & more'), findsOneWidget);
+    expect(find.textContaining('not a Mews page'), findsNothing);
   });
 
   testWidgets('shows the load-failed error page', (tester) async {

app/test/fake_fetch.dart

@@ -5,6 +5,7 @@ import 'package:actors/actors.dart';
 import 'package:mews/application/actors/fetch_actor.dart';
 import 'package:mews/mews/atom.dart';
 import 'package:mews/mews/gemtext.dart';
+import 'package:mews/mews/html.dart';
 import 'package:mews/mews/parser.dart';
 import 'package:mews/protocols/tofu.dart';
 
@@ -103,7 +104,9 @@ class FakeFetch implements Actor<FetchCommand, FetchResult> {
     try {
       return FetchPageOk(page: parseMewsPage(source: source, url: url), source: source);
     } on MewsParseException catch (e) {
-      return FetchParseError(e.message, source);
+      final converted = convertHtmlPage(url: url, source: source);
+      return FetchParseError(e.message,
+          simplified: converted.page, declaresMews: converted.declaresMews);
     }
   }
 

app/test/fetch_actor_test.dart

@@ -35,15 +35,35 @@ void main() {
     expect(ok.notModified, isFalse);
   });
 
-  test('ParseSource reports broken markup with its source', () async {
+  test('ParseSource offers a web page as a simplified page', () async {
     final actor = Actor.create(FetchHandler.new);
     addTearDown(actor.close);
 
-    final result = await actor.send(
-        const ParseSource(url: 'https://example.com/broken.html', source: '<p>nope'));
+    final result = await actor.send(const ParseSource(
+        url: 'https://example.com/broken.html',
+        source: '<h1>Loose</h1><p>Never closed'));
 
     expect(result, isA<FetchParseError>());
-    expect((result as FetchParseError).source, '<p>nope');
+    final error = result as FetchParseError;
+    // No conformance marker, so this is an ordinary web page rather than a
+    // Mews page with a markup error.
+    expect(error.declaresMews, isFalse);
+    expect(error.simplified, isNotNull);
+    expect(error.simplified!.body.whereType<HeadingBlock>(), isNotEmpty);
+  });
+
+  test('ParseSource reports a markup error on a page claiming Mews', () async {
+    final actor = Actor.create(FetchHandler.new);
+    addTearDown(actor.close);
+
+    final result = await actor.send(const ParseSource(
+        url: 'https://example.com/broken.html',
+        source: '<html><head><title>T</title>'
+            '<meta name="mews-profile" content="0.1"></head>'
+            '<body><p>Never closed</body></html>'));
+
+    expect(result, isA<FetchParseError>());
+    expect((result as FetchParseError).declaresMews, isTrue);
   });
 
   // The protocol transports are exercised through the same handler a fetch

app/test/html_test.dart

@@ -0,0 +1,265 @@
+import 'dart:convert';
+
+import 'package:flutter_test/flutter_test.dart';
+
+import 'package:mews/mews/document.dart';
+import 'package:mews/mews/html.dart';
+
+/// Ordinary web markup: a doctype, unclosed tags, text loose inside wrappers,
+/// and plenty the Mews subset has nothing to draw for.
+const webPage = '''
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>A Journal</title>
+  <style>body { color: red }</style>
+  <script>ignore.me()</script>
+</head>
+<body>
+  <nav><a href="/">Home</a></nav>
+  <div class="wrap" @click="toggle()" v-bind:href="x">
+    Loose text in a div.
+    <h2>On old phones</h2>
+    <p>Notes on <em>old phones</em> and <a href="about.html">the about page</a>.
+    <ul>
+      <li>First
+      <li>Second
+    </ul>
+    <blockquote>Worth keeping.</blockquote>
+    <pre>  kept as is</pre>
+    <table><tr><th>Model<td>Price<tr><td>3310<td>Ten</table>
+    <hr>
+    <svg><circle r="4"/></svg>
+    <iframe src="https://ads.example/frame"></iframe>
+    <button>Press me</button>
+  </div>
+</body>
+</html>
+''';
+
+/// Every bit of readable text in a page, so a test can say what survived the
+/// conversion and what did not.
+String textOf(List<MewsBlock> blocks) {
+  final parts = <String>[];
+
+  void addInline(List<MewsInline> runs) {
+    for (final run in runs) {
+      switch (run) {
+        case MewsText(:final text):
+          parts.add(text);
+        case MewsEmphasis(:final children):
+          addInline(children);
+        case MewsLink(:final children):
+          addInline(children);
+        case MewsLineBreak():
+          break;
+      }
+    }
+  }
+
+  void addBlocks(List<MewsBlock> blocks) {
+    for (final block in blocks) {
+      switch (block) {
+        case HeadingBlock(:final inline):
+          addInline(inline);
+        case ParagraphBlock(:final inline):
+          addInline(inline);
+        case AddressBlock(:final inline):
+          addInline(inline);
+        case QuoteBlock(:final children):
+          addBlocks(children);
+        case PreBlock(:final text):
+          parts.add(text);
+        case ListBlock(:final items):
+          for (final item in items) {
+            addBlocks(item.blocks);
+          }
+        case DefinitionListBlock(:final items):
+          for (final item in items) {
+            addInline(item.term);
+            addInline(item.definition);
+          }
+        case TableBlock(:final rows):
+          for (final row in rows) {
+            for (final cell in row.cells) {
+              addInline(cell.inline);
+            }
+          }
+        case ImageBlock(:final alt):
+          parts.add(alt);
+        case RuleBlock():
+          break;
+        case FormBlock():
+          break;
+      }
+    }
+  }
+
+  addBlocks(blocks);
+  return parts.join(' ');
+}
+
+({MewsPage page, bool declaresMews}) convert(String source,
+        {String url = 'https://example.com/journal'}) =>
+    convertHtmlPage(url: url, source: source);
+
+void main() {
+  test('converts a web page into the blocks Mews supports', () {
+    final page = convert(webPage).page;
+    final kinds = page.body.map((b) => b.runtimeType.toString()).toList();
+
+    expect(page.title, 'A Journal');
+    expect(page.lang, 'en');
+    expect(kinds, contains('HeadingBlock'));
+    expect(kinds, contains('ParagraphBlock'));
+    expect(kinds, contains('ListBlock'));
+    expect(kinds, contains('QuoteBlock'));
+    expect(kinds, contains('PreBlock'));
+    expect(kinds, contains('TableBlock'));
+    expect(kinds, contains('RuleBlock'));
+  });
+
+  test('a converted page is marked as a simplified view', () {
+    expect(convert(webPage).page.specVersion, htmlSpecVersion);
+    // The marker is what makes a page Mews, and this one never claimed it.
+    expect(convert(webPage).declaresMews, isFalse);
+  });
+
+  test('a page claiming the Mews marker is told apart', () {
+    const claimed = '<html><head><title>T</title>'
+        '<meta name="mews-profile" content="0.1"></head>'
+        '<body><p>Never closed</body></html>';
+    expect(convert(claimed).declaresMews, isTrue);
+  });
+
+  test('keeps text sitting directly inside an unknown wrapper', () {
+    final page = convert(webPage).page;
+    final text = page.body
+        .whereType<ParagraphBlock>()
+        .expand((b) => b.inline.whereType<MewsText>())
+        .map((t) => t.text)
+        .join(' ');
+    expect(text, contains('Loose text in a div.'));
+  });
+
+  test('closes tags the page left open', () {
+    final page = convert('<body><p>One<p>Two<ul><li>A<li>B</ul></body>').page;
+    expect(page.body.whereType<ParagraphBlock>().length, 2);
+    expect(page.body.whereType<ListBlock>().single.items.length, 2);
+  });
+
+  test('leaves out markup that is not content', () {
+    final all = textOf(convert(webPage).page.body);
+    expect(all, isNot(contains('ignore.me')));
+    expect(all, isNot(contains('color: red')));
+    expect(all, isNot(contains('Press me')));
+    expect(all, isNot(contains('circle')));
+    // A nav is a list of real links, so it is kept rather than stripped.
+    expect(all, contains('Home'));
+    // The table's cells come through.
+    expect(all, contains('3310'));
+  });
+
+  test('resolves a relative link against the page address', () {
+    final page = convert(webPage).page;
+    final links = page.body
+        .whereType<ParagraphBlock>()
+        .expand((b) => b.inline.whereType<MewsLink>())
+        .map((l) => l.href);
+    expect(links, contains('https://example.com/about.html'));
+  });
+
+  test('an image on another site falls back to its alt text', () {
+    const source = '<body><img src="https://cdn.example/a.png" alt="A chart">'
+        '<img src="/own.png" alt="Ours"></body>';
+    final images = convert(source).page.body.whereType<ImageBlock>().toList();
+    expect(images.first.allowed, isFalse);
+    expect(images.first.alt, 'A chart');
+    expect(images.last.allowed, isTrue);
+  });
+
+  test('an attribute no XML name allows does not stop the page', () {
+    // The wrapper in webPage carries @click and v-bind:href.
+    expect(() => convert(webPage), returnsNormally);
+    expect(convert(webPage).page.body, isNotEmpty);
+  });
+
+  test('reads the charset the document declares, not the one assumed', () {
+    const html = '<html><head><meta charset="utf-8"><title>T</title></head>'
+        '<body><p>Beschädigte Seite</p></body></html>';
+    final page = convertHtmlPage(
+            url: 'https://example.com/', bytes: utf8.encode(html))
+        .page;
+    final text = page.body
+        .whereType<ParagraphBlock>()
+        .expand((b) => b.inline.whereType<MewsText>())
+        .map((t) => t.text)
+        .join();
+    expect(text, contains('Beschädigte Seite'));
+  });
+
+  test('a nested table becomes a table of its own', () {
+    // Old web layout markup: a table inside a cell of another.
+    const source = '<body><table>'
+        '<tr><th>Outer head</th><td>Outer cell</td></tr>'
+        '<tr><td>Holds one'
+        '<table><tr><td>Inner a</td><td>Inner b</td></tr></table>'
+        '</td><td>Beside it</td></tr>'
+        '</table></body>';
+    final tables = convert(source).page.body.whereType<TableBlock>().toList();
+    expect(tables.length, 2);
+
+    String cells(TableBlock t) => t.rows
+        .map((r) =>
+            r.cells.map((c) => textOf([ParagraphBlock(c.inline)])).join('|'))
+        .join(' / ');
+
+    // The outer table keeps its own two rows; the inner row is not hoisted
+    // into it, and the inner cells do not run together in the cell's text.
+    expect(tables.first.rows.length, 2);
+    expect(cells(tables.first), contains('Outer head'));
+    expect(cells(tables.first), contains('Holds one'));
+    expect(cells(tables.first), contains('Beside it'));
+    expect(cells(tables.first), isNot(contains('Inner a')));
+
+    // The inner table stands on its own, with its content intact.
+    expect(tables.last.rows.length, 1);
+    expect(cells(tables.last), contains('Inner a'));
+    expect(cells(tables.last), contains('Inner b'));
+  });
+
+  test('leaves a form out but keeps what it wrapped', () {
+    const source = '<body>'
+        '<form action="/search" method="get">'
+        '<label for="q">Search</label>'
+        '<input type="text" id="q" name="q">'
+        '<select name="in"><option>Titles</option></select>'
+        '<textarea name="notes" rows="2" cols="4">Draft</textarea>'
+        '<input type="submit" value="Go">'
+        '</form>'
+        '<form action="/wrap"><h2>Wrapped heading</h2>'
+        '<p>Real content inside a form.</p></form>'
+        '</body>';
+    final page = convert(source).page;
+    final text = textOf(page.body);
+
+    // No form survives, so nothing in the view invites an answer the reader
+    // cannot send.
+    expect(page.body.whereType<FormBlock>(), isEmpty);
+    expect(text, isNot(contains('Search')));
+    expect(text, isNot(contains('Titles')));
+    expect(text, isNot(contains('Draft')));
+    expect(text, isNot(contains('Go')));
+
+    // The form's wrapper goes, its content stays.
+    expect(page.body.whereType<HeadingBlock>(), isNotEmpty);
+    expect(text, contains('Wrapped heading'));
+    expect(text, contains('Real content inside a form.'));
+  });
+
+  test('markup with nothing in it still yields a page', () {
+    final page = convert('').page;
+    expect(page.url, 'https://example.com/journal');
+    expect(page.body, isEmpty);
+  });
+}

app/test/parser_test.dart

@@ -159,12 +159,6 @@ void main() {
     expect(submit.label, 'Sign guestbook');
   });
 
-  test('mewsPlainText recovers text from broken markup', () {
-    const broken = '<html><head><title>Half</title></head><body><p>Visible &amp; intact.</p>';
-    final text = mewsPlainText(broken);
-    expect(text, contains('Visible & intact.'));
-  });
-
   test('pretty-printed markup leaves no phantom blank content', () {
     final source = '''
 <?xml version="1.0" encoding="UTF-8"?>

app/test/reader_widget_test.dart

@@ -82,6 +82,107 @@ void main() {
     expect(opened, 'https://example.com/next.html');
   });
 
+  testWidgets('two forms with one action stay apart', (tester) async {
+    // A web page often carries its search box twice, once per header, both
+    // submitting to the same address.
+    const search = FormBlock(
+      action: 'https://example.com/w/index.php',
+      method: 'get',
+      fields: [MewsTextField(name: 'search', password: false, label: 'Search')],
+      submitLabel: 'Go',
+    );
+    final page = MewsPage(
+      url: 'https://example.com/article',
+      lang: 'en',
+      direction: TextDirection.ltr,
+      variant: MewsVariant.plain,
+      specVersion: '0.1',
+      entries: const [],
+      body: const [search, ParagraphBlock([MewsText('Between them.')]), search],
+    );
+
+    await tester.pumpWidget(
+      MaterialApp(
+        // A form's fields are Material widgets and need a sheet to sit on.
+        home: Scaffold(
+          body: StringsScope(
+            strings: await _strings(),
+            child: MewsPageView(
+              page: page,
+              palette: paletteFor(
+                  pageVariant: MewsVariant.plain, readerVariant: 'site', dark: false),
+              fontFamily: 'Atkinson Hyperlegible',
+              fontScale: 1.0,
+              visitedUrls: const {},
+              onLinkTap: (_) {},
+              onFormSubmit: (_, _) {},
+            ),
+          ),
+        ),
+      ),
+    );
+    await tester.pumpAndSettle();
+
+    expect(tester.takeException(), isNull);
+    expect(find.byType(TextField), findsNWidgets(2));
+  });
+
+  testWidgets('a table with ragged rows renders without holes', (tester) async {
+    // A web page's rows rarely match: here a header row of two, a row of
+    // three, and one spanning cell. Flutter's Table would assert on these.
+    final page = MewsPage(
+      url: 'https://example.com/table.html',
+      lang: 'en',
+      direction: TextDirection.ltr,
+      variant: MewsVariant.plain,
+      specVersion: '0.1',
+      entries: const [],
+      body: const [
+        TableBlock(
+          rows: [
+            MewsTableRow([
+              MewsTableCell(header: true, inline: [MewsText('Model')]),
+              MewsTableCell(header: true, inline: [MewsText('Price')]),
+            ]),
+            MewsTableRow([
+              MewsTableCell(header: false, inline: [MewsText('3310')]),
+              MewsTableCell(header: false, inline: [MewsText('Ten')]),
+              MewsTableCell(header: false, inline: [MewsText('Spare')]),
+            ]),
+            MewsTableRow([
+              MewsTableCell(header: false, inline: [MewsText('Only one')]),
+            ]),
+            MewsTableRow([]),
+          ],
+        ),
+      ],
+    );
+
+    await tester.pumpWidget(
+      MaterialApp(
+        home: StringsScope(
+          strings: await _strings(),
+          child: MewsPageView(
+            page: page,
+            palette: paletteFor(
+                pageVariant: MewsVariant.plain, readerVariant: 'site', dark: false),
+            fontFamily: 'Atkinson Hyperlegible',
+            fontScale: 1.0,
+            visitedUrls: const {},
+            onLinkTap: (_) {},
+            onFormSubmit: (_, _) {},
+          ),
+        ),
+      ),
+    );
+    await tester.pumpAndSettle();
+
+    expect(tester.takeException(), isNull);
+    expect(find.text('Model'), findsOneWidget);
+    expect(find.text('Spare'), findsOneWidget);
+    expect(find.text('Only one'), findsOneWidget);
+  });
+
   testWidgets('Tab walks the links on a page', (tester) async {
     final page = MewsPage(
       url: 'https://example.com/index.html',