randogoth/the-mews
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat: offer a non-Mews web page as a simplified view or in the browser

Browse source
This commit is contained in:
randogoth 2026-10-12 11:13:34 +03:00
parent cea66fbb42
commit 00e42c925c
24 changed files with 863 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

128
app/lib/mews/html.dart Normal file
View file

@ -0,0 +1,128 @@
/// Converts an ordinary web page into the Mews document model, so a page the
/// app cannot show as Mews can still be read in the structure Mews supports.
///
/// Mews is a strict subset of XHTML Mobile Profile 1.2 (mews-spec section 4),
/// so a web page's elements already are Mews elements plus extras. The work is
/// therefore only to parse tolerantly — real HTML is not well-formed XML, which
/// is what [parseMewsPage] requires (spec 7.1) — and hand the result to the
/// same walk. Elements outside the subset are unwrapped and keep their content;
/// markup that is not content at all is dropped by the walk itself.
library;
import 'package:html/dom.dart' as dom;
import 'package:html/parser.dart' as html_parser;
import 'package:xml/xml.dart';
import 'document.dart';
import 'parser.dart';
/// Stands in for the conformance marker a converted page has none of, so the
/// renderer applies reader presentation and can tell a simplified view apart.
/// Mirrors how gemtext.dart marks a converted capsule.
const htmlSpecVersion = 'html';
/// The whole page is built at once by the renderer rather than lazily, so a
/// very long web page is cut off instead of stalling the first frame.
const _maxBlocks = 2000;
/// Valid XML names, minus the colon: real pages carry attributes like
/// `@click` and `v-bind:href` that [XmlElement] will not accept.
final _xmlName = RegExp(r'^[A-Za-z_][A-Za-z0-9_.\-]*$');
/// A form on the web is not reading. The reader cannot usefully answer a
/// search box or a sign-in here, and submitting one would land them on another
/// page this client cannot show as Mews. The form itself is unwrapped rather
/// than dropped, so any real content inside it survives.
const _unwrapped = {'form'};
/// A form's controls, which have nothing to say once the form is gone.
const _dropped = {
'input', 'select', 'textarea', 'label', 'fieldset', 'optgroup', 'option',
};
/// Converts [bytes], or [source] when there are none, into a page plus whether
/// the markup declared itself a Mews page. A page carrying the conformance
/// marker that still failed to parse has a real markup error; a page without it
/// is simply not Mews, and the two deserve different words.
///
/// Prefer [bytes]: the HTML parser reads the document's own `<meta charset>`
/// from them, while an already-decoded string has been through whatever charset
/// the HTTP header claimed.
({MewsPage page, bool declaresMews}) convertHtmlPage({
required String url,
String? source,
List<int>? bytes,
}) {
final document = html_parser.parse(bytes ?? source ?? '');
final declaresMews =
document.querySelector('meta[name="mews-profile"]') != null;
final root = document.documentElement;
final copied = root == null ? null : _copy(root).whereType<XmlElement>().firstOrNull;
if (copied == null) {
throw const MewsParseException('The markup has no root element.');
}
final page = parseMewsDocument(
XmlDocument([copied]),
url: url,
specVersionOverride: htmlSpecVersion,
);
return (page: _capped(page), declaresMews: declaresMews);
}
/// Copies an HTML element into the XML tree the Mews walk reads, as the nodes
/// that stand in for it: itself, or just its children where the element is in
/// the way, or nothing at all. Comments and the doctype are left behind, and
/// an element XML cannot name keeps its content the way an unknown wrapper
/// does. Beyond that the walk stays the place that decides what is shown.
List<XmlNode> _copy(dom.Element element) {
final name = element.localName;
if (name != null && _dropped.contains(name)) return const [];
final children = <XmlNode>[];
for (final node in element.nodes) {
if (node is dom.Element) {
children.addAll(_copy(node));
} else if (node is dom.Text) {
children.add(XmlText(node.text));
}
}
if (name == null || !_xmlName.hasMatch(name) || _unwrapped.contains(name)) {
return children;
}
final attributes = <String, String>{};
for (final entry in element.attributes.entries) {
final key = entry.key;
final attribute = key is dom.AttributeName ? key.name : key.toString();
// Last one wins, as a browser does; duplicates would be rejected here.
if (_xmlName.hasMatch(attribute)) attributes[attribute] = entry.value;
}
return [
XmlElement(
XmlName.qualified(name),
[for (final a in attributes.entries) XmlAttribute(XmlName.qualified(a.key), a.value)],
children,
)
];
}
MewsPage _capped(MewsPage page) {
if (page.body.length <= _maxBlocks) return page;
return MewsPage(
url: page.url,
body: page.body.sublist(0, _maxBlocks),
lang: page.lang,
direction: page.direction,
variant: page.variant,
entries: page.entries,
specVersion: page.specVersion,
title: page.title,
description: page.description,
author: page.author,
atomFeedHref: page.atomFeedHref,
);
}

72
app/lib/mews/parser.dart
View file

@ -26,7 +26,16 @@ MewsPage parseMewsPage({required String source, required String url}) {
} on XmlException catch (e) {
throw MewsParseException(e.toString());
}
return parseMewsDocument(doc, url: url);
}
/// Builds the page from an already-parsed document. A tolerant HTML parse
/// arrives here too (see html.dart), so the mapping below stays the one place
/// that decides what the Mews subset shows, whatever parsed the markup.
/// [specVersionOverride] stands in for the conformance marker a converted
/// document carries none of.
MewsPage parseMewsDocument(XmlDocument doc,
{required String url, String? specVersionOverride}) {
final html = doc.rootElement;
if (html.name.qualified != 'html') {
throw const MewsParseException('The root element is not html.');
@ -47,7 +56,7 @@ MewsPage parseMewsPage({required String source, required String url}) {
final dir = _dirOf(html) ?? _dirOf(bodyElement);
return MewsPage(
url: url,
specVersion: headData.specVersion,
specVersion: specVersionOverride ?? headData.specVersion,
title: headData.title,
description: headData.description,
author: headData.author,
@ -139,7 +148,12 @@ List<MewsEntry> _entriesOf(XmlElement body, String pageUrl) {
// Blocks --------------------------------------------------------------------
/// Elements whose whole subtree is not content and must never reach the model.
const _droppedElements = {'script', 'style', 'object', 'meta', 'link', 'head'};
const _droppedElements = {
'script', 'style', 'object', 'meta', 'link', 'head',
// Not content on the web either: markup the renderer has nothing to draw
// for, and controls that are interface rather than text.
'svg', 'iframe', 'noscript', 'template', 'canvas', 'audio', 'video', 'button',
};
void _appendBlocks(XmlElement element, String pageUrl, List<MewsBlock> out) {
switch (element.name.qualified) {
@ -170,7 +184,9 @@ void _appendBlocks(XmlElement element, String pageUrl, List<MewsBlock> out) {
case 'dl':
out.add(_definitionBlock(element, pageUrl));
case 'table':
out.add(_tableBlock(element, pageUrl));
final nested = <MewsBlock>[];
out.add(_tableBlock(element, pageUrl, nested));
out.addAll(nested);
case 'img':
out.add(_imageBlock(element, pageUrl));
case 'form':
@ -180,11 +196,11 @@ void _appendBlocks(XmlElement element, String pageUrl, List<MewsBlock> out) {
_appendBlocks(child, pageUrl, out);
}
default:
// Unknown or dropped wrappers: keep the readable content, drop the rest.
// Unknown wrappers: keep the readable content, drop the rest. Mixed
// content has to be handled here, not just element children, because
// text sits directly inside a div on most of the web.
if (!_droppedElements.contains(element.name.qualified)) {
for (final child in element.children.whereType<XmlElement>()) {
_appendBlocks(child, pageUrl, out);
}
out.addAll(_mixedBlocks(element, pageUrl));
}
}
}
@ -255,18 +271,31 @@ DefinitionListBlock _definitionBlock(XmlElement element, String pageUrl) {
return DefinitionListBlock(items);
}
TableBlock _tableBlock(XmlElement element, String pageUrl) {
/// The nearest table a node sits in, so a row can say which table owns it.
XmlElement? _enclosingTable(XmlNode node) => node.ancestors
.whereType<XmlElement>()
.where((e) => e.name.qualified == 'table')
.firstOrNull;
/// Builds the table, adding to [embedded] any table nested in one of its
/// cells. Tables cannot nest in the model (spec 4.2), so a nested one — old
/// web layout markup — becomes a table of its own rather than being flattened
/// into the cell that held it.
TableBlock _tableBlock(XmlElement element, String pageUrl,
[List<MewsBlock>? embedded]) {
final rows = <MewsTableRow>[];
for (final row in element.descendants
.whereType<XmlElement>()
.where((e) => e.name.qualified == 'tr')) {
// Rows are found among the descendants because thead, tbody and tfoot sit
// between, and an HTML parse inserts a tbody even where the markup has
// none. A row owned by a nested table belongs to that table, not this one.
for (final row in element.descendants.whereType<XmlElement>().where(
(e) => e.name.qualified == 'tr' && _enclosingTable(e) == element)) {
final cells = <MewsTableCell>[];
for (final cell in row.children.whereType<XmlElement>()) {
final isHeader = cell.name.qualified == 'th';
if (!isHeader && cell.name.qualified != 'td') continue;
cells.add(MewsTableCell(
header: isHeader,
inline: _inline(cell, pageUrl),
inline: _inline(cell, pageUrl, embedded),
colspan: int.tryParse(cell.getAttribute('colspan') ?? '') ?? 1,
rowspan: int.tryParse(cell.getAttribute('rowspan') ?? '') ?? 1,
));
@ -489,6 +518,11 @@ void _appendInline(XmlNode node, String pageUrl, List<MewsInline> out,
// Every image is shown in its own block (spec 4.2), so an inline image
// is lifted out of the paragraph that contains it.
embedded?.add(_imageBlock(node, pageUrl));
case 'table':
// Likewise a table, which cannot sit inside another (spec 4.2). Without
// this it would fall to the default below and its cells would run
// together as text.
embedded?.add(_tableBlock(node, pageUrl, embedded));
default:
if (!_droppedElements.contains(node.name.qualified)) {
for (final child in node.children) {
@ -497,17 +531,3 @@ void _appendInline(XmlNode node, String pageUrl, List<MewsInline> out,
}
}
}
/// Recovers readable text from a page that failed XML parsing, for the "show
/// its text" escape hatch (spec 8.1). Best effort by design.
String mewsPlainText(String source) {
final withoutBlocks = source.replaceAll(RegExp(r'<(script|style)[\s\S]*?</\1\s*>', caseSensitive: false), '');
final stripped = withoutBlocks.replaceAll(RegExp(r'<[^>]+>'), ' ');
return stripped
.replaceAll('&amp;', '&')
.replaceAll('&lt;', '<')
.replaceAll('&gt;', '>')
.replaceAll('&quot;', '"')
.replaceAll(RegExp(r'\s+'), ' ')
.trim();
}