the-mews/app/lib/mews/html.dart

/// Converts an ordinary web page into the Mews document model, so a page the
/// app cannot show as Mews can still be read in the structure Mews supports.
///
/// Mews is a strict subset of XHTML Mobile Profile 1.2 (mews-spec section 4),
/// so a web page's elements already are Mews elements plus extras. The work is
/// therefore only to parse tolerantly — real HTML is not well-formed XML, which
/// is what [parseMewsPage] requires (spec 7.1) — and hand the result to the
/// same walk. Elements outside the subset are unwrapped and keep their content;
/// markup that is not content at all is dropped by the walk itself.
library;

import 'package:html/dom.dart' as dom;
import 'package:html/parser.dart' as html_parser;
import 'package:xml/xml.dart';

import 'document.dart';
import 'parser.dart';

/// Stands in for the conformance marker a converted page has none of, so the
/// renderer applies reader presentation and can tell a simplified view apart.
/// Mirrors how gemtext.dart marks a converted capsule.
const htmlSpecVersion = 'html';

/// The whole page is built at once by the renderer rather than lazily, so a
/// very long web page is cut off instead of stalling the first frame.
const _maxBlocks = 2000;

/// Valid XML names, minus the colon: real pages carry attributes like
/// `@click` and `v-bind:href` that [XmlElement] will not accept.
final _xmlName = RegExp(r'^[A-Za-z_][A-Za-z0-9_.\-]*$');

/// A form on the web is not reading. The reader cannot usefully answer a
/// search box or a sign-in here, and submitting one would land them on another
/// page this client cannot show as Mews. The form itself is unwrapped rather
/// than dropped, so any real content inside it survives.
const _unwrapped = {'form'};

/// A form's controls, which have nothing to say once the form is gone.
const _dropped = {
  'input', 'select', 'textarea', 'label', 'fieldset', 'optgroup', 'option',
};

/// Converts [bytes], or [source] when there are none, into a page plus whether
/// the markup declared itself a Mews page. A page carrying the conformance
/// marker that still failed to parse has a real markup error; a page without it
/// is simply not Mews, and the two deserve different words.
///
/// Prefer [bytes]: the HTML parser reads the document's own `<meta charset>`
/// from them, while an already-decoded string has been through whatever charset
/// the HTTP header claimed.
({MewsPage page, bool declaresMews}) convertHtmlPage({
  required String url,
  String? source,
  List<int>? bytes,
}) {
  final document = html_parser.parse(bytes ?? source ?? '');
  final declaresMews =
      document.querySelector('meta[name="mews-profile"]') != null;

  final root = document.documentElement;
  final copied = root == null ? null : _copy(root).whereType<XmlElement>().firstOrNull;
  if (copied == null) {
    throw const MewsParseException('The markup has no root element.');
  }

  final page = parseMewsDocument(
    XmlDocument([copied]),
    url: url,
    specVersionOverride: htmlSpecVersion,
  );
  return (page: _capped(page), declaresMews: declaresMews);
}

/// Copies an HTML element into the XML tree the Mews walk reads, as the nodes
/// that stand in for it: itself, or just its children where the element is in
/// the way, or nothing at all. Comments and the doctype are left behind, and
/// an element XML cannot name keeps its content the way an unknown wrapper
/// does. Beyond that the walk stays the place that decides what is shown.
List<XmlNode> _copy(dom.Element element) {
  final name = element.localName;
  if (name != null && _dropped.contains(name)) return const [];

  final children = <XmlNode>[];
  for (final node in element.nodes) {
    if (node is dom.Element) {
      children.addAll(_copy(node));
    } else if (node is dom.Text) {
      children.add(XmlText(node.text));
    }
  }

  if (name == null || !_xmlName.hasMatch(name) || _unwrapped.contains(name)) {
    return children;
  }

  final attributes = <String, String>{};
  for (final entry in element.attributes.entries) {
    final key = entry.key;
    final attribute = key is dom.AttributeName ? key.name : key.toString();
    // Last one wins, as a browser does; duplicates would be rejected here.
    if (_xmlName.hasMatch(attribute)) attributes[attribute] = entry.value;
  }

  return [
    XmlElement(
      XmlName.qualified(name),
      [for (final a in attributes.entries) XmlAttribute(XmlName.qualified(a.key), a.value)],
      children,
    )
  ];
}

MewsPage _capped(MewsPage page) {
  if (page.body.length <= _maxBlocks) return page;
  return MewsPage(
    url: page.url,
    body: page.body.sublist(0, _maxBlocks),
    lang: page.lang,
    direction: page.direction,
    variant: page.variant,
    entries: page.entries,
    specVersion: page.specVersion,
    title: page.title,
    description: page.description,
    author: page.author,
    atomFeedHref: page.atomFeedHref,
  );
}