the-mews/app/lib/mews/parser.dart

import 'package:xml/xml.dart';

import 'document.dart';

/// Thrown when a page is not well-formed XML; the client must show a clear
/// error, never guess (spec 8.1).
class MewsParseException implements Exception {
  const MewsParseException(this.message);

  final String message;

  @override
  String toString() => message;
}

final _entryDatePattern = RegExp(r'^(\d{4}-\d{2}-\d{2}) (.+)$');

/// Parses a fetched document into a [MewsPage]. Pages are parsed as XML
/// whatever the MIME type (spec 7.1). Elements outside the Mews subset keep
/// their text content (so ordinary XHTML still renders) but scripts, styles
/// and objects are dropped entirely, and no author styling can enter the model.
MewsPage parseMewsPage({required String source, required String url}) {
  XmlDocument doc;
  try {
    doc = XmlDocument.parse(source);
  } on XmlException catch (e) {
    throw MewsParseException(e.toString());
  }

  final html = doc.rootElement;
  if (html.name.qualified != 'html') {
    throw const MewsParseException('The root element is not html.');
  }

  final head = html.findElements('head').firstOrNull;
  final bodyElement = html.findElements('body').firstOrNull;
  if (head == null || bodyElement == null) {
    throw const MewsParseException('The page has no head or no body.');
  }

  final headData = _parseHead(head);
  final blocks = <MewsBlock>[];
  for (final child in bodyElement.children.whereType<XmlElement>()) {
    _appendBlocks(child, url, blocks);
  }

  final dir = _dirOf(html) ?? _dirOf(bodyElement);
  return MewsPage(
    url: url,
    specVersion: headData.specVersion,
    title: headData.title,
    description: headData.description,
    author: headData.author,
    atomFeedHref: _feedHref(headData.atomFeedHref, url),
    lang: html.getAttribute('xml:lang') ??
        html.getAttribute('lang') ??
        bodyElement.getAttribute('lang') ??
        'en',
    direction: dir == 'rtl' ? TextDirection.rtl : TextDirection.ltr,
    variant: _variantOf(bodyElement.getAttribute('class')),
    entries: _entriesOf(bodyElement, url),
    body: blocks,
  );
}

class _HeadData {
  String? title;
  String? description;
  String? author;
  String? specVersion;
  String? atomFeedHref;
}

_HeadData _parseHead(XmlElement head) {
  final data = _HeadData();
  for (final element in head.children.whereType<XmlElement>()) {
    switch (element.name.qualified) {
      case 'title':
        data.title = element.innerText.trim();
      case 'meta':
        final name = element.getAttribute('name');
        if (name == 'mews-profile') {
          data.specVersion = element.getAttribute('content');
        } else if (name == 'description') {
          data.description = element.getAttribute('content');
        } else if (name == 'author') {
          data.author = element.getAttribute('content');
        }
      case 'link':
        final rel = element.getAttribute('rel');
        final type = element.getAttribute('type');
        if (rel == 'alternate' && type == 'application/atom+xml') {
          data.atomFeedHref = element.getAttribute('href');
        }
    }
  }
  return data;
}

String? _dirOf(XmlElement element) => element.getAttribute('dir');

MewsVariant _variantOf(String? classes) {
  switch (classes) {
    case 'mews-warm':
      return MewsVariant.warm;
    case 'mews-cool':
      return MewsVariant.cool;
    case 'mews-green':
      return MewsVariant.green;
    case 'mews-mono':
      return MewsVariant.mono;
    default:
      return MewsVariant.plain;
  }
}

List<MewsEntry> _entriesOf(XmlElement body, String pageUrl) {
  final entries = <MewsEntry>[];
  for (final ul in body.descendants.whereType<XmlElement>().where((e) =>
      e.name.qualified == 'ul' || e.name.qualified == 'ol')) {
    for (final li in ul.findElements('li')) {
      final link = li.children.whereType<XmlElement>().firstOrNull;
      if (link == null || link.name.qualified != 'a') continue;
      final match = _entryDatePattern.firstMatch(link.innerText.trim());
      if (match == null) continue;
      final date = DateTime.tryParse(match.group(1)!);
      if (date == null) continue;
      entries.add(MewsEntry(
        url: Uri.parse(pageUrl).resolve(link.getAttribute('href') ?? '').toString(),
        date: date,
        title: match.group(2)!.trim(),
      ));
    }
  }
  entries.sort((a, b) => b.date.compareTo(a.date));
  return entries;
}

// Blocks --------------------------------------------------------------------

/// Elements whose whole subtree is not content and must never reach the model.
const _droppedElements = {'script', 'style', 'object', 'meta', 'link', 'head'};

void _appendBlocks(XmlElement element, String pageUrl, List<MewsBlock> out) {
  switch (element.name.qualified) {
    case 'h1':
    case 'h2':
    case 'h3':
    case 'h4':
    case 'h5':
    case 'h6':
      out.add(HeadingBlock(int.parse(element.name.qualified[1]), _inline(element, pageUrl)));
    case 'p':
      final embedded = <MewsBlock>[];
      out.add(ParagraphBlock(_inline(element, pageUrl, embedded)));
      out.addAll(embedded);
    case 'blockquote':
      out.add(QuoteBlock(_mixedBlocks(element, pageUrl)));
    case 'pre':
      out.add(PreBlock(element.innerText));
    case 'address':
      final embedded = <MewsBlock>[];
      out.add(AddressBlock(_inline(element, pageUrl, embedded)));
      out.addAll(embedded);
    case 'hr':
      out.add(const RuleBlock());
    case 'ul':
    case 'ol':
      out.add(_listBlock(element, pageUrl));
    case 'dl':
      out.add(_definitionBlock(element, pageUrl));
    case 'table':
      out.add(_tableBlock(element, pageUrl));
    case 'img':
      out.add(_imageBlock(element, pageUrl));
    case 'form':
      out.add(_formBlock(element, pageUrl));
    case 'fieldset':
      for (final child in element.children.whereType<XmlElement>()) {
        _appendBlocks(child, pageUrl, out);
      }
    default:
      // Unknown or dropped wrappers: keep the readable content, drop the rest.
      if (!_droppedElements.contains(element.name.qualified)) {
        for (final child in element.children.whereType<XmlElement>()) {
          _appendBlocks(child, pageUrl, out);
        }
      }
  }
}

/// Inline-only elements; text and images directly inside these become part of
/// the enclosing paragraph's inline run.
const _inlineElements = {
  'a', 'br', 'em', 'strong', 'b', 'i', 'code', 'kbd', 'samp', 'var', 'q',
  'cite', 'abbr', 'dfn', 'img',
};

/// Builds blocks from mixed content: consecutive inline runs (text, links,
/// images) become paragraphs, block-level children append as blocks. Runs
/// that are nothing but markup whitespace never produce a paragraph, so
/// pretty-printed quotes and list items start without a blank line.
List<MewsBlock> _mixedBlocks(XmlElement element, String pageUrl) {
  final blocks = <MewsBlock>[];
  final embedded = <MewsBlock>[];
  final inline = <MewsInline>[];

  void flush() {
    _dropEdgeWhitespace(inline);
    if (inline.isNotEmpty) {
      blocks.add(ParagraphBlock(List.of(inline)));
      inline.clear();
    }
  }

  for (final node in element.children) {
    final isInlineNode = node is XmlText ||
        (node is XmlElement && _inlineElements.contains(node.name.qualified));
    if (isInlineNode) {
      _appendInline(node, pageUrl, inline, embedded);
    } else if (node is XmlElement) {
      flush();
      _appendBlocks(node, pageUrl, blocks);
    }
  }
  flush();
  blocks.addAll(embedded);
  return blocks;
}

ListBlock _listBlock(XmlElement element, String pageUrl) {
  final items = <MewsListItem>[];
  for (final li in element.findElements('li')) {
    items.add(MewsListItem(_mixedBlocks(li, pageUrl)));
  }
  return ListBlock(
    ordered: element.name.qualified == 'ol',
    start: int.tryParse(element.getAttribute('start') ?? '') ?? 1,
    items: items,
  );
}

DefinitionListBlock _definitionBlock(XmlElement element, String pageUrl) {
  final items = <DefinitionItem>[];
  List<MewsInline>? term;
  for (final child in element.children.whereType<XmlElement>()) {
    switch (child.name.qualified) {
      case 'dt':
        term = _inline(child, pageUrl);
      case 'dd':
        items.add(DefinitionItem(term ?? const [], _inline(child, pageUrl)));
        term = null;
    }
  }
  return DefinitionListBlock(items);
}

TableBlock _tableBlock(XmlElement element, String pageUrl) {
  final rows = <MewsTableRow>[];
  for (final row in element.descendants
      .whereType<XmlElement>()
      .where((e) => e.name.qualified == 'tr')) {
    final cells = <MewsTableCell>[];
    for (final cell in row.children.whereType<XmlElement>()) {
      final isHeader = cell.name.qualified == 'th';
      if (!isHeader && cell.name.qualified != 'td') continue;
      cells.add(MewsTableCell(
        header: isHeader,
        inline: _inline(cell, pageUrl),
        colspan: int.tryParse(cell.getAttribute('colspan') ?? '') ?? 1,
        rowspan: int.tryParse(cell.getAttribute('rowspan') ?? '') ?? 1,
      ));
    }
    rows.add(MewsTableRow(cells));
  }
  return TableBlock(
    rows: rows,
    caption: element.findElements('caption').firstOrNull?.innerText.trim(),
    summary: element.getAttribute('summary'),
  );
}

/// An image loads only from the page's own registered domain, never as a
/// data: URI (spec 4.2). Anything else shows its alt text instead.
ImageBlock _imageBlock(XmlElement element, String pageUrl) {
  final src = element.getAttribute('src') ?? '';
  final alt = element.getAttribute('alt') ?? '';
  final resolved = Uri.tryParse(pageUrl)?.resolve(src);
  final allowed = resolved != null &&
      !resolved.scheme.startsWith('data') &&
      _sameSite(resolved, Uri.parse(pageUrl));
  return ImageBlock(src: resolved?.toString() ?? src, alt: alt, allowed: allowed);
}

/// Rough registrable-domain match: the last two labels. Fine for the small
/// web; a public-suffix list is out of scope for v1.
/// The page's Atom feed as an absolute address (spec 6.2), or null when the
/// page declares none or points at another site: a client must not load
/// another site's resources without the reader acting (specs 7.4 and 8.1).
String? _feedHref(String? href, String pageUrl) {
  if (href == null || href.isEmpty) return null;
  final page = Uri.tryParse(pageUrl);
  if (page == null) return null;
  final resolved = page.resolve(href);
  if (resolved.scheme != 'http' && resolved.scheme != 'https') return null;
  return _sameSite(resolved, page) ? resolved.toString() : null;
}

bool _sameSite(Uri image, Uri page) {
  String? domain(Uri uri) {
    final host = uri.host;
    if (host.isEmpty) return null;
    final labels = host.split('.');
    return labels.length <= 2 ? host : labels.sublist(labels.length - 2).join('.');
  }

  final imageDomain = domain(image);
  return imageDomain != null && imageDomain == domain(page);
}

FormBlock _formBlock(XmlElement element, String pageUrl) {
  final labels = <String, String>{};
  final fields = <MewsFormField>[];

  for (final child in element.descendants.whereType<XmlElement>()) {
    switch (child.name.qualified) {
      case 'label':
        final text = child.innerText.trim();
        final forId = child.getAttribute('for');
        final controlId = child.children
            .whereType<XmlElement>()
            .map((e) => e.getAttribute('id'))
            .firstOrNull;
        if (forId != null) {
          labels[forId] = text;
        } else if (controlId != null) {
          labels[controlId] = text;
        }
      case 'input':
        fields.add(_inputField(child, labels[child.getAttribute('id')]));
      case 'select':
        final options = <MewsSelectOption>[];
        for (final option in child.descendants.whereType<XmlElement>().where(
            (e) => e.name.qualified == 'option')) {
          options.add(MewsSelectOption(
            value: option.getAttribute('value') ?? option.innerText.trim(),
            selected: option.getAttribute('selected') != null,
          ));
        }
        fields.add(MewsSelectField(
          name: child.getAttribute('name') ?? '',
          options: options,
          label: labels[child.getAttribute('id')],
        ));
      case 'textarea':
        fields.add(MewsTextAreaField(
          name: child.getAttribute('name') ?? '',
          value: child.innerText,
          rows: int.tryParse(child.getAttribute('rows') ?? '') ?? 3,
          label: labels[child.getAttribute('id')],
        ));
    }
  }

  String? submitLabel;
  for (final field in fields.whereType<MewsSubmitField>()) {
    submitLabel ??= field.label;
  }
  final action = element.getAttribute('action') ?? pageUrl;
  return FormBlock(
    action: Uri.parse(pageUrl).resolve(action).toString(),
    method: (element.getAttribute('method') ?? 'get').toLowerCase(),
    fields: fields,
    submitLabel: submitLabel,
  );
}

MewsFormField _inputField(XmlElement element, String? label) {
  final name = element.getAttribute('name') ?? '';
  final value = element.getAttribute('value') ?? '';
  final checked = element.getAttribute('checked') != null;
  switch (element.getAttribute('type') ?? 'text') {
    case 'password':
      return MewsTextField(name: name, password: true, value: value, label: label);
    case 'hidden':
      return MewsHiddenField(name: name, value: value);
    case 'checkbox':
      return MewsCheckboxField(name: name, checked: checked, value: value.isEmpty ? 'on' : value, label: label);
    case 'radio':
      return MewsRadioField(name: name, checked: checked, value: value, label: label);
    case 'submit':
      return MewsSubmitField(reset: false, label: value.isEmpty ? null : value, name: name);
    case 'reset':
      return MewsSubmitField(reset: true, label: value.isEmpty ? null : value, name: name);
    default:
      return MewsTextField(
        name: name,
        password: false,
        value: value,
        maxLength: int.tryParse(element.getAttribute('maxlength') ?? ''),
        inputMode: element.getAttribute('inputmode'),
        label: label,
      );
  }
}

// Inline ---------------------------------------------------------------------

/// Pretty-printed markup leaves whitespace runs between tags; browsers treat
/// them as a single space, and whitespace-only runs at the edges of inline
/// content as nothing at all.
final _whitespaceRun = RegExp(r'\s+');

List<MewsInline> _inline(XmlElement element, String pageUrl, [List<MewsBlock>? embedded]) {
  final out = <MewsInline>[];
  for (final child in element.children) {
    _appendInline(child, pageUrl, out, embedded);
  }
  _dropEdgeWhitespace(out);
  return out;
}

/// Trims markup whitespace from the edges of a run: whitespace-only nodes are
/// dropped, and the first/last text node loses its leading/trailing spaces,
/// so indented markup renders no blank first or last lines. Interior
/// whitespace stays: it separates inline elements the author typed apart.
void _dropEdgeWhitespace(List<MewsInline> out) {
  bool blank(MewsInline node) => node is MewsText && node.text.trim().isEmpty;
  while (out.isNotEmpty && blank(out.first)) {
    out.removeAt(0);
  }
  while (out.isNotEmpty && blank(out.last)) {
    out.removeLast();
  }
  if (out.isNotEmpty && out.first is MewsText) {
    final text = (out.first as MewsText).text.trimLeft();
    if (text.isEmpty) {
      out.removeAt(0);
    } else {
      out[0] = MewsText(text);
    }
  }
  if (out.isNotEmpty && out.last is MewsText) {
    final text = (out.last as MewsText).text.trimRight();
    if (text.isEmpty) {
      out.removeLast();
    } else {
      out[out.length - 1] = MewsText(text);
    }
  }
}

void _appendInline(XmlNode node, String pageUrl, List<MewsInline> out,
    [List<MewsBlock>? embedded]) {
  if (node is XmlText) {
    final text = node.value.replaceAll(_whitespaceRun, ' ');
    if (text.isNotEmpty) out.add(MewsText(text));
    return;
  }
  if (node is! XmlElement) return;

  switch (node.name.qualified) {
    case 'a':
      final href = node.getAttribute('href') ?? '';
      out.add(MewsLink(Uri.parse(pageUrl).resolve(href).toString(), _inline(node, pageUrl, embedded)));
    case 'br':
      out.add(const MewsLineBreak());
    case 'em':
    case 'i':
    case 'cite':
    case 'abbr':
      out.add(MewsEmphasis(EmphasisStyle.emphasis, _inline(node, pageUrl, embedded)));
    case 'strong':
    case 'b':
      out.add(MewsEmphasis(EmphasisStyle.strong, _inline(node, pageUrl, embedded)));
    case 'code':
      out.add(MewsEmphasis(EmphasisStyle.code, _inline(node, pageUrl, embedded)));
    case 'kbd':
      out.add(MewsEmphasis(EmphasisStyle.sample, _inline(node, pageUrl, embedded)));
    case 'samp':
      out.add(MewsEmphasis(EmphasisStyle.sample, _inline(node, pageUrl, embedded)));
    case 'var':
      out.add(MewsEmphasis(EmphasisStyle.variable, _inline(node, pageUrl, embedded)));
    case 'dfn':
      out.add(MewsEmphasis(EmphasisStyle.term, _inline(node, pageUrl, embedded)));
    case 'q':
      out.add(MewsEmphasis(EmphasisStyle.quote, _inline(node, pageUrl, embedded)));
    case 'img':
      // Every image is shown in its own block (spec 4.2), so an inline image
      // is lifted out of the paragraph that contains it.
      embedded?.add(_imageBlock(node, pageUrl));
    default:
      if (!_droppedElements.contains(node.name.qualified)) {
        for (final child in node.children) {
          _appendInline(child, pageUrl, out, embedded);
        }
      }
  }
}

/// Recovers readable text from a page that failed XML parsing, for the "show
/// its text" escape hatch (spec 8.1). Best effort by design.
String mewsPlainText(String source) {
  final withoutBlocks = source.replaceAll(RegExp(r'<(script|style)[\s\S]*?</\1\s*>', caseSensitive: false), '');
  final stripped = withoutBlocks.replaceAll(RegExp(r'<[^>]+>'), ' ');
  return stripped
      .replaceAll('&amp;', '&')
      .replaceAll('&lt;', '<')
      .replaceAll('&gt;', '>')
      .replaceAll('&quot;', '"')
      .replaceAll(RegExp(r'\s+'), ' ')
      .trim();
}